diff options
| author | Sergey G. Brester <serg.brester@sebres.de> | 2023-03-23 12:01:50 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-03-23 12:01:50 +0100 |
| commit | 2c0360d1788d6569c6274ac690799d2a92c459df (patch) | |
| tree | ce0742af14e84f84ba8cf4c500d8ea9b75e175b2 | |
| parent | 7e88ae0ee66628893a283d6fed06a347f9f6673e (diff) | |
| parent | d1d1730de01de45820db062d811d9b91f261ea83 (diff) | |
| download | fail2ban-2c0360d1788d6569c6274ac690799d2a92c459df.tar.gz | |
Merge branch 'master' into nginx-forbidden
241 files changed, 9989 insertions, 3983 deletions
diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..0cbdbf83 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +ChangeLog linguist-language=Markdown diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..543f316a --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +github: [sebres] +custom: [paypal.me/sebres] diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index cb4b4bc6..00000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,49 +0,0 @@ -_We will be very grateful, if your problem was described as completely as possible, -enclosing excerpts from logs (if possible within DEBUG mode, if no errors evident -within INFO mode), and configuration in particular of effected relevant settings -(e.g., with ` fail2ban-client -d | grep 'affected-jail-name' ` for a particular -jail troubleshooting). -Thank you in advance for the details, because such issues like "It does not work" -alone could not help to resolve anything! -Thanks! (remove this paragraph and other comments upon reading)_ - -### Environment: - -_Fill out and check (`[x]`) the boxes which apply. If your Fail2Ban version is outdated, -and you can't verify that the issue persists in the recent release, better seek support -from the distribution you obtained Fail2Ban from_ - -- Fail2Ban version (including any possible distribution suffixes): -- OS, including release name/version: -- [ ] Fail2Ban installed via OS/distribution mechanisms -- [ ] You have not applied any additional foreign patches to the codebase -- [ ] Some customizations were done to the configuration (provide details below is so) - -### The issue: - -_Summary here_ - -#### Steps to reproduce - -#### Expected behavior - -#### Observed behavior - -#### Any additional information - -### Configuration, dump and another helpful excerpts - -#### Any customizations done to /etc/fail2ban/ configuration -``` -``` - -#### Relevant parts of /var/log/fail2ban.log file: -_preferably obtained while running fail2ban with `loglevel = 4`_ - -``` -``` - -#### Relevant lines from monitored log files in question: - -``` -```
\ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..33d94e10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,70 @@ +--- +name: Bug report +about: Report a bug within the fail2ban engines (not filters or jails) +title: '[BR]: ' +labels: bug +assignees: '' + +--- + +<!-- + - Before reporting, please make sure to search the open and closed issues for any reports in the past. + - Use this issue template to report a bug in the fail2ban engine (not in a filter or jail). + - If you want to request a feature or a new filter, please use "Feature request" or "Filter request" instead. + - If you have rather some question, please open or join to some discussion. + + We will be very grateful, if your problem was described as completely as possible, + enclosing excerpts from logs (if possible within DEBUG mode, if no errors evident + within INFO mode), and configuration in particular of effected relevant settings + (e.g., with ` fail2ban-client -d | grep 'affected-jail-name' ` for a particular + jail troubleshooting). + Thank you in advance for the details, because such issues like "It does not work" + alone could not help to resolve anything! + Thanks! + (you can remove this paragraph and other comments upon reading) +--> + +### Environment: + +<!-- + Fill out and check (`[x]`) the boxes which apply. If your Fail2Ban version is outdated, + and you can't verify that the issue persists in the recent release, better seek support + from the distribution you obtained Fail2Ban from +--> + +- Fail2Ban version <!-- including any possible distribution suffixes --> : +- OS, including release name/version : +- [ ] Fail2Ban installed via OS/distribution mechanisms +- [ ] You have not applied any additional foreign patches to the codebase +- [ ] Some customizations were done to the configuration (provide details below is so) + +### The issue: + +<!-- summary here --> + +#### Steps to reproduce + +#### Expected behavior + +#### Observed behavior + +#### Any additional information + + +### Configuration, dump and another helpful excerpts + +#### Any customizations done to /etc/fail2ban/ configuration +<!-- put your configuration excerpts between next 2 lines --> +``` +``` + +#### Relevant parts of /var/log/fail2ban.log file: +<!-- preferably obtained while running fail2ban with `loglevel = 4` --> +<!-- put your log excerpt between next 2 lines --> +``` +``` + +#### Relevant lines from monitored log files: +<!-- put your log excerpt between next 2 lines --> +``` +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..41812e82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,35 @@ +--- +name: Feature request +about: Suggest an idea or an enhancement for this project +title: '[RFE]: ' +labels: enhancement +assignees: '' + +--- + +<!-- + - Before requesting, please make sure to search the open and closed issues for any requests in the past. + - Use this issue template to request a feature in the fail2ban engine (not a new filter or jail). + - If you want to request a new filter or failregex, please use "Filter request" instead. + - If you have rather some question, please open or join to some discussion. +--> + +#### Feature request type +<!-- + Please provide a summary description of the feature request. +--> + +#### Description +<!-- + Please describe the feature in more detail. +--> + +#### Considered alternatives +<!-- + A clear and concise description of any alternative solutions or features you've considered. +--> + +#### Any additional information +<!-- + Add any other context or screenshots about the feature request here. +--> diff --git a/.github/ISSUE_TEMPLATE/filter_request.md b/.github/ISSUE_TEMPLATE/filter_request.md new file mode 100644 index 00000000..caf02f90 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/filter_request.md @@ -0,0 +1,59 @@ +--- +name: Filter request +about: Request a new jail or filter to be supported or existing filter extended with new failregex +title: '[FR]: ' +labels: filter-request +assignees: '' + +--- + +<!-- + - Before requesting, please make sure to search the open and closed issues for any requests in the past. + - Sometimes failregex have been already requested before but are not implemented yet due to various reasons. + - If there are no hits for your concerns, please proceed otherwise add a comment to the related issue (also if it is closed). + - If you want to request a new feature, please use "Feature request" instead. + - If you have rather some question, please open or join to some discussion. +--> + +### Environment: + +<!-- + Fill out and check (`[x]`) the boxes which apply. +--> + +- Fail2Ban version <!-- including any possible distribution suffixes --> : +- OS, including release name/version : + +#### Service, project or product which log or journal should be monitored + +- Name of filter or jail in Fail2Ban (if already exists) : +- Service, project or product name, including release name/version : +- Repository or URL (if known) : +- Service type : +- Ports and protocols the service is listening : + +#### Log or journal information +<!-- Delete unrelated group --> + +<!-- Log file --> + +- Log file name(s) : + +<!-- Systemd journal --> + +- Journal identifier or unit name : + +#### Any additional information + + +### Relevant lines from monitored log files: + +#### failures in sense of fail2ban filter (fail2ban must match): +<!-- put your log excerpt between next 2 lines --> +``` +``` + +#### legitimate messages (fail2ban should not consider as failures): +<!-- put your log excerpt between next 2 lines --> +``` +``` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3a17ccc2..350d6ee2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,8 @@ Before submitting your PR, please review the following checklist: - [ ] **CHOOSE CORRECT BRANCH**: if filing a bugfix/enhancement - against 0.9.x series, choose `master` branch + against certain release version, choose `0.9`, `0.10` or `0.11` branch, + for dev-edition use `master` branch - [ ] **CONSIDER adding a unit test** if your PR resolves an issue - [ ] **LIST ISSUES** this PR resolves - [ ] **MAKE SURE** this PR doesn't break existing tests diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..e681e417 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,100 @@ +name: CI + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + pull_request: + paths-ignore: + - 'doc/**' + - 'files/**' + - 'man/**' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ubuntu-20.04 + strategy: + matrix: + python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11.0-beta.3', pypy2, pypy3] + fail-fast: false + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Grant systemd-journal access + run: sudo usermod -a -G systemd-journal "$USER" || echo 'no systemd-journal access' + + - name: Python version + run: | + F2B_PY=$(python -c "import sys; print(sys.version)") + echo "Python: ${{ matrix.python-version }} -- ${F2B_PY/$'\n'/ }" + F2B_PYV=$(echo "${F2B_PY}" | grep -oP '^\d+(?:\.\d+)') + F2B_PY=${F2B_PY:0:1} + echo "Set F2B_PY=$F2B_PY, F2B_PYV=$F2B_PYV" + echo "F2B_PY=$F2B_PY" >> $GITHUB_ENV + echo "F2B_PYV=$F2B_PYV" >> $GITHUB_ENV + # for GHA we need to monitor all journals, since it cannot be found using SYSTEM_ONLY(4): + echo "F2B_SYSTEMD_DEFAULT_FLAGS=0" >> $GITHUB_ENV + + - name: Install dependencies + run: | + if [[ "$F2B_PY" = 3 ]]; then python -m pip install --upgrade pip || echo "can't upgrade pip"; fi + if [[ "$F2B_PY" = 3 ]] && ! command -v 2to3x -v 2to3 > /dev/null; then + #pip install 2to3 + sudo apt-get -y install 2to3 + fi + #sudo apt-get -y install python${F2B_PY/2/}-pyinotify || echo 'inotify not available' + python -m pip install pyinotify || echo 'inotify not available' + #sudo apt-get -y install python${F2B_PY/2/}-systemd || echo 'systemd not available' + sudo apt-get -y install libsystemd-dev || echo 'systemd dependencies seems to be unavailable' + python -m pip install systemd-python || echo 'systemd not available' + #readline if available as module: + python -c 'import readline' 2> /dev/null || python -m pip install readline || echo 'readline not available' + + - name: Before scripts + run: | + cd "$GITHUB_WORKSPACE" + # Manually execute 2to3 for now + if [[ "$F2B_PY" = 3 ]]; then echo "2to3 ..." && ./fail2ban-2to3; fi + _debug() { echo -n "$1 "; err=$("${@:2}" 2>&1) && echo 'OK' || echo -e "FAIL\n$err"; } + # (debug) output current preferred encoding: + _debug 'Encodings:' python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' + # (debug) backend availabilities: + echo 'Backends:' + _debug '- systemd:' python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + #_debug '- systemd (root): ' sudo python -c 'from fail2ban.server.filtersystemd import FilterSystemd' + _debug '- pyinotify:' python -c 'from fail2ban.server.filterpyinotify import FilterPyinotify' + + - name: Test suite + run: | + if [[ "$F2B_PY" = 2 ]]; then + python setup.py test + elif dpkg --compare-versions "$F2B_PYV" lt 3.10; then + python bin/fail2ban-testcases --verbosity=2 + else + echo "Skip systemd backend since systemd-python module must be fixed for python >= v.3.10 in GHA ..." + python bin/fail2ban-testcases --verbosity=2 -i "[sS]ystemd|[jJ]ournal" + fi + + #- name: Test suite (debug some systemd tests only) + #run: python bin/fail2ban-testcases --verbosity=2 "[sS]ystemd|[jJ]ournal" + #run: python bin/fail2ban-testcases --verbosity=2 -l 5 "test_WrongChar" + + - name: Build + run: python setup.py build + + #- name: Test initd scripts + # run: shellcheck -s bash -e SC1090,SC1091 files/debian-initd @@ -10,3 +10,4 @@ htmlcov __pycache__ .vagrant/ .idea/ +.venv/ diff --git a/.travis.yml b/.travis.yml index 16a71249..502af5be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,24 @@ # vim ft=yaml # travis-ci.org definition for Fail2Ban build # https://travis-ci.org/fail2ban/fail2ban/ + +#os: linux + language: python -python: - - 2.6 - - 2.7 - - pypy - # disabled until coverage module fixes up compatibility issue - # - 3.2 - - 3.3 - - 3.4 - - 3.5 - - 3.6 - - 3.7-dev - # disabled since setuptools dropped support for Python 3.0 - 3.2 - # - pypy3 - - pypy3.3-5.5-alpha +dist: xenial + +matrix: + fast_finish: true + include: + - python: 2.7 + #- python: pypy + - python: 3.4 + - python: 3.5 + - python: 3.6 + - python: 3.7 + - python: 3.8 + - python: 3.9-dev + - python: pypy3.5 before_install: - echo "running under $TRAVIS_PYTHON_VERSION" - if [[ $TRAVIS_PYTHON_VERSION == 2* || $TRAVIS_PYTHON_VERSION == pypy* && $TRAVIS_PYTHON_VERSION != pypy3* ]]; then export F2B_PY=2; fi @@ -29,18 +32,29 @@ install: # Install Python packages / dependencies # coverage - travis_retry pip install coverage - # coveralls - - travis_retry pip install coveralls codecov + # coveralls (note coveralls doesn't support 2.6 now): + #- if [[ $TRAVIS_PYTHON_VERSION != 2.6* ]]; then F2B_COV=1; else F2B_COV=0; fi + - F2B_COV=1 + - if [[ "$F2B_COV" = 1 ]]; then travis_retry pip install coveralls; fi + # codecov: + - travis_retry pip install codecov # dnspython or dnspython3 - - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython; fi - - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3; fi + - if [[ "$F2B_PY" = 2 ]]; then travis_retry pip install dnspython || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry pip install dnspython3 || echo 'not installed'; fi + # python systemd bindings: + - if [[ "$F2B_PY" = 2 ]]; then travis_retry sudo apt-get install -qq python-systemd || echo 'not installed'; fi + - if [[ "$F2B_PY" = 3 ]]; then travis_retry sudo apt-get install -qq python3-systemd || echo 'not installed'; fi # gamin - install manually (not in PyPI) - travis-ci system Python is 2.7 - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then (travis_retry sudo apt-get install -qq python-gamin && cp /usr/share/pyshared/gamin.py /usr/lib/pyshared/python2.7/_gamin.so $VIRTUAL_ENV/lib/python2.7/site-packages/) || echo 'not installed'; fi # pyinotify - - travis_retry pip install pyinotify + - travis_retry pip install pyinotify || echo 'not installed' + # Install helper tools + - sudo apt-get install shellcheck before_script: # Manually execute 2to3 for now - if [[ "$F2B_PY" = 3 ]]; then ./fail2ban-2to3; fi + # (debug) output current preferred encoding: + - python -c 'import locale, sys; from fail2ban.helpers import PREFER_ENC; print(PREFER_ENC, locale.getpreferredencoding(), (sys.stdout and sys.stdout.encoding))' script: # Keep the legacy setup.py test approach of checking coverage for python2 - if [[ "$F2B_PY" = 2 ]]; then coverage run setup.py test; fi @@ -48,13 +62,14 @@ script: - if [[ "$F2B_PY" = 3 ]]; then coverage run bin/fail2ban-testcases --verbosity=2; fi # Use $VENV_BIN (not python) or else sudo will always run the system's python (2.7) - sudo $VENV_BIN/pip install . - # Doc files should get installed on Travis under Linux - - test -e /usr/share/doc/fail2ban/FILTERS + # Doc files should get installed on Travis under Linux (some builds/python's seem to use another path segment) + - test -e /usr/share/doc/fail2ban/FILTERS && echo 'found' || echo 'not found' + # Test initd script + - shellcheck -s bash -e SC1090,SC1091 files/debian-initd after_success: - - coveralls + - if [[ "$F2B_COV" = 1 ]]; then coveralls; fi - codecov -matrix: - fast_finish: true + # Might be worth looking into #notifications: # email: true @@ -1,3 +1,4 @@ +<!-- vim: syntax=Markdown --> __ _ _ ___ _ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ @@ -6,32 +7,296 @@ Fail2Ban: Changelog =================== -Incompatibility list (compared to v.0.9): +ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition ----------- -* Filter (or `failregex`) internal capture-groups: +### Fixes +* circumvent SEGFAULT in a python's socket module by getaddrinfo with disabled IPv6 (gh-3438) +* `action.d/cloudflare-token.conf` - fixes gh-3479, url-encode args by unban + +### New Features and Enhancements +* better auto-detection for IPv6 support (`allowipv6 = auto` by default), trying to check sysctl net.ipv6.conf.all.disable_ipv6 + (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces + if available for platform and uses DNS to find local IPv6 as a fallback only +* improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132) + + +ver. 1.0.2 (2022/11/09) - finally-war-game-test-tape-not-a-nuclear-alarm +----------- + +### Fixes +* backend `systemd`: code review and several fixes: + - wait only if it is necessary, e. g. in operational mode and if no more entries retrieved (end of journal); + - ensure we give enough time after possible rotation, vacuuming or adding/removing journal files, + and move cursor back and forth to avoid entering dead space +* `filter.d/named-refused.conf`: + - support BIND named log categories, gh-3388 + - allow `info:` as possible error prefix too ("query (cache) denied" may occur as info) +* `filter.d/dovecot.conf`: + - fixes regression introduced in gh-3210: resolve extremely long search by repeated apply of non-greedy RE-part + with following branches (it may be extremely slow up to infinite search depending on message), gh-3370 + - fixes regression and matches new format in aggressive mode too (amend to gh-3210) + +### New Features and Enhancements + + +ver. 1.0.1 (2022/09/27) - energy-equals-mass-times-the-speed-of-light-squared +----------- - - If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should - rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)` - (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings). +### Compatibility +* the minimum supported python version is now 2.7, if you have previous python version + you can use the 0.11 version of fail2ban or upgrade python (or even build it from source). +* potential incompatibility by parsing of options of `backend`, `filter` and `action` parameters (if they + are partially incorrect), because fail2ban could throw an error now (doesn't silently bypass it anymore). +* due to fix for CVE-2021-32749 (GHSA-m985-3f3v-cwmm) the mailing action using mailutils may require extra configuration, + if it is not compatible or doesn't support `-E 'set escape'` (e. g. with `mailcmd` parameter), see gh-3059 +* automatic invocation of 2to3 is removed in setup now (gh-3098), there is also no option `--disable-2to3` anymore, + `./fail2ban-2to3` should be called outside before setup +* to v.0.11: + - due to change of `actioncheck` behavior (gh-488), some actions can be incompatible as regards + the invariant check, if `actionban` or `actionunban` would not throw an error (exit code + different from 0) in case of unsane environment. + - actions that have used tag `<ip>` (instead of `<fid>` or `<F-ID>`) to get failure-ID may become + incompatible, if filter uses IP-related tags (like `<ADDR>` or `<HOST>`) additionally to `<F-ID>` + and the values are different (gh-3217) - Of course you can always define your own capture-group (like below `_cond_ip_`) to do this. - ``` - testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1" - fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$" - ``` - - New internal groups (currently reserved for internal usage): - `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if - mapping from tag `<F-*>` used in failregex (e. g. `user` by `<F-USER>`). +### Fixes +* theoretical RCE vulnerability in mailing action using mailutils (mail-whois), CVE-2021-32749, GHSA-m985-3f3v-cwmm +* readline fixed to consider interim new-line character as part of code point in multi-byte logs + (e. g. unicode encoding like utf-16be, utf-16le); +* [stability] solves race condition with uncontrolled growth of failure list (jail with too many matches, + that did not cause ban), behavior changed to ban ASAP, gh-2945 +* fixes search for the best datepattern - e. g. if line is too short, boundaries check for previously known + unprecise pattern may fail on incomplete lines (logging break-off, no flush, etc), gh-3020 +* [stability, performance] backend `systemd`: + - fixes error "local variable 'line' referenced before assignment", introduced in 55d7d9e2, gh-3097 + - don't update database too often (every 10 ticks or ~ 10 seconds in production) + - fixes wrong time point of "in operation" mode, gh-2882 + - better avoidance of landing in dead space by seeks over journals (improved seek to time) + - fixes missing space in message (tag `<matches>`) between timestamp and host if the message read from systemd journal, gh-3293 +* [stability] backend `pyinotify`: fixes sporadic runtime error "dictionary changed size during iteration" +* several backends optimizations (in file and journal filters): + - don't need to wait if we still had log-entries from last iteration (which got interrupted for servicing) + - rewritten update log/journal position, it is more stable and faster now (fewer DB access and surely up-to-date at end) +* `paths-debian.conf`: + - add debian path to roundcube error logs +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-`;" + reverted the incompatibility gh-3047 introduced in a038fd5, gh-2821, because this depends now on firewalld backend + (e. g. `-` vs. `:` related to `iptables` vs. `nftables`) +* `action.d/nginx-block-map.conf`: reload nginx only if it is running (also avoid error in nginx-errorlog, gh-2949) +* `action.d/ufw.conf`: + - fixed handling on IPv6 (using prepend, gh-2331, gh-3018) + - application names containing spaces can be used now (gh-656, gh-1532, gh-3018) +* `filter.d/apache-fakegooglebot.conf`: + - better, more precise regex and datepattern (closes possible weakness like gh-3013) + - `filter.d/ignorecommands/apache-fakegooglebot` - added timeout parameter (default 55 seconds), avoid fail with timeout + (default 1 minute) by reverse lookup on some slow DNS services (googlebots must be resolved fast), gh-2951 +* `filter.d/apache-overflows.conf` - extended to match AH00126 error (Invalid URI ...), gh-2908 +* `filter.d/asterisk.conf` - add transport to asterisk RE: call rejection messages can have the transport prefixed to the IP address, gh-2913 +* `filter.d/courier-auth.conf`: + - consider optional port after IP, gh-3211 + - regex is rewritten without catch-all's and right anchor, so it is more stable against further modifications now +* `filter.d/dovecot.conf`: + - adjusted for updated dovecot log format with `read(size=...)` in message (gh-3210) + - parse everything in parenthesis by auth-worker info, e. g. can match (pid=...,uid=...) too (amend to gh-2553) + - extended to match prefix like `conn unix:auth-worker (uid=143): auth-worker<13247>:` + (authenticate from external service like exim), gh-2553 + - fixed "Authentication failure" regex, matches "Password mismatch" in title case (gh-2880) +* `filter.d/drupal-auth.conf` - more strict regex, extended to match "Login attempt failed from" (gh-2742) +* `filter.d/exim-common.conf` - pid-prefix extended to match `mx1 exim[...]:` (gh-2553) +* `filter.d/lighttpd-auth.conf` - adjusted to the current source code + avoiding catch-all's, etc (gh-3116) +* `filter.d/named-refused.conf`: + - added support for alternate names (suffix), FreeIPA renames the BIND9 named daemon to named-pkcs11, gh-2636 + - fixes prefix for messages from systemd journal (no mandatory space ahead, because don't have timestamp), gh-2899 +* `filter.d/nginx-*.conf` - added journalmatch to nginx filters, gh-2935 +* `filter.d/nsd.conf` - support for current log format, gh-2965 +* `filter.d/postfix.conf`: fixes and new vectors, review and combining several regex to single RE: + - mode `ddos` (and `aggressive`) extended: + * to consider abusive handling of clients hitting command limit, gh-3040 + * to handle postscreen's PREGREET and HANGUP messages, gh-2898 + - matches rejects with "undeliverable address" (sender/recipient verification) additionally to "Unknown user", gh-3039 + both are configurable now via extended parameter and can be disabled using `exre-user=` supplied in filter parameters + - reject: BDAT/DATA from, gh-2927 + - (since regex is more precise now) token selector changed to `[A-Z]{4}`, e. g. no matter what a command is supplied now + (RCPT, EHLO, VRFY, DATA, BDAT or something else) + - matches "Command rejected" and "Data command rejected" now + - matches RCPT from unknown, 504 5.5.2, need fully-qualified hostname, gh-2995 + - matches 550 5.7.25 Client host rejected, gh-2996 +* `filter.d/sendmail-auth.conf`: + - detect several "authentication failure" messages, sendmail 8.16.1, gh-2757 + - detect user not found, gh-3030 + - detect failures without user part, gh-3324 +* `filter.d/sendmail-reject.conf`: + - fix reverse DNS for ... (gh-3012) + - fixed regex to consider "Connection rate limit exceeded" with different combination of arguments +* `filter.d/sshd.conf`: + - mode `ddos` extended - recognizes messages "kex_exchange_identification: Connection closed / reset by pear", gh-3086 + (fixed possible regression of f77398c) + - mode `ddos` extended - recognizes new message "banner exchange: invalid format" generated by port scanner + (https payload on ssh port), gh-3169 +* `filter.d/zoneminder.conf` - support new log format (ERR instead of WAR), add detection of non-existent user login attempts, gh-2984 +* amend to gh-980 fixing several actions (correctly supporting new enhancements now) +* fixed typo by `--dump-pretty` option which did never work (only `--dp` was working) +* fixes start of fail2ban-client in docker: speedup daemonization process by huge open files limit, gh-3334 +* provides details of failed regex compilation in the error message we throw in Regex-constructor + (it's good to know what exactly is wrong) +* fixed failed update of database didn't signal with an error, gh-3352: + - client and server exit with error code by failure during start process (in foreground mode) + - added fallback to repair if database cannot be upgraded + +### New Features and Enhancements +* python 3.10 and 3.11 compatibility (and GHA-CI support) +* `actioncheck` behavior is changed now (gh-488), so invariant check as well as restore or repair + of sane environment (in case of recognized unsane state) would only occur on action errors (e. g. + if ban or unban operations are exiting with other code as 0) +* better recognition of log rotation, better performance by reopen: avoid unnecessary seek to begin of file + (and hash calculation) +* file filter reads only complete lines (ended with new-line) now, so waits for end of line (for its completion) +* datedetector: + - token `%Z` must recognize zone abbreviation `Z` (GMT/UTC) also (similar to `%z`) + - token `%Z` recognizes all known zone abbreviation besides Z, GMT, UTC correctly, if it is matching + (`%z` remains unchanged for backwards-compatibility, see comment in code) + - date patterns `%ExY` and `%Exy` accept every year from 19xx up to current century (+3 years) in `fail2ban-regex` + - better grouping algorithm for resulting century RE for `%ExY` and `%Exy` +* actions differentiate tags `<ip>` and `<fid>` (`<F-ID>`), if IP-address deviates from ID then the value + of `<ip>` is not equal `<fid>` anymore (gh-3217) +* action info extended with new members for jail info (usable as tags in command actions), gh-10: + - `<jail.found>`, `<jail.found_total>` - current and total found failures + - `<jail.banned>`, `<jail.banned_total>` - current and total bans +* `filter.d/monitorix.conf` - added new filter and jail for Monitorix, gh-2679 +* `filter.d/mssql-auth.conf` - new filter and jail for Microsoft SQL Server, gh-2642 +* `filter.d/nginx-bad-request.conf` - added filter to find bad requests (400), gh-2750 +* `filter.d/nginx-http-auth.conf` - extended with parameter mode, so additionally to `auth` (or `normal`) + mode `fallback` (or combined as `aggressive`) can find SSL errors while SSL handshaking, gh-2881 +* `filter.d/scanlogd.conf` - new filter and jail, add support for filtering out detected port scans via scanlogd, gh-2950 +* `action.d/apprise.conf` - added Apprise support (50+ Notifications), gh-2565 +* `action.d/badips.*` - removed actions, badips.com is no longer active, gh-2889 +* `action.d/cloudflare.conf` - better IPv6 capability, gh-2891 +* `action.d/cloudflare-token.conf` - added support for Cloudflare Token APIs. This method is more restrictive and therefore safter than using API Keys. +* `action.d/ipthreat.conf` - new action for IPThreat integration, gh-3349 +* `action.d/ufw.conf` (gh-3018): + - new option `add` (default `prepend`), can be supplied as `insert 1` for ufw versions before v.0.36 (gh-2331, gh-3018) + - new options `kill-mode` and `kill` to drop established connections of intruder (see action for details, gh-3018) +* `iptables` and `iptables-ipset` actions extended to support multiple protocols with single action + for multiport or oneport type (back-ported from nftables action); +* `iptables` actions are more breakdown-safe: start wouldn't fail if chain or rule already exists + (e. g. created by previous instance and doesn't get purged properly); ultimately closes gh-980 +* `ipset` actions are more breakdown-safe: start wouldn't fail if set with this name already exists + (e. g. created by previous instance and don't deleted properly) +* replace internals of several `iptables` and `iptables-ipset` actions using internals of iptables include: + - better check mechanism (using `-C`, option `--check` is available long time); + - additionally iptables-ipset is a common action for `iptables-ipset-proto6-*` now (which become obsolete now); + - many features of different iptables actions are combinable as single chain/rule (can be supplied to action as parameters); + - iptables is a replacement for iptables-common now, several actions using this as include now become obsolete; +* new logtarget SYSTEMD-JOURNAL, gh-1403 +* fail2ban.conf: new fail2ban configuration option `allowipv6` (default `auto`), can be used to allow or disallow IPv6 + interface in fail2ban immediately by start (e. g. if fail2ban starts before network interfaces), gh-2804 +* invalidate IP/DNS caches by reload, so inter alia would allow to recognize IPv6IsAllowed immediately, previously + retarded up to cache max-time (5m), gh-2804 +* OpenRC (Gentoo, mainly) service script improvements, gh-2182 +* suppress unneeded info "Jail is not a JournalFilter instance" (moved to debug level), gh-3186 +* implements new interpolation variable `%(fail2ban_confpath)s` (automatically substituted from config-reader path, + default `/etc/fail2ban` or `/usr/local/etc/fail2ban` depending on distribution); `ignorecommands_dir` is unneeded anymore, + thus removed from `paths-common.conf`, fixes gh-3005 +* `fail2ban-regex`: accepts filter parameters containing new-line + + +ver. 0.11.2 (2020/11/23) - heal-the-world-with-security-tools +----------- -* v.0.10 uses more precise date template handling, that can be theoretically incompatible to some - user configurations resp. `datepattern`. +### Compatibility +* to v.0.10: + - 0.11 is totally compatible to 0.10 (configuration- and API-related stuff), but the database + got some new tables and fields (auto-converted during the first start), so once updated to 0.11, you + have to remove the database /var/lib/fail2ban/fail2ban.sqlite3 (or its different to 0.10 schema) + if you would need to downgrade to 0.10 for some reason. +* to v.0.9: + - Filter (or `failregex`) internal capture-groups: + + * If you've your own `failregex` or custom filters using conditional match `(?P=host)`, you should + rewrite the regex like in example below resp. using `(?:(?P=ip4)|(?P=ip6)` instead of `(?P=host)` + (or `(?:(?P=ip4)|(?P=ip6)|(?P=dns))` corresponding your `usedns` and `raw` settings). + + Of course you can always define your own capture-group (like below `_cond_ip_`) to do this. + ``` + testln="1500000000 failure from 192.0.2.1: bad host 192.0.2.1" + fail2ban-regex "$testln" "^\s*failure from (?P<_cond_ip_><HOST>): bad host (?P=_cond_ip_)$" + ``` + * New internal groups (currently reserved for internal usage): + `ip4`, `ip6`, `dns`, `fid`, `fport`, additionally `user` and another captures in lower case if + mapping from tag `<F-*>` used in failregex (e. g. `user` by `<F-USER>`). -* Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are - IPv6-capable now. + - v.0.10 and 0.11 use more precise date template handling, that can be theoretically incompatible to some + user configurations resp. `datepattern`. + - Since v0.10 fail2ban supports the matching of IPv6 addresses, but not all ban actions are + IPv6-capable now. -ver. 0.11.0-dev-0 (20??/??/??) - development nightly edition +### Fixes +* [stability] prevent race condition - no ban if filter (backend) is continuously busy if + too many messages will be found in log, e. g. initial scan of large log-file or journal (gh-2660) +* pyinotify-backend sporadically avoided initial scanning of log-file by start +* python 3.9 compatibility (and Travis CI support) +* restoring a large number (500+ depending on files ulimit) of current bans when using PyPy fixed +* manual ban is written to database, so can be restored by restart (gh-2647) +* `jail.conf`: don't specify `action` directly in jails (use `action_` or `banaction` instead) +* no mails-action added per default anymore (e. g. to allow that `action = %(action_mw)s` should be specified + per jail or in default section in jail.local), closes gh-2357 +* ensure we've unique action name per jail (also if parameter `actname` is not set but name deviates from standard name, gh-2686) +* don't use `%(banaction)s` interpolation because it can be complex value (containing `[...]` and/or quotes), + so would bother the action interpolation +* fixed type conversion in config readers (take place after all interpolations get ready), that allows to + specify typed parameters variable (as substitutions) as well as to supply it in other sections or as init parameters. +* `action.d/*-ipset*.conf`: several ipset actions fixed (no timeout per default anymore), so no discrepancy + between ipset and fail2ban (removal from ipset will be managed by fail2ban only, gh-2703) +* `action.d/cloudflare.conf`: fixed `actionunban` (considering new-line chars and optionally real json-parsing + with `jq`, gh-2140, gh-2656) +* `action.d/nftables.conf` (type=multiport only): fixed port range selector, replacing `:` with `-` (gh-2763) +* `action.d/firewallcmd-*.conf` (multiport only): fixed port range selector, replacing `:` with `-` (gh-2821) +* `action.d/bsd-ipfw.conf`: fixed selection of rule-no by large list or initial `lowest_rule_num` (gh-2836) +* `filter.d/common.conf`: avoid substitute of default values in related `lt_*` section, `__prefix_line` + should be interpolated in definition section (inside the filter-config, gh-2650) +* `filter.d/dovecot.conf`: + - add managesieve and submission support (gh-2795); + - accept messages with more verbose logging (gh-2573); +* `filter.d/courier-smtp.conf`: prefregex extended to consider port in log-message (gh-2697) +* `filter.d/traefik-auth.conf`: filter extended with parameter mode (`normal`, `ddos`, `aggressive`) to handle + the match of username differently (gh-2693): + - `normal`: matches 401 with supplied username only + - `ddos`: matches 401 without supplied username only + - `aggressive`: matches 401 and any variant (with and without username) +* `filter.d/sshd.conf`: normalizing of user pattern in all RE's, allowing empty user (gh-2749) + +### New Features and Enhancements +* fail2ban-regex: + - speedup formatted output (bypass unneeded stats creation) + - extended with prefregex statistic + - more informative output for `datepattern` (e. g. set from filter) - pattern : description +* parsing of action in jail-configs considers space between action-names as separator also + (previously only new-line was allowed), for example `action = a b` would specify 2 actions `a` and `b` +* new filter and jail for GitLab recognizing failed application logins (gh-2689) +* new filter and jail for Grafana recognizing failed application logins (gh-2855) +* new filter and jail for SoftEtherVPN recognizing failed application logins (gh-2723) +* `filter.d/guacamole.conf` extended with `logging` parameter to follow webapp-logging if it's configured (gh-2631) +* `filter.d/bitwarden.conf` enhanced to support syslog (gh-2778) +* introduced new prefix `{UNB}` for `datepattern` to disable word boundaries in regex; +* datetemplate: improved anchor detection for capturing groups `(^...)`; +* datepattern: improved handling with wrong recognized timestamps (timezones, no datepattern, etc) + as well as some warnings signaling user about invalid pattern or zone (gh-2814): + - filter gets mode in-operation, which gets activated if filter starts processing of new messages; + in this mode a timestamp read from log-line that appeared recently (not an old line), deviating too much + from now (up too 24h), will be considered as now (assuming a timezone issue), so could avoid unexpected + bypass of failure (previously exceeding `findtime`); + - better interaction with non-matching optional datepattern or invalid timestamps; + - implements special datepattern `{NONE}` - allow to find failures totally without date-time in log messages, + whereas filter will use now as timestamp (gh-2802) +* performance optimization of `datepattern` (better search algorithm in datedetector, especially for single template); +* fail2ban-client: extended to unban IP range(s) by subnet (CIDR/mask) or hostname (DNS), gh-2791; +* extended capturing of alternate tags in filter, allowing combine of multiple groups to single tuple token with new tag + prefix `<F-TUPLE_`, that would combine value of `<F-V>` with all value of `<F-TUPLE_V?_n?>` tags (gh-2755) + + +ver. 0.11.1 (2020/01/11) - this-is-the-way ----------- ### Fixes @@ -55,9 +320,141 @@ ver. 0.11.0-dev-0 (20??/??/??) - development nightly edition * algorithm of restore current bans after restart changed: update the restored ban-time (and therefore end of ban) of the ticket with ban-time of jail (as maximum), for all tickets with ban-time greater (or persistent); not affected if ban-time of the jail is unchanged between stop/start. +* added new setup-option `--without-tests` to skip building and installing of tests files (gh-2287). +* added new command `fail2ban-client get <JAIL> banip ?sep-char|--with-time?` to get the banned ip addresses (gh-1916). + + +ver. 0.10.5 (2020/01/10) - deserve-more-respect-a-jedis-weapon-must +----------- + +Yes, Hrrrm... + +### Fixes +* [compatibility] systemd backend: default flags changed to SYSTEM_ONLY(4), fixed in gh-2444 in order to ignore + user session files per default, so could prevent "Too many open files" errors on a lot of user sessions (see gh-2392) +* [grave] fixed parsing of multi-line filters (`maxlines` > 1) together with systemd backend, + now systemd-filter replaces newlines in message from systemd journal with `\n` (otherwise + multi-line parsing may be broken, because removal of matched string from multi-line buffer window + is confused by such extra new-lines, so they are retained and got matched on every followed + message, see gh-2431) +* [stability] prevent race condition - no unban if the bans occur continuously (gh-2410); + now an unban-check will happen not later than 10 tickets get banned regardless there are + still active bans available (precedence of ban over unban-check is 10 now) +* fixed read of included config-files (`.local` overwrites options of `.conf` for config-files + included with before/after) +* `action.d/abuseipdb.conf`: switched to use AbuseIPDB API v2 (gh-2302) +* `action.d/badips.py`: fixed start of banaction on demand (which may be IP-family related), gh-2390 +* `action.d/helpers-common.conf`: rewritten grep arguments, now options `-wF` used to match only + whole words and fixed string (not as pattern), gh-2298 +* `filter.d/apache-auth.conf`: + - ignore errors from mod_evasive in `normal` mode (mode-controlled now) (gh-2548); + - extended with option `mode` - `normal` (default) and `aggressive` +* `filter.d/sshd.conf`: + - matches `Bad protocol version identification` in `ddos` and `aggressive` modes (gh-2404). + - captures `Disconnecting ...: Change of username or service not allowed` (gh-2239, gh-2279) + - captures `Disconnected from ... [preauth]`, preauth phase only, different handling by `extra` + (with supplied user only) and `ddos`/`aggressive` mode (gh-2115, gh-2239, gh-2279) +* `filter.d/mysqld-auth.conf`: + - MYSQL 8.0.13 compatibility (log-error-verbosity = 3), log-format contains few additional words + enclosed in brackets after "[Note]" (gh-2314) +* `filter.d/sendmail-reject.conf`: + - `mode=extra` now captures port IDs of `TLSMTA` and `MSA` (defaults for ports 465 and 587 on some distros) +* `files/fail2ban.service.in`: fixed systemd-unit template - missing nftables dependency (gh-2313) +* several `action.d/mail*`: fixed usage with multiple log files (ultimate fix for gh-976, gh-2341) +* `filter.d/sendmail-reject.conf`: fixed journal usage for some systems (e. g. CentOS): if only identifier + set to `sm-mta` (no unit `sendmail`) for some messages (gh-2385) +* `filter.d/asterisk.conf`: asterisk can log additional timestamp if logs into systemd-journal + (regex extended with optional part matching this, gh-2383) +* `filter.d/postfix.conf`: + - regexp's accept variable suffix code in status of postfix for precise messages (gh-2442) + - extended with new postfix filter mode `errors` to match "too many errors" (gh-2439), + also included within modes `normal`, `more` (`extra` and `aggressive`), since postfix + parameter `smtpd_hard_error_limit` is default 20 (additionally consider `maxretry`) +* `filter.d/named-refused.conf`: + - support BIND 9.11.0 log format (includes an additional field @0xXXX..., gh-2406); + - `prefregex` extended, more selective now (denied/NOTAUTH suffix moved from failregex, so no catch-all there anymore) +* `filter.d/sendmail-auth.conf`, `filter.d/sendmail-reject.conf` : + - ID in prefix can be longer as 14 characters (gh-2563); +* all filters would accept square brackets around IPv4 addresses also (e. g. monit-filter, gh-2494) +* avoids unhandled exception during flush (gh-2588) +* fixes pass2allow-ftp jail - due to inverted handling, action should prohibit access per default for any IP, + therefore reset start on demand parameter for this action (it will be started immediately by repair); +* auto-detection of IPv6 subsystem availability (important for not on-demand actions or jails, like pass2allow); +### New Features +* new replacement tags for failregex to match subnets in form of IP-addresses with CIDR mask (gh-2559): + - `<CIDR>` - helper regex to match CIDR (simple integer form of net-mask); + - `<SUBNET>` - regex to match sub-net addresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional); +* grouped tags (`<ADDR>`, `<HOST>`, `<SUBNET>`) recognize IP addresses enclosed in square brackets +* new failregex-flag tag `<F-MLFGAINED>` for failregex, signaled that the access to service was gained + (ATM used similar to tag `<F-NOFAIL>`, but it does not add the log-line to matches, gh-2279) +* filters: introduced new configuration parameter `logtype` (default `file` for file-backends, and + `journal` for journal-backends, gh-2387); can be also set to `rfc5424` to force filters (which include common.conf) + to use RFC 5424 conform prefix-line per default (gh-2467); +* for better performance and safety the option `logtype` can be also used to + select short prefix-line for file-backends too for all filters using `__prefix_line` (`common.conf`), + if message logged only with `hostname svc[nnnn]` prefix (often the case on several systems): +```ini +[jail] +backend = auto +filter = flt[logtype=short] +``` +* `filter.d/common.conf`: differentiate `__prefix_line` for file/journal logtype's (speedup and fix parsing + of systemd-journal); +* `filter.d/traefik-auth.conf`: used to ban hosts, that were failed through traefik +* `filter.d/znc-adminlog.conf`: new filter for ZNC (IRC bouncer); requires the adminlog module to be loaded -ver. 0.10.4-dev-1 (20??/??/??) - development edition +### Enhancements +* introduced new options: `dbmaxmatches` (fail2ban.conf) and `maxmatches` (jail.conf) to contol + how many matches per ticket fail2ban can hold in memory and store in database (gh-2402, gh-2118); +* fail2ban.conf: introduced new section `[Thread]` and option `stacksize` to configure default size + of the stack for threads running in fail2ban (gh-2356), it could be set in `fail2ban.local` to + avoid runtime error "can't start new thread" (see gh-969); +* jail-reader extended (amend to gh-1622): actions support multi-line options now (interpolations + containing new-line); +* fail2ban-client: extended to ban/unban multiple tickets (see gh-2351, gh-2349); + Syntax: + - `fail2ban-client set <jain> banip <ip1> ... <ipN>` + - `fail2ban-client set <jain> unbanip [--report-absent] <ip1> ... <ipN>` +* fail2ban-client: extended with new feature which allows to inform fail2ban about single or multiple + attempts (failure) for IP (resp. failure-ID), see gh-2351; + Syntax: + - `fail2ban-client set <jail> attempt <ip> [<failure-message1> ... <failure-messageN>]` +* `action.d/nftables.conf`: + - isolate fail2ban rules into a dedicated table and chain (gh-2254) + - `nftables-allports` supports multiple protocols in single rule now + - combined nftables actions to single action `nftables`: + * `nftables-common` is removed (replaced with single action `nftables` now) + * `nftables-allports` is obsolete, superseded by `nftables[type=allports]` + * `nftables-multiport` is obsolete, superseded by `nftables[type=multiport]` + - allowed multiple protocols in `nftables[type=multiport]` action (single set with multiple rules + in chain), following configuration in jail would replace 3 separate actions, see + https://github.com/fail2ban/fail2ban/pull/2254#issuecomment-534684675 +* `action.d/badips.py`: option `loglevel` extended with level of summary message, + following example configuration logging summary with NOTICE and rest with DEBUG log-levels: + `action = badips.py[loglevel="debug, notice"]` +* samplestestcase.py (testSampleRegexsFactory) extended: + - allow coverage of journal logtype; + - new option `fileOptions` to set common filter/test options for whole test-file; +* large enhancement: auto-reban, improved invariant check and conditional operations (gh-2588): + - improves invariant check and repair (avoid unhandled exception, consider family on conditional operations, etc), + prepared for bulk re-ban in repair case (if bulk-ban becomes implemented); + - automatic reban (repeat banning action) after repair/restore sane environment, if already logged ticket causes + new failures (via new action operation `actionreban` or `actionban` if still not defined in action); + * introduces banning epoch for actions and tickets (to distinguish or recognize removed set of the tickets); + * invariant check avoids repair by unban/stop (unless parameter `actionrepair_on_unban` set to `true`); + * better handling for all conditional operations (distinguish families for certain operations like + repair/flush/stop, prepared for other families, e. g. if different handling for subnets expected, etc); + * partially implements gh-980 (more breakdown safe handling); + * closes gh-1680 (better as large-scale banning implementation with on-demand reban by failure, + at least unless a bulk-ban gets implemented); +* fail2ban-regex - several enhancements and fixes: + - improved usage output (don't put a long help if an error occurs); + - new option `--no-check-all` to avoid check of all regex's (first matched only); + - new option `-o`, `--out` to set token only provided in output (disables check-all and outputs only expected data). + + +ver. 0.10.4 (2018/10/04) - ten-four-on-due-date-ten-four ----------- ### Fixes @@ -72,6 +469,9 @@ ver. 0.10.4-dev-1 (20??/??/??) - development edition - extended with mode parameter, allows to avoid matching of messages like `auth challenge (REGISTER)` (see gh-2163) (currently `extra` as default to be backwards-compatible), see comments in filter how to set it to mode `normal`. +* `filter.d/domino-smtp.conf`: + - recognizes failures logged using another format (something like session-id, IP enclosed in square brackets); + - failregex extended to catch connections rejected for policy reasons (gh-2228); * `action.d/hostsdeny.conf`: fix parameter in config (dynamic parameters stating with '_' are protected and don't allowed in command-actions), see gh-2114; * decoding stability fix by wrong encoded characters like utf-8 surrogate pairs, etc (gh-2171): @@ -262,12 +262,16 @@ FileContainer Keeps the position pointer -dnsutils.py -~~~~~~~~~~~ +ipdns.py +~~~~~~~~ DNSUtils - Utility class for DNS and IP handling + Utility class for DNS handling + +IPAddr + + Object-class for IP address handling filter*.py @@ -278,6 +278,7 @@ to tune it. fail2ban-regex -D ... will present Debuggex URLs for the regexs and sample log files that you pass into it. In general use when using regex debuggers for generating fail2ban filters: + * use regex from the ./fail2ban-regex output (to ensure all substitutions are done) * replace <HOST> with (?&.ipv4) @@ -5,11 +5,11 @@ bin/fail2ban-testcases ChangeLog config/action.d/abuseipdb.conf config/action.d/apf.conf -config/action.d/badips.conf -config/action.d/badips.py +config/action.d/apprise.conf config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf +config/action.d/cloudflare-token.conf config/action.d/complain.conf config/action.d/dshield.conf config/action.d/dummy.conf @@ -25,8 +25,8 @@ config/action.d/hostsdeny.conf config/action.d/ipfilter.conf config/action.d/ipfw.conf config/action.d/iptables-allports.conf -config/action.d/iptables-common.conf config/action.d/iptables.conf +config/action.d/iptables-ipset.conf config/action.d/iptables-ipset-proto4.conf config/action.d/iptables-ipset-proto6-allports.conf config/action.d/iptables-ipset-proto6.conf @@ -34,6 +34,7 @@ config/action.d/iptables-multiport.conf config/action.d/iptables-multiport-log.conf config/action.d/iptables-new.conf config/action.d/iptables-xt_recent-echo.conf +config/action.d/ipthreat.conf config/action.d/mail-buffered.conf config/action.d/mail.conf config/action.d/mail-whois-common.conf @@ -42,7 +43,7 @@ config/action.d/mail-whois-lines.conf config/action.d/mynetwatchman.conf config/action.d/netscaler.conf config/action.d/nftables-allports.conf -config/action.d/nftables-common.conf +config/action.d/nftables.conf config/action.d/nftables-multiport.conf config/action.d/nginx-block-map.conf config/action.d/npf.conf @@ -81,7 +82,9 @@ config/filter.d/apache-pass.conf config/filter.d/apache-shellshock.conf config/filter.d/assp.conf config/filter.d/asterisk.conf +config/filter.d/bitwarden.conf config/filter.d/botsearch-common.conf +config/filter.d/centreon.conf config/filter.d/common.conf config/filter.d/counter-strike.conf config/filter.d/courier-auth.conf @@ -98,6 +101,8 @@ config/filter.d/exim.conf config/filter.d/exim-spam.conf config/filter.d/freeswitch.conf config/filter.d/froxlor-auth.conf +config/filter.d/gitlab.conf +config/filter.d/grafana.conf config/filter.d/groupoffice.conf config/filter.d/gssftpd.conf config/filter.d/guacamole.conf @@ -108,10 +113,13 @@ config/filter.d/kerio.conf config/filter.d/lighttpd-auth.conf config/filter.d/mongodb-auth.conf config/filter.d/monit.conf +config/filter.d/monitorix.conf +config/filter.d/mssql-auth.conf config/filter.d/murmur.conf config/filter.d/mysqld-auth.conf config/filter.d/nagios.conf config/filter.d/named-refused.conf +config/filter.d/nginx-bad-request.conf config/filter.d/nginx-botsearch.conf config/filter.d/nginx-http-auth.conf config/filter.d/nginx-limit-req.conf @@ -130,6 +138,7 @@ config/filter.d/pure-ftpd.conf config/filter.d/qmail.conf config/filter.d/recidive.conf config/filter.d/roundcube-auth.conf +config/filter.d/scanlogd.conf config/filter.d/screensharingd.conf config/filter.d/selinux-common.conf config/filter.d/selinux-ssh.conf @@ -137,6 +146,7 @@ config/filter.d/sendmail-auth.conf config/filter.d/sendmail-reject.conf config/filter.d/sieve.conf config/filter.d/slapd.conf +config/filter.d/softethervpn.conf config/filter.d/sogo-auth.conf config/filter.d/solid-pop3d.conf config/filter.d/squid.conf @@ -145,11 +155,13 @@ config/filter.d/sshd.conf config/filter.d/stunnel.conf config/filter.d/suhosin.conf config/filter.d/tine20.conf +config/filter.d/traefik-auth.conf config/filter.d/uwimap-auth.conf config/filter.d/vsftpd.conf config/filter.d/webmin-auth.conf config/filter.d/wuftpd.conf config/filter.d/xinetd-fail.conf +config/filter.d/znc-adminlog.conf config/filter.d/zoneminder.conf config/jail.conf config/paths-arch.conf @@ -213,15 +225,17 @@ fail2ban/setup.py fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py -fail2ban/tests/action_d/test_badips.py fail2ban/tests/action_d/test_smtp.py fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py fail2ban/tests/banmanagertestcase.py fail2ban/tests/clientbeautifiertestcase.py fail2ban/tests/clientreadertestcase.py +fail2ban/tests/config/action.d/action.conf fail2ban/tests/config/action.d/brokenaction.conf fail2ban/tests/config/fail2ban.conf +fail2ban/tests/config/filter.d/checklogtype.conf +fail2ban/tests/config/filter.d/checklogtype_test.conf fail2ban/tests/config/filter.d/simple.conf fail2ban/tests/config/filter.d/test.conf fail2ban/tests/config/filter.d/test.local @@ -260,6 +274,8 @@ fail2ban/tests/files/database_v1.db fail2ban/tests/files/database_v2.db fail2ban/tests/files/filter.d/substition.conf fail2ban/tests/files/filter.d/testcase01.conf +fail2ban/tests/files/filter.d/testcase02.conf +fail2ban/tests/files/filter.d/testcase02.local fail2ban/tests/files/filter.d/testcase-common.conf fail2ban/tests/files/ignorecommand.py fail2ban/tests/files/logs/3proxy @@ -275,9 +291,11 @@ fail2ban/tests/files/logs/apache-pass fail2ban/tests/files/logs/apache-shellshock fail2ban/tests/files/logs/assp fail2ban/tests/files/logs/asterisk +fail2ban/tests/files/logs/bitwarden fail2ban/tests/files/logs/bsd/syslog-plain.txt fail2ban/tests/files/logs/bsd/syslog-v.txt fail2ban/tests/files/logs/bsd/syslog-vv.txt +fail2ban/tests/files/logs/centreon fail2ban/tests/files/logs/counter-strike fail2ban/tests/files/logs/courier-auth fail2ban/tests/files/logs/courier-smtp @@ -292,6 +310,8 @@ fail2ban/tests/files/logs/exim fail2ban/tests/files/logs/exim-spam fail2ban/tests/files/logs/freeswitch fail2ban/tests/files/logs/froxlor-auth +fail2ban/tests/files/logs/gitlab +fail2ban/tests/files/logs/grafana fail2ban/tests/files/logs/groupoffice fail2ban/tests/files/logs/gssftpd fail2ban/tests/files/logs/guacamole @@ -301,10 +321,13 @@ fail2ban/tests/files/logs/kerio fail2ban/tests/files/logs/lighttpd-auth fail2ban/tests/files/logs/mongodb-auth fail2ban/tests/files/logs/monit +fail2ban/tests/files/logs/monitorix +fail2ban/tests/files/logs/mssql-auth fail2ban/tests/files/logs/murmur fail2ban/tests/files/logs/mysqld-auth fail2ban/tests/files/logs/nagios fail2ban/tests/files/logs/named-refused +fail2ban/tests/files/logs/nginx-bad-request fail2ban/tests/files/logs/nginx-botsearch fail2ban/tests/files/logs/nginx-http-auth fail2ban/tests/files/logs/nginx-limit-req @@ -323,25 +346,30 @@ fail2ban/tests/files/logs/pure-ftpd fail2ban/tests/files/logs/qmail fail2ban/tests/files/logs/recidive fail2ban/tests/files/logs/roundcube-auth +fail2ban/tests/files/logs/scanlogd fail2ban/tests/files/logs/screensharingd fail2ban/tests/files/logs/selinux-ssh fail2ban/tests/files/logs/sendmail-auth fail2ban/tests/files/logs/sendmail-reject fail2ban/tests/files/logs/sieve fail2ban/tests/files/logs/slapd +fail2ban/tests/files/logs/softethervpn fail2ban/tests/files/logs/sogo-auth fail2ban/tests/files/logs/solid-pop3d fail2ban/tests/files/logs/squid fail2ban/tests/files/logs/squirrelmail fail2ban/tests/files/logs/sshd +fail2ban/tests/files/logs/sshd-journal fail2ban/tests/files/logs/stunnel fail2ban/tests/files/logs/suhosin fail2ban/tests/files/logs/tine20 +fail2ban/tests/files/logs/traefik-auth fail2ban/tests/files/logs/uwimap-auth fail2ban/tests/files/logs/vsftpd fail2ban/tests/files/logs/webmin-auth fail2ban/tests/files/logs/wuftpd fail2ban/tests/files/logs/xinetd-fail +fail2ban/tests/files/logs/znc-adminlog fail2ban/tests/files/logs/zoneminder fail2ban/tests/files/logs/zzz-generic-example fail2ban/tests/files/logs/zzz-sshd-obsolete-multiline @@ -371,12 +399,12 @@ files/cacti/fail2ban_stats.sh files/cacti/README files/debian-initd files/fail2ban-logrotate +files/fail2ban-openrc.conf +files/fail2ban-openrc.init.in files/fail2ban.service.in files/fail2ban-tmpfiles.conf files/fail2ban.upstart files/gen_badbots -files/gentoo-confd -files/gentoo-initd files/ipmasq-ZZZzzz_fail2ban.rul files/logwatch/fail2ban files/logwatch/fail2ban-0.8.log @@ -2,7 +2,7 @@ / _|__ _(_) |_ ) |__ __ _ _ _ | _/ _` | | |/ /| '_ \/ _` | ' \ |_| \__,_|_|_/___|_.__/\__,_|_||_| - v0.11.0.dev1 20??/??/?? + v1.0.3.dev1 20??/??/?? ## Fail2Ban: ban hosts that cause multiple authentication errors @@ -22,7 +22,8 @@ mechanisms if you really want to protect services. ------|------ This README is a quick introduction to Fail2Ban. More documentation, FAQ, and HOWTOs -to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki) +to be found on fail2ban(1) manpage, [Wiki](https://github.com/fail2ban/fail2ban/wiki), +[Developers documentation](https://fail2ban.readthedocs.io/) and the website: https://www.fail2ban.org Installation: @@ -32,7 +33,8 @@ Installation: this case, you should use that instead.** Required: -- [Python2 >= 2.6 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org) +- [Python2 >= 2.7 or Python >= 3.2](https://www.python.org) or [PyPy](https://pypy.org) +- python-setuptools, python-distutils or python3-setuptools for installation from source Optional: - [pyinotify >= 0.8.3](https://github.com/seb-m/pyinotify), may require: @@ -45,11 +47,11 @@ Optional: To install: - tar xvfj fail2ban-0.11.0.tar.bz2 - cd fail2ban-0.11.0 + tar xvfj fail2ban-master.tar.bz2 + cd fail2ban-master sudo python setup.py install -Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, 0.11 +Alternatively, you can clone the source from GitHub to a directory of Your choice, and do the install from there. Pick the correct branch, for example, master or 0.11 git clone https://github.com/fail2ban/fail2ban.git cd fail2ban @@ -88,11 +90,11 @@ fail2ban(1) and jail.conf(5) manpages for further references. Code status: ------------ -* travis-ci.org: [](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) +* travis-ci.org: [](https://travis-ci.org/fail2ban/fail2ban?branch=master) / [](https://travis-ci.org/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [](https://travis-ci.org/fail2ban/fail2ban?branch=0.10) (0.10 branch) -* coveralls.io: [](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) +* coveralls.io: [](https://coveralls.io/github/fail2ban/fail2ban?branch=master) / [](https://coveralls.io/github/fail2ban/fail2ban?branch=0.11) (0.11 branch) / [](https://coveralls.io/github/fail2ban/fail2ban?branch=0.10) / (0.10 branch) -* codecov.io: [](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) +* codecov.io: [](https://codecov.io/gh/fail2ban/fail2ban/branch/master) / [](https://codecov.io/gh/fail2ban/fail2ban/branch/0.11) (0.11 branch) / [](https://codecov.io/gh/fail2ban/fail2ban/branch/0.10) (0.10 branch) Contact: -------- @@ -33,6 +33,7 @@ Christoph Haas Christos Psonis craneworks Cyril Jaquier +Daniel Aleksandersen Daniel B. Cid Daniel B. Daniel Black diff --git a/config/action.d/abuseipdb.conf b/config/action.d/abuseipdb.conf index c53ed489..ed958c86 100644 --- a/config/action.d/abuseipdb.conf +++ b/config/action.d/abuseipdb.conf @@ -21,14 +21,13 @@ # # Example, for ssh bruteforce (in section [sshd] of `jail.local`): # action = %(known/action)s -# %(action_abuseipdb)s[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] +# abuseipdb[abuseipdb_apikey="my-api-key", abuseipdb_category="18,22"] # -# See below for catagories. +# See below for categories. # -# Original Ref: https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban # Added to fail2ban by Andrew James Collett (ajcollett) -## abuseIPDB Catagories, `the abuseipdb_category` MUST be set in the jail.conf action call. +## abuseIPDB Categories, `the abuseipdb_category` MUST be set in the jail.conf action call. # Example, for ssh bruteforce: action = %(action_abuseipdb)s[abuseipdb_category="18,22"] # ID Title Description # 3 Fraud Orders @@ -47,6 +46,9 @@ [Definition] +# bypass action for restored tickets +norestored = 1 + # Option: actionstart # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD @@ -80,13 +82,10 @@ actioncheck = # wherever you install the helper script. For the PHP helper script, see # <https://wiki.shaunc.com/wikka.php?wakka=ReportingToAbuseIPDBWithFail2Ban> # -# --ciphers ecdhe_ecdsa_aes_256_sha is used to workaround a -# "NSS error -12286" from curl as it attempts to connect using -# SSLv3. See https://www.centos.org/forums/viewtopic.php?t=52732 # Tags: See jail.conf(5) man page # Values: CMD # -actionban = lgm=$(printf '%%s\n...' "<matches>"); curl --fail --tlsv1.1 --data "key=<abuseipdb_apikey>" --data-urlencode "comment=$lgm" --data "ip=<ip>" --data "category=<abuseipdb_category>" "https://www.abuseipdb.com/report/json" +actionban = lgm=$(printf '%%.1000s\n...' "<matches>"); curl -sSf "https://api.abuseipdb.com/api/v2/report" -H "Accept: application/json" -H "Key: <abuseipdb_apikey>" --data-urlencode "comment=$lgm" --data-urlencode "ip=<ip>" --data "categories=<abuseipdb_category>" # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -101,5 +100,5 @@ actionunban = # Notes Your API key from abuseipdb.com # Values: STRING Default: None # Register for abuseipdb [https://www.abuseipdb.com], get api key and set below. -# You will need to set the catagory in the action call. +# You will need to set the category in the action call. abuseipdb_apikey = diff --git a/config/action.d/apprise.conf b/config/action.d/apprise.conf new file mode 100644 index 00000000..37c42ea2 --- /dev/null +++ b/config/action.d/apprise.conf @@ -0,0 +1,49 @@ +# Fail2Ban configuration file +# +# Author: Chris Caron <lead2gold@gmail.com> +# +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = printf %%b "The jail <name> as been started successfully." | <apprise> -t "[Fail2Ban] <name>: started on `uname -n`" + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = printf %%b "The jail <name> has been stopped." | <apprise> -t "[Fail2Ban] <name>: stopped on `uname -n`" + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = printf %%b "The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>" | <apprise> -n "warning" -t "[Fail2Ban] <name>: banned <ip> from `uname -n`" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] + +# Define location of the default apprise configuration file to use +# +config = /etc/fail2ban/apprise.conf +# +apprise = apprise -c "<config>" diff --git a/config/action.d/badips.conf b/config/action.d/badips.conf deleted file mode 100644 index 6f9513f6..00000000 --- a/config/action.d/badips.conf +++ /dev/null @@ -1,19 +0,0 @@ -# Fail2ban reporting to badips.com -# -# Note: This reports an IP only and does not actually ban traffic. Use -# another action in the same jail if you want bans to occur. -# -# Set the category to the appropriate value before use. -# -# To get see register and optional key to get personalised graphs see: -# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key - -[Definition] - -actionban = curl --fail --user-agent "<agent>" http://www.badips.com/add/<category>/<ip> - -[Init] - -# Option: category -# Notes.: Values are from the list here: http://www.badips.com/get/categories -category = diff --git a/config/action.d/badips.py b/config/action.d/badips.py deleted file mode 100644 index 4e50890c..00000000 --- a/config/action.d/badips.py +++ /dev/null @@ -1,389 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import sys -if sys.version_info < (2, 7): # pragma: no cover - raise ImportError("badips.py action requires Python >= 2.7") -import json -import threading -import logging -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.request import Request, urlopen - from urllib.parse import urlencode - from urllib.error import HTTPError -else: # pragma: 3.x no cover - from urllib2 import Request, urlopen, HTTPError - from urllib import urlencode - -from fail2ban.server.actions import ActionBase - - -class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable - """Fail2Ban action which reports bans to badips.com, and also - blacklist bad IPs listed on badips.com by using another action's - ban method. - - Parameters - ---------- - jail : Jail - The jail which the action belongs to. - name : str - Name assigned to the action. - category : str - Valid badips.com category for reporting failures. - score : int, optional - Minimum score for bad IPs. Default 3. - age : str, optional - Age of last report for bad IPs, per badips.com syntax. - Default "24h" (24 hours) - key : str, optional - Key issued by badips.com to report bans, for later retrieval - of personalised content. - banaction : str, optional - Name of banaction to use for blacklisting bad IPs. If `None`, - no blacklist of IPs will take place. - Default `None`. - bancategory : str, optional - Name of category to use for blacklisting, which can differ - from category used for reporting. e.g. may want to report - "postfix", but want to use whole "mail" category for blacklist. - Default `category`. - bankey : str, optional - Key issued by badips.com to blacklist IPs reported with the - associated key. - updateperiod : int, optional - Time in seconds between updating bad IPs blacklist. - Default 900 (15 minutes) - agent : str, optional - User agent transmitted to server. - Default `Fail2Ban/ver.` - - Raises - ------ - ValueError - If invalid `category`, `score`, `banaction` or `updateperiod`. - """ - - TIMEOUT = 10 - _badips = "https://www.badips.com" - def _Request(self, url, **argv): - return Request(url, headers={'User-Agent': self.agent}, **argv) - - def __init__(self, jail, name, category, score=3, age="24h", key=None, - banaction=None, bancategory=None, bankey=None, updateperiod=900, agent="Fail2Ban", - timeout=TIMEOUT): - super(BadIPsAction, self).__init__(jail, name) - - self.timeout = timeout - self.agent = agent - self.category = category - self.score = score - self.age = age - self.key = key - self.banaction = banaction - self.bancategory = bancategory or category - self.bankey = bankey - self.updateperiod = updateperiod - - self._bannedips = set() - # Used later for threading.Timer for updating badips - self._timer = None - - @staticmethod - def isAvailable(timeout=1): - try: - response = urlopen(Request("/".join([BadIPsAction._badips]), - headers={'User-Agent': "Fail2Ban"}), timeout=timeout) - return True, '' - except Exception as e: # pragma: no cover - return False, e - - def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) - messages = {} - try: - messages = json.loads(response.read().decode('utf-8')) - except: - pass - self._logSys.error( - "%s. badips.com response: '%s'", what, - messages.get('err', 'Unknown')) - - def getCategories(self, incParents=False): - """Get badips.com categories. - - Returns - ------- - set - Set of categories. - - Raises - ------ - HTTPError - Any issues with badips.com request. - ValueError - If badips.com response didn't contain necessary information - """ - try: - response = urlopen( - self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch categories") - raise - else: - response_json = json.loads(response.read().decode('utf-8')) - if not 'categories' in response_json: - err = "badips.com response lacked categories specification. Response was: %s" \ - % (response_json,) - self._logSys.error(err) - raise ValueError(err) - categories = response_json['categories'] - categories_names = set( - value['Name'] for value in categories) - if incParents: - categories_names.update(set( - value['Parent'] for value in categories - if "Parent" in value)) - return categories_names - - def getList(self, category, score, age, key=None): - """Get badips.com list of bad IPs. - - Parameters - ---------- - category : str - Valid badips.com category. - score : int - Minimum score for bad IPs. - age : str - Age of last report for bad IPs, per badips.com syntax. - key : str, optional - Key issued by badips.com to fetch IPs reported with the - associated key. - - Returns - ------- - set - Set of bad IPs. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "?".join([ - "/".join([self._badips, "get", "list", category, str(score)]), - urlencode({'age': age})]) - if key: - url = "&".join([url, urlencode({'key': key})]) - self._logSys.debug('badips.com: get list, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch bad IP list") - raise - else: - return set(response.read().decode('utf-8').split()) - - @property - def category(self): - """badips.com category for reporting IPs. - """ - return self._category - - @category.setter - def category(self, category): - if category not in self.getCategories(): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - category) - raise ValueError("Invalid category: %s" % category) - self._category = category - - @property - def bancategory(self): - """badips.com bancategory for fetching IPs. - """ - return self._bancategory - - @bancategory.setter - def bancategory(self, bancategory): - if bancategory != "any" and bancategory not in self.getCategories(incParents=True): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - bancategory) - raise ValueError("Invalid bancategory: %s" % bancategory) - self._bancategory = bancategory - - @property - def score(self): - """badips.com minimum score for fetching IPs. - """ - return self._score - - @score.setter - def score(self, score): - score = int(score) - if 0 <= score <= 5: - self._score = score - else: - raise ValueError("Score must be 0-5") - - @property - def banaction(self): - """Jail action to use for banning/unbanning. - """ - return self._banaction - - @banaction.setter - def banaction(self, banaction): - if banaction is not None and banaction not in self._jail.actions: - self._logSys.error("Action name '%s' not in jail '%s'", - banaction, self._jail.name) - raise ValueError("Invalid banaction") - self._banaction = banaction - - @property - def updateperiod(self): - """Period in seconds between banned bad IPs will be updated. - """ - return self._updateperiod - - @updateperiod.setter - def updateperiod(self, updateperiod): - updateperiod = int(updateperiod) - if updateperiod > 0: - self._updateperiod = updateperiod - else: - raise ValueError("Update period must be integer greater than 0") - - def _banIPs(self, ips): - for ip in ips: - try: - self._jail.actions[self.banaction].ban({ - 'ip': ip, - 'failures': 0, - 'matches': "", - 'ipmatches': "", - 'ipjailmatches': "", - }) - except Exception as e: - self._logSys.error( - "Error banning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._bannedips.add(ip) - self._logSys.debug( - "Banned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - - def _unbanIPs(self, ips): - for ip in ips: - try: - self._jail.actions[self.banaction].unban({ - 'ip': ip, - 'failures': 0, - 'matches': "", - 'ipmatches': "", - 'ipjailmatches': "", - }) - except Exception as e: - self._logSys.info( - "Error unbanning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._logSys.debug( - "Unbanned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - finally: - self._bannedips.remove(ip) - - def start(self): - """If `banaction` set, blacklists bad IPs. - """ - if self.banaction is not None: - self.update() - - def update(self): - """If `banaction` set, updates blacklisted IPs. - - Queries badips.com for list of bad IPs, removing IPs from the - blacklist if no longer present, and adds new bad IPs to the - blacklist. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - - try: - ips = self.getList( - self.bancategory, self.score, self.age, self.bankey) - # Remove old IPs no longer listed - self._unbanIPs(self._bannedips - ips) - # Add new IPs which are now listed - self._banIPs(ips - self._bannedips) - - self._logSys.debug( - "Updated IPs for jail '%s'. Update again in %i seconds", - self._jail.name, self.updateperiod) - finally: - self._timer = threading.Timer(self.updateperiod, self.update) - self._timer.start() - - def stop(self): - """If `banaction` set, clears blacklisted IPs. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - self._unbanIPs(self._bannedips.copy()) - - def ban(self, aInfo): - """Reports banned IP to badips.com. - - Parameters - ---------- - aInfo : dict - Dictionary which includes information in relation to - the ban. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - if self.key: - url = "?".join([url, urlencode({'key': self.key})]) - self._logSys.debug('badips.com: ban, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to ban") - raise - else: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.debug( - "Response from badips.com report: '%s'", - messages['suc']) - -Action = BadIPsAction diff --git a/config/action.d/bsd-ipfw.conf b/config/action.d/bsd-ipfw.conf index 5116b0d8..444192d3 100644 --- a/config/action.d/bsd-ipfw.conf +++ b/config/action.d/bsd-ipfw.conf @@ -14,7 +14,10 @@ # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 < b) {} else if ($1 == b) { b = $1 + 1 } else { e = b } } END { if (e) exit e <br> else exit b }'; num=$?; ipfw -q add $num <blocktype> <block> from table\(<table>\) to me <port>; echo $num > "<startstatefile>" ) +actionstart = ipfw show | fgrep -c -m 1 -s 'table(<table>)' > /dev/null 2>&1 || ( + num=$(ipfw show | awk 'BEGIN { b = <lowest_rule_num> } { if ($1 == b) { b = $1 + 1 } } END { print b }'); + ipfw -q add "$num" <blocktype> <block> from table\(<table>\) to me <port>; echo "$num" > "<startstatefile>" + ) # Option: actionstop diff --git a/config/action.d/cloudflare-token.conf b/config/action.d/cloudflare-token.conf new file mode 100644 index 00000000..287621eb --- /dev/null +++ b/config/action.d/cloudflare-token.conf @@ -0,0 +1,93 @@ +# +# Author: Logic-32 +# +# IMPORTANT +# +# Please set jail.local's permission to 640 because it contains your CF API token. +# +# This action depends on curl. +# +# To get your Cloudflare API token: https://developers.cloudflare.com/api/tokens/create/ +# +# Cloudflare Firewall API: https://developers.cloudflare.com/firewall/api/cf-firewall-rules/endpoints/ + +[Definition] + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: <ip> IP address +# <failures> number of failures +# <time> unix timestamp of the ban time +# Values: CMD +actionban = curl -s -X POST "<_cf_api_url>" \ + <_cf_api_prms> \ + --data '{"mode":"<cfmode>","configuration":{"target":"<cftarget>","value":"<ip>"},"notes":"<notes>"}' + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: <ip> IP address +# <failures> number of failures +# <time> unix timestamp of the ban time +# Values: CMD +# +actionunban = id=$(curl -s -X GET "<_cf_api_url>" \ + --data-urlencode "mode=<cfmode>" --data-urlencode "notes=<notes>" --data-urlencode "configuration.target=<cftarget>" --data-urlencode "configuration.value=<ip>" \ + <_cf_api_prms> \ + | awk -F"[,:}]" '{for(i=1;i<=NF;i++){if($i~/'id'\042/){print $(i+1)}}}' \ + | tr -d ' "' \ + | head -n 1) + if [ -z "$id" ]; then echo "<name>: id for <ip> cannot be found using target <cftarget>"; exit 0; fi; \ + curl -s -X DELETE "<_cf_api_url>/$id" \ + <_cf_api_prms> \ + --data '{"cascade": "none"}' + +_cf_api_url = https://api.cloudflare.com/client/v4/zones/<cfzone>/firewall/access_rules/rules +_cf_api_prms = -H "Authorization: Bearer <cftoken>" -H "Content-Type: application/json" + +[Init] + +# Declare your Cloudflare Authorization Bearer Token in the [DEFAULT] section of your jail.local file. + +# The Cloudflare <ZONE_ID> of hte domain you want to manage. +# +# cfzone = + +# Your personal Cloudflare token. Ideally restricted to just have "Zone.Firewall Services" permissions. +# +# cftoken = + +# Target of the firewall rule. Default is "ip" (v4). +# +cftarget = ip + +# The firewall mode Cloudflare should use. Default is "block" (deny access). +# Consider also "js_challenge" or other "allowed_modes" if you want. +# +cfmode = block + +# The message to include in the firewall IP banning rule. +# +notes = Fail2Ban <name> + +[Init?family=inet6] +cftarget = ip6 diff --git a/config/action.d/cloudflare.conf b/config/action.d/cloudflare.conf index 1c48a37f..4af87080 100644 --- a/config/action.d/cloudflare.conf +++ b/config/action.d/cloudflare.conf @@ -5,7 +5,7 @@ # # Please set jail.local's permission to 640 because it contains your CF API key. # -# This action depends on curl. +# This action depends on curl (and optionally jq). # Referenced from http://www.normyee.net/blog/2012/02/02/adding-cloudflare-support-to-fail2ban by NORM YEE # # To get your CloudFlare API Key: https://www.cloudflare.com/a/account/my-account @@ -43,9 +43,9 @@ actioncheck = # API v1 #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>' # API v4 -actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \ - -H 'Content-Type: application/json' -d '{ "mode": "block", "configuration": { "target": "ip", "value": "<ip>" } }' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ + -d '{"mode":"block","configuration":{"target":"<cftarget>","value":"<ip>"},"notes":"Fail2Ban <name>"}' \ + <_cf_api_url> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -58,9 +58,14 @@ actionban = curl -s -o /dev/null -X POST -H 'X-Auth-Email: <cfuser>' -H 'X-Auth- # API v1 #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=<cftoken>' -d 'email=<cfuser>' -d 'key=<ip>' # API v4 -actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \ - https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules/$(curl -s -X GET -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' \ - 'https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules?mode=block&configuration_target=ip&configuration_value=<ip>&page=1&per_page=1' | cut -d'"' -f6) +actionunban = id=$(curl -s -X GET <_cf_api_prms> \ + "<_cf_api_url>?mode=block&configuration_target=<cftarget>&configuration_value=<ip>&page=1&per_page=1¬es=Fail2Ban%%20<name>" \ + | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) + if [ -z "$id" ]; then echo "<name>: id for <ip> cannot be found"; exit 0; fi; + curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" + +_cf_api_url = https://api.cloudflare.com/client/v4/user/firewall/access_rules/rules +_cf_api_prms = -H 'X-Auth-Email: <cfuser>' -H 'X-Auth-Key: <cftoken>' -H 'Content-Type: application/json' [Init] @@ -76,3 +81,8 @@ actionunban = curl -s -o /dev/null -X DELETE -H 'X-Auth-Email: <cfuser>' -H 'X-A cftoken = cfuser = + +cftarget = ip + +[Init?family=inet6] +cftarget = ip6 diff --git a/config/action.d/complain.conf b/config/action.d/complain.conf index 3a5f882c..4d73b058 100644 --- a/config/action.d/complain.conf +++ b/config/action.d/complain.conf @@ -102,7 +102,7 @@ logpath = /dev/null # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/dshield.conf b/config/action.d/dshield.conf index c128bef3..3d5a7a53 100644 --- a/config/action.d/dshield.conf +++ b/config/action.d/dshield.conf @@ -179,7 +179,7 @@ tcpflags = # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff --git a/config/action.d/firewallcmd-ipset.conf b/config/action.d/firewallcmd-ipset.conf index a1065224..c36ba694 100644 --- a/config/action.d/firewallcmd-ipset.conf +++ b/config/action.d/firewallcmd-ipset.conf @@ -18,20 +18,45 @@ before = firewallcmd-common.conf [Definition] -actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt> +actionstart = <ipstype_<ipsettype>/actionstart> firewall-cmd --direct --add-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype> -actionflush = ipset flush <ipmset> +actionflush = <ipstype_<ipsettype>/actionflush> actionstop = firewall-cmd --direct --remove-rule <family> filter <chain> 0 <actiontype> -m set --match-set <ipmset> src -j <blocktype> <actionflush> - ipset destroy <ipmset> + <ipstype_<ipsettype>/actionstop> + +actionban = <ipstype_<ipsettype>/actionban> + +# actionprolong = %(actionban)s + +actionunban = <ipstype_<ipsettype>/actionunban> + +[ipstype_ipset] + +actionstart = ipset -exist create <ipmset> hash:ip timeout <default-ipsettime> <familyopt> + +actionflush = ipset flush <ipmset> -actionban = ipset add <ipmset> <ip> timeout <bantime> -exist +actionstop = ipset destroy <ipmset> -actionprolong = %(actionban)s +actionban = ipset -exist add <ipmset> <ip> timeout <ipsettime> -actionunban = ipset del <ipmset> <ip> -exist +actionunban = ipset -exist del <ipmset> <ip> + +[ipstype_firewalld] + +actionstart = firewall-cmd --direct --new-ipset=<ipmset> --type=hash:ip --option=timeout=<default-ipsettime> <firewalld_familyopt> + +# TODO: there doesn't seem to be an explicit way to invoke the ipset flush function using firewall-cmd +actionflush = + +actionstop = firewall-cmd --direct --delete-ipset=<ipmset> + +actionban = firewall-cmd --ipset=<ipmset> --add-entry=<ip> + +actionunban = firewall-cmd --ipset=<ipmset> --remove-entry=<ip> [Init] @@ -42,11 +67,25 @@ actionunban = ipset del <ipmset> <ip> -exist # chain = INPUT_direct -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 + +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>'] +timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0) -default-timeout = 600 +# Option: ipsettype +# Notes.: defines type of ipset used for match-set (firewalld or ipset) +# Values: firewalld or ipset +# Default: ipset +ipsettype = ipset # Option: actiontype # Notes.: defines additions to the blocking rule @@ -67,14 +106,16 @@ multiport = -p <protocol> -m multiport --dports <port> ipmset = f2b-<name> familyopt = +firewalld_familyopt = [Init?family=inet6] ipmset = f2b-<name>6 -familyopt = <sp>family inet6 +familyopt = family inet6 +firewalld_familyopt = --option=family=inet6 # DEV NOTES: # -# Author: Edgar Hoch and Daniel Black +# Author: Edgar Hoch, Daniel Black, Sergey Brester and Mihail Politaev # firewallcmd-new / iptables-ipset-proto6 combined for maximium goodness diff --git a/config/action.d/firewallcmd-rich-logging.conf b/config/action.d/firewallcmd-rich-logging.conf index badfee83..21e45087 100644 --- a/config/action.d/firewallcmd-rich-logging.conf +++ b/config/action.d/firewallcmd-rich-logging.conf @@ -1,6 +1,6 @@ # Fail2Ban configuration file # -# Author: Donald Yandt +# Authors: Donald Yandt, Sergey G. Brester # # Because of the rich rule commands requires firewalld-0.3.1+ # This action uses firewalld rich-rules which gives you a cleaner iptables since it stores rules according to zones and not @@ -10,36 +10,15 @@ # # If you use the --permanent rule you get a xml file in /etc/firewalld/zones/<zone>.xml that can be shared and parsed easliy # -# Example commands to view rules: -# firewall-cmd [--zone=<zone>] --list-rich-rules -# firewall-cmd [--zone=<zone>] --list-all -# firewall-cmd [--zone=zone] --query-rich-rule='rule' +# This is an derivative of firewallcmd-rich-rules.conf, see there for details and other parameters. [INCLUDES] -before = firewallcmd-common.conf +before = firewallcmd-rich-rules.conf [Definition] -actionstart = - -actionstop = - -actioncheck = - -# you can also use zones and/or service names. -# -# zone example: -# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' port port='<port>' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>" -# -# service name example: -# firewall-cmd --zone=<zone> --add-rich-rule="rule family='<family>' source address='<ip>' service name='<service>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>" -# -# Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp - -actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done - -actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype>"; done +rich-suffix = log prefix='f2b-<name>' level='<level>' limit value='<rate>/m' <rich-blocktype> [Init] @@ -48,4 +27,3 @@ level = info # log rate per minute rate = 1 - diff --git a/config/action.d/firewallcmd-rich-rules.conf b/config/action.d/firewallcmd-rich-rules.conf index bed71797..75a27d88 100644 --- a/config/action.d/firewallcmd-rich-rules.conf +++ b/config/action.d/firewallcmd-rich-rules.conf @@ -35,8 +35,10 @@ actioncheck = # # Because rich rules can only handle single or a range of ports we must split ports and execute the command for each port. Ports can be single and ranges separated by a comma or space for an example: http, https, 22-60, 18 smtp -actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done - -actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' <rich-blocktype>"; done +fwcmd_rich_rule = rule family='<family>' source address='<ip>' port port='$p' protocol='<protocol>' %(rich-suffix)s +actionban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="%(fwcmd_rich_rule)s"; done + +actionunban = ports="<port>"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="%(fwcmd_rich_rule)s"; done +rich-suffix = <rich-blocktype>
\ No newline at end of file diff --git a/config/action.d/helpers-common.conf b/config/action.d/helpers-common.conf index b036f68f..03422a87 100644 --- a/config/action.d/helpers-common.conf +++ b/config/action.d/helpers-common.conf @@ -4,8 +4,9 @@ # _grep_logs_args = 'test' # (printf %%b "Log-excerpt contains 'test':\n"; %(_grep_logs)s; printf %%b "Log-excerpt contains 'test':\n") | mail ... # -_grep_logs = logpath="<logpath>"; grep <grepopts> -E %(_grep_logs_args)s $logpath | <greplimit> -_grep_logs_args = "(^|[^0-9a-fA-F:])$(echo '<ip>' | sed 's/\./\\./g')([^0-9a-fA-F:]|$)" +_grep_logs = logpath="<logpath>"; grep <grepopts> %(_grep_logs_args)s $logpath | <greplimit> +# options `-wF` used to match only whole words and fixed string (not as pattern) +_grep_logs_args = -wF "<ip>" # Used for actions, that should not by executed if ticket was restored: _bypass_if_restored = if [ '<restored>' = '1' ]; then exit 0; fi; diff --git a/config/action.d/iptables-allports.conf b/config/action.d/iptables-allports.conf index caf9ab81..51c4694d 100644 --- a/config/action.d/iptables-allports.conf +++ b/config/action.d/iptables-allports.conf @@ -4,52 +4,12 @@ # Modified: Yaroslav O. Halchenko <debian@onerussian.com> # made active on all ports from original iptables.conf # -# +# Obsolete: superseded by iptables[type=allports] [INCLUDES] -before = iptables-common.conf - +before = iptables.conf [Definition] -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = <iptables> -N f2b-<name> - <iptables> -A f2b-<name> -j <returntype> - <iptables> -I <chain> -p <protocol> -j f2b-<name> - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = <iptables> -D <chain> -p <protocol> -j f2b-<name> - <actionflush> - <iptables> -X f2b-<name> - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = <iptables> -n -L <chain> | grep -q 'f2b-<name>[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = <iptables> -I f2b-<name> 1 -s <ip> -j <blocktype> - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = <iptables> -D f2b-<name> -s <ip> -j <blocktype> - -[Init] - +type = allports diff --git a/config/action.d/iptables-common.conf b/config/action.d/iptables-common.conf deleted file mode 100644 index e016ef2f..00000000 --- a/config/action.d/iptables-common.conf +++ /dev/null @@ -1,92 +0,0 @@ -# Fail2Ban configuration file -# -# Author: Daniel Black -# -# This is a included configuration file and includes the definitions for the iptables -# used in all iptables based actions by default. -# -# The user can override the defaults in iptables-common.local -# -# Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de> -# made config file IPv6 capable (see new section Init?family=inet6) - -[INCLUDES] - -after = iptables-blocktype.local - iptables-common.local -# iptables-blocktype.local is obsolete - -[Definition] - -# Option: actionflush -# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) -# Values: CMD -# -actionflush = <iptables> -F f2b-<name> - - -[Init] - -# Option: chain -# Notes specifies the iptables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: INPUT -chain = INPUT - -# Default name of the chain -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp | icmp | all ] Default: tcp -# -protocol = tcp - -# Option: blocktype -# Note: This is what the action does with rules. This can be any jump target -# as per the iptables man page (section 8). Common values are DROP -# REJECT, REJECT --reject-with icmp-port-unreachable -# Values: STRING -blocktype = REJECT --reject-with icmp-port-unreachable - -# Option: returntype -# Note: This is the default rule on "actionstart". This should be RETURN -# in all (blocking) actions, except REJECT in allowing actions. -# Values: STRING -returntype = RETURN - -# Option: lockingopt -# Notes.: Option was introduced to iptables to prevent multiple instances from -# running concurrently and causing irratic behavior. -w was introduced -# in iptables 1.4.20, so might be absent on older systems -# See https://github.com/fail2ban/fail2ban/issues/1122 -# Values: STRING -lockingopt = -w - -# Option: iptables -# Notes.: Actual command to be executed, including common to all calls options -# Values: STRING -iptables = iptables <lockingopt> - - -[Init?family=inet6] - -# Option: blocktype (ipv6) -# Note: This is what the action does with rules. This can be any jump target -# as per the iptables man page (section 8). Common values are DROP -# REJECT, REJECT --reject-with icmp6-port-unreachable -# Values: STRING -blocktype = REJECT --reject-with icmp6-port-unreachable - -# Option: iptables (ipv6) -# Notes.: Actual command to be executed, including common to all calls options -# Values: STRING -iptables = ip6tables <lockingopt> - diff --git a/config/action.d/iptables-ipset-proto4.conf b/config/action.d/iptables-ipset-proto4.conf index 99ebbf8c..37624284 100644 --- a/config/action.d/iptables-ipset-proto4.conf +++ b/config/action.d/iptables-ipset-proto4.conf @@ -19,7 +19,7 @@ [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] @@ -28,7 +28,7 @@ before = iptables-common.conf # Values: CMD # actionstart = ipset --create f2b-<name> iphash - <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype> + <_ipt_add_rules> # Option: actionflush @@ -41,7 +41,7 @@ actionflush = ipset --flush f2b-<name> # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) # Values: CMD # -actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set f2b-<name> src -j <blocktype> +actionstop = <_ipt_del_rules> <actionflush> ipset --destroy f2b-<name> @@ -61,5 +61,6 @@ actionban = ipset --test f2b-<name> <ip> || ipset --add f2b-<name> <ip> # actionunban = ipset --test f2b-<name> <ip> && ipset --del f2b-<name> <ip> -[Init] +# Several capabilities used internaly: +rule-jump = -m set --match-set f2b-<name> src -j <blocktype> diff --git a/config/action.d/iptables-ipset-proto6-allports.conf b/config/action.d/iptables-ipset-proto6-allports.conf index c851233c..1aa7fd6f 100644 --- a/config/action.d/iptables-ipset-proto6-allports.conf +++ b/config/action.d/iptables-ipset-proto6-allports.conf @@ -15,65 +15,13 @@ # # Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de> # made config file IPv6 capable (see new section Init?family=inet6) +# +# Obsolete: superseded by iptables-ipset[type=allports] [INCLUDES] -before = iptables-common.conf +before = iptables-ipset.conf [Definition] -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt> - <iptables> -I <chain> -m set --match-set <ipmset> src -j <blocktype> - -# Option: actionflush -# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) -# Values: CMD -# -actionflush = ipset flush <ipmset> - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = <iptables> -D <chain> -m set --match-set <ipmset> src -j <blocktype> - <actionflush> - ipset destroy <ipmset> - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = ipset add <ipmset> <ip> timeout <bantime> -exist - -actionprolong = %(actionban)s - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = ipset del <ipmset> <ip> -exist - -[Init] - -# Option: default-timeout -# Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 - -default-timeout = 600 - -ipmset = f2b-<name> -familyopt = - - -[Init?family=inet6] - -ipmset = f2b-<name>6 -familyopt = <sp>family inet6 +type = allports diff --git a/config/action.d/iptables-ipset-proto6.conf b/config/action.d/iptables-ipset-proto6.conf index 12c3ddd6..ef744984 100644 --- a/config/action.d/iptables-ipset-proto6.conf +++ b/config/action.d/iptables-ipset-proto6.conf @@ -15,65 +15,13 @@ # # Modified: Alexander Koeppe <format_c@online.de>, Serg G. Brester <serg.brester@sebres.de> # made config file IPv6 capable (see new section Init?family=inet6) +# +# Obsolete: superseded by iptables-ipset[type=multiport] [INCLUDES] -before = iptables-common.conf +before = iptables-ipset.conf [Definition] -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = ipset create <ipmset> hash:ip timeout <default-timeout><familyopt> - <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype> - -# Option: actionflush -# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) -# Values: CMD -# -actionflush = ipset flush <ipmset> - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype> - <actionflush> - ipset destroy <ipmset> - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = ipset add <ipmset> <ip> timeout <bantime> -exist - -actionprolong = %(actionban)s - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = ipset del <ipmset> <ip> -exist - -[Init] - -# Option: default-timeout -# Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 - -default-timeout = 600 - -ipmset = f2b-<name> -familyopt = - - -[Init?family=inet6] - -ipmset = f2b-<name>6 -familyopt = <sp>family inet6 +type = multiport diff --git a/config/action.d/iptables-ipset.conf b/config/action.d/iptables-ipset.conf new file mode 100644 index 00000000..b44e6ec4 --- /dev/null +++ b/config/action.d/iptables-ipset.conf @@ -0,0 +1,90 @@ +# Fail2Ban configuration file +# +# Authors: Sergey G Brester (sebres), Daniel Black, Alexander Koeppe +# +# This is for ipset protocol 6 (and hopefully later) (ipset v6.14). +# Use ipset -V to see the protocol and version. Version 4 should use +# iptables-ipset-proto4.conf. +# +# This requires the program ipset which is normally in package called ipset. +# +# IPset was a feature introduced in the linux kernel 2.6.39 and 3.0.0 kernels. +# +# If you are running on an older kernel you make need to patch in external +# modules. +# + +[INCLUDES] + +before = iptables.conf + +[Definition] + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = ipset -exist create <ipmset> hash:ip timeout <default-ipsettime> <familyopt> + <_ipt_add_rules> + +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = ipset flush <ipmset> + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = <_ipt_del_rules> + <actionflush> + ipset destroy <ipmset> + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = ipset -exist add <ipmset> <ip> timeout <ipsettime> + +# actionprolong = %(actionban)s + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = ipset -exist del <ipmset> <ip> + +# Several capabilities used internaly: + +rule-jump = -m set --match-set <ipmset> src -j <blocktype> + + +[Init] + +# Option: default-ipsettime +# Notes: specifies default timeout in seconds (handled default ipset timeout only) +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 + +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>'] +timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0) + +ipmset = f2b-<name> +familyopt = + + +[Init?family=inet6] + +ipmset = f2b-<name>6 +familyopt = family inet6 diff --git a/config/action.d/iptables-multiport-log.conf b/config/action.d/iptables-multiport-log.conf index df126dbf..322a7491 100644 --- a/config/action.d/iptables-multiport-log.conf +++ b/config/action.d/iptables-multiport-log.conf @@ -11,7 +11,7 @@ [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] diff --git a/config/action.d/iptables-multiport.conf b/config/action.d/iptables-multiport.conf index 41b00c54..008208e0 100644 --- a/config/action.d/iptables-multiport.conf +++ b/config/action.d/iptables-multiport.conf @@ -3,50 +3,12 @@ # Author: Cyril Jaquier # Modified by Yaroslav Halchenko for multiport banning # +# Obsolete: superseded by iptables[type=multiport] [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = <iptables> -N f2b-<name> - <iptables> -A f2b-<name> -j <returntype> - <iptables> -I <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name> - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = <iptables> -D <chain> -p <protocol> -m multiport --dports <port> -j f2b-<name> - <actionflush> - <iptables> -X f2b-<name> - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = <iptables> -n -L <chain> | grep -q 'f2b-<name>[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = <iptables> -I f2b-<name> 1 -s <ip> -j <blocktype> - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = <iptables> -D f2b-<name> -s <ip> -j <blocktype> - -[Init] - +type = multiport diff --git a/config/action.d/iptables-new.conf b/config/action.d/iptables-new.conf index 39a17099..170cb934 100644 --- a/config/action.d/iptables-new.conf +++ b/config/action.d/iptables-new.conf @@ -4,51 +4,12 @@ # Copied from iptables.conf and modified by Yaroslav Halchenko # to fulfill the needs of bugreporter dbts#350746. # -# +# Obsolete: superseded by iptables[pre-rule='-m state --state NEW<sp>'] [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = <iptables> -N f2b-<name> - <iptables> -A f2b-<name> -j <returntype> - <iptables> -I <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name> - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = <iptables> -D <chain> -m state --state NEW -p <protocol> --dport <port> -j f2b-<name> - <actionflush> - <iptables> -X f2b-<name> - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = <iptables> -n -L <chain> | grep -q 'f2b-<name>[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = <iptables> -I f2b-<name> 1 -s <ip> -j <blocktype> - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = <iptables> -D f2b-<name> -s <ip> -j <blocktype> - -[Init] - +pre-rule = -m state --state NEW<sp>
\ No newline at end of file diff --git a/config/action.d/iptables-xt_recent-echo.conf b/config/action.d/iptables-xt_recent-echo.conf index 97449222..c3c175b3 100644 --- a/config/action.d/iptables-xt_recent-echo.conf +++ b/config/action.d/iptables-xt_recent-echo.conf @@ -7,10 +7,14 @@ [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] +_ipt_chain_rule = -m recent --update --seconds 3600 --name <iptname> -j <blocktype> +_ipt_for_proto-iter = +_ipt_for_proto-done = + # Option: actionstart # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD @@ -33,7 +37,9 @@ before = iptables-common.conf # own rules. The 3600 second timeout is independent and acts as a # safeguard in case the fail2ban process dies unexpectedly. The # shorter of the two timeouts actually matters. -actionstart = if [ `id -u` -eq 0 ];then <iptables> -I <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi +actionstart = if [ `id -u` -eq 0 ];then + { %(_ipt_check_rule)s >/dev/null 2>&1; } || { <iptables> -I <chain> %(_ipt_chain_rule)s; } + fi # Option: actionflush # @@ -46,13 +52,15 @@ actionflush = # Values: CMD # actionstop = echo / > /proc/net/xt_recent/<iptname> - if [ `id -u` -eq 0 ];then <iptables> -D <chain> -m recent --update --seconds 3600 --name <iptname> -j <blocktype>;fi + if [ `id -u` -eq 0 ];then + <iptables> -D <chain> %(_ipt_chain_rule)s; + fi # Option: actioncheck -# Notes.: command executed once before each actionban command +# Notes.: command executed as invariant check (error by ban) # Values: CMD # -actioncheck = test -e /proc/net/xt_recent/<iptname> +actioncheck = { <iptables> -C <chain> %(_ipt_chain_rule)s; } && test -e /proc/net/xt_recent/<iptname> # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -72,7 +80,7 @@ actionunban = echo -<ip> > /proc/net/xt_recent/<iptname> [Init] -iptname = f2b-<name> +iptname = f2b-<name> [Init?family=inet6] diff --git a/config/action.d/iptables.conf b/config/action.d/iptables.conf index 8ed5fdad..67d496f5 100644 --- a/config/action.d/iptables.conf +++ b/config/action.d/iptables.conf @@ -1,28 +1,35 @@ # Fail2Ban configuration file # -# Author: Cyril Jaquier -# +# Authors: Sergey G. Brester (sebres), Cyril Jaquier, Daniel Black, +# Yaroslav O. Halchenko, Alexander Koeppe et al. # -[INCLUDES] +[Definition] -before = iptables-common.conf +# Option: type +# Notes.: type of the action. +# Values: [ oneport | multiport | allports ] Default: oneport +# +type = oneport -[Definition] +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action) +# Values: CMD +# +actionflush = <iptables> -F f2b-<name> # Option: actionstart # Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). # Values: CMD # -actionstart = <iptables> -N f2b-<name> - <iptables> -A f2b-<name> -j <returntype> - <iptables> -I <chain> -p <protocol> --dport <port> -j f2b-<name> +actionstart = { <iptables> -C f2b-<name> -j <returntype> >/dev/null 2>&1; } || { <iptables> -N f2b-<name> || true; <iptables> -A f2b-<name> -j <returntype>; } + <_ipt_add_rules> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) # Values: CMD # -actionstop = <iptables> -D <chain> -p <protocol> --dport <port> -j f2b-<name> +actionstop = <_ipt_del_rules> <actionflush> <iptables> -X f2b-<name> @@ -30,7 +37,7 @@ actionstop = <iptables> -D <chain> -p <protocol> --dport <port> -j f2b-<name> # Notes.: command executed once before each actionban command # Values: CMD # -actioncheck = <iptables> -n -L <chain> | grep -q 'f2b-<name>[ \t]' +actioncheck = <_ipt_check_rules> # Option: actionban # Notes.: command executed when banning an IP. Take care that the @@ -48,5 +55,108 @@ actionban = <iptables> -I f2b-<name> 1 -s <ip> -j <blocktype> # actionunban = <iptables> -D f2b-<name> -s <ip> -j <blocktype> +# Option: pre-rule +# Notes.: prefix parameter(s) inserted to the begin of rule. No default (empty) +# +pre-rule = + +rule-jump = -j <_ipt_rule_target> + +# Several capabilities used internaly: + +_ipt_for_proto-iter = for proto in $(echo '<protocol>' | sed 's/,/ /g'); do +_ipt_for_proto-done = done + +_ipt_add_rules = <_ipt_for_proto-iter> + { %(_ipt_check_rule)s >/dev/null 2>&1; } || { <iptables> -I <chain> %(_ipt_chain_rule)s; } + <_ipt_for_proto-done> + +_ipt_del_rules = <_ipt_for_proto-iter> + <iptables> -D <chain> %(_ipt_chain_rule)s + <_ipt_for_proto-done> + +_ipt_check_rules = <_ipt_for_proto-iter> + %(_ipt_check_rule)s + <_ipt_for_proto-done> + +_ipt_chain_rule = <pre-rule><ipt_<type>/_chain_rule> +_ipt_check_rule = <iptables> -C <chain> %(_ipt_chain_rule)s +_ipt_rule_target = f2b-<name> + +[ipt_oneport] + +_chain_rule = -p $proto --dport <port> <rule-jump> + +[ipt_multiport] + +_chain_rule = -p $proto -m multiport --dports <port> <rule-jump> + +[ipt_allports] + +_chain_rule = -p $proto <rule-jump> + + [Init] +# Option: chain +# Notes specifies the iptables chain to which the Fail2Ban rules should be +# added +# Values: STRING Default: INPUT +chain = INPUT + +# Default name of the chain +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp | icmp | all ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the iptables man page (section 8). Common values are DROP +# REJECT, REJECT --reject-with icmp-port-unreachable +# Values: STRING +blocktype = REJECT --reject-with icmp-port-unreachable + +# Option: returntype +# Note: This is the default rule on "actionstart". This should be RETURN +# in all (blocking) actions, except REJECT in allowing actions. +# Values: STRING +returntype = RETURN + +# Option: lockingopt +# Notes.: Option was introduced to iptables to prevent multiple instances from +# running concurrently and causing irratic behavior. -w was introduced +# in iptables 1.4.20, so might be absent on older systems +# See https://github.com/fail2ban/fail2ban/issues/1122 +# Values: STRING +lockingopt = -w + +# Option: iptables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +iptables = iptables <lockingopt> + + +[Init?family=inet6] + +# Option: blocktype (ipv6) +# Note: This is what the action does with rules. This can be any jump target +# as per the iptables man page (section 8). Common values are DROP +# REJECT, REJECT --reject-with icmp6-port-unreachable +# Values: STRING +blocktype = REJECT --reject-with icmp6-port-unreachable + +# Option: iptables (ipv6) +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +iptables = ip6tables <lockingopt> diff --git a/config/action.d/ipthreat.conf b/config/action.d/ipthreat.conf new file mode 100644 index 00000000..193a60f2 --- /dev/null +++ b/config/action.d/ipthreat.conf @@ -0,0 +1,107 @@ +# IPThreat configuration file +# +# Added to fail2ban by Jeff Johnson (jjxtra) +# +# Action to report IP address to ipthreat.net +# +# You must sign up to obtain an API key from ipthreat.net and request bulk report permissions +# https://ipthreat.net/integrations +# +# IPThreat is a 100% free site and service, all data is licensed under a creative commons by attribution license +# Please do not integrate if you do not agree to the license +# +# IMPORTANT: +# +# Reporting an IP is a serious action. Make sure that it is legit. +# Consider using this action only for: +# * IP that has been banned more than once +# * High max retry to avoid user mis-typing password +# * Filters that are unlikely to be human error +# +# Example: +# ``` +# action = %(known/action)s +# ipthreat[] +# ``` +# +# The action accepts the following arguments: ipthreat[ipthreat_flags="8",ipthreat_system="SSH", ipthreat_apikey=...] +# In most cases your action could be as simple as: ipthreat[], since the default flags and system are set to the most correct default values. +# You can optionally override ipthreat_system and ipthreat_flags if desired. +# The ipthreat_apikey must be set at the bottom of this configuration file. +# +# `ipthreat_system` is a short name of the system attacked, i.e. SSH, SMTP, MYSQL, PHP, etc. +# +# For `ipthreat_flags`, most cases will use 8 (BruteForce) which is the default, but you could use others. +# You can use the name or the ordinal. +# Multiple values are comma separated. +# ``` +# Name Ordinal Description +# Dns 1 Abuse/attack of dns (domain name server) +# Fraud 2 General fraud, whether orders, misuse of payment info, etc +# DDos 4 Distributed denial of service attack, whether through http requests, large ping attack, etc +# BruteForce 8 Brute force login attack +# Proxy 16 IP is a proxy like TOR or other proxy server +# Spam 32 Email, comment or other type of spam +# Vpn 64 IP is part of a VPN +# Hacking 128 General hacking outside of brute force attack (includes vulnerability scans, sql injection, etc.). Use port scan flag instead if it's just probe on ports. +# BadBot 256 Bad bot that is not honoring robots.txt or just flooding with too many requests, etc +# Compromised 512 The ip has been taken over by malware or botnet +# Phishing 1024 The ip is involved in phishing or spoofing +# Iot 2048 The ip has targetted an iot (Internet of Things) device +# PortScan 4096 Port scan +# See https://ipthreat.net/bulkreportformat for more information +# ``` + +[Definition] + +# bypass action for restored tickets +norestored = 1 + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = curl -sSf "https://api.ipthreat.net/api/report" -X POST -H "Content-Type: application/json" -H "X-API-KEY: <ipthreat_apikey>" -d "{\"ip\":\"<ip>\",\"flags\":\"<ipthreat_flags>\",\"system\":\"<ipthreat_system>\",\"notes\":\"fail2ban\"}" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] +# Option: ipthreat_apikey +# Notes Your API key from ipthreat.net +# Values: STRING Default: None +# Register for ipthreat [https://ipthreat.net], get api key and set below. +# You will need to set the flags and system in the action call in jail.conf +ipthreat_apikey = + +# By default, the ipthreat system is the name of the fail2ban jail +ipthreat_system = <name> + +# By default the ip threat flags is 8 (brute force), but you can override this per jail if desired +ipthreat_flags = 8
\ No newline at end of file diff --git a/config/action.d/mail-buffered.conf b/config/action.d/mail-buffered.conf index 325f185b..79b84104 100644 --- a/config/action.d/mail-buffered.conf +++ b/config/action.d/mail-buffered.conf @@ -17,7 +17,7 @@ actionstart = printf %%b "Hi,\n The jail <name> has been started successfully.\n Output will be buffered until <lines> lines are available.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -28,13 +28,13 @@ actionstop = if [ -f <tmpfile> ]; then These hosts have been banned by Fail2Ban.\n `cat <tmpfile>` Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary from <fq-hostname>" <dest> rm <tmpfile> fi printf %%b "Hi,\n The jail <name> has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> # Option: actioncheck # Notes.: command executed once before each actionban command @@ -55,7 +55,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile> These hosts have been banned by Fail2Ban.\n `cat <tmpfile>` \nRegards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: Summary" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: Summary" <dest> rm <tmpfile> fi diff --git a/config/action.d/mail-whois-common.conf b/config/action.d/mail-whois-common.conf index b0d27afc..ecf3a5d9 100644 --- a/config/action.d/mail-whois-common.conf +++ b/config/action.d/mail-whois-common.conf @@ -17,7 +17,7 @@ _whois = whois <ip> || echo "missing whois program" # character set before sending it to a mail program # make sure you have 'file' and 'iconv' commands installed when opting for that _whois_target_charset = UTF-8 -_whois_convert_charset = whois <ip> | +_whois_convert_charset = (%(_whois)s) | { WHOIS_OUTPUT=$(cat) ; WHOIS_CHARSET=$(printf %%b "$WHOIS_OUTPUT" | file -b --mime-encoding -) ; printf %%b "$WHOIS_OUTPUT" | iconv -f $WHOIS_CHARSET -t %(_whois_target_charset)s//TRANSLIT - ; } # choose between _whois and _whois_convert_charset in mail-whois-common.local diff --git a/config/action.d/mail-whois-lines.conf b/config/action.d/mail-whois-lines.conf index 3a3e56b2..d2818cb9 100644 --- a/config/action.d/mail-whois-lines.conf +++ b/config/action.d/mail-whois-lines.conf @@ -72,7 +72,7 @@ actionunban = # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Default name of the chain # diff --git a/config/action.d/mail-whois.conf b/config/action.d/mail-whois.conf index 7fea34c4..ab33b616 100644 --- a/config/action.d/mail-whois.conf +++ b/config/action.d/mail-whois.conf @@ -20,7 +20,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail <name> has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -29,7 +29,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail <name> has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> # Option: actioncheck # Notes.: command executed once before each actionban command @@ -49,7 +49,7 @@ actionban = printf %%b "Hi,\n Here is more information about <ip> :\n `%(_whois_command)s`\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/mail.conf b/config/action.d/mail.conf index 5d8c0e15..f4838ddc 100644 --- a/config/action.d/mail.conf +++ b/config/action.d/mail.conf @@ -16,7 +16,7 @@ norestored = 1 actionstart = printf %%b "Hi,\n The jail <name> has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: started on <fq-hostname>" <dest> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -25,7 +25,7 @@ actionstart = printf %%b "Hi,\n actionstop = printf %%b "Hi,\n The jail <name> has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: stopped on <fq-hostname>" <dest> # Option: actioncheck # Notes.: command executed once before each actionban command @@ -43,7 +43,7 @@ actionban = printf %%b "Hi,\n The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest> + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] <name>: banned <ip> from <fq-hostname>" <dest> # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff --git a/config/action.d/nftables-allports.conf b/config/action.d/nftables-allports.conf index 6c69da39..908abe40 100644 --- a/config/action.d/nftables-allports.conf +++ b/config/action.d/nftables-allports.conf @@ -6,17 +6,12 @@ # Modified: Alexander Belykh <albel727@ngs.ru> # adapted for nftables # +# Obsolete: superseded by nftables[type=allports] [INCLUDES] -before = nftables-common.conf +before = nftables.conf [Definition] -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = meta l4proto <protocol> - -[Init] +type = allports diff --git a/config/action.d/nftables-common.conf b/config/action.d/nftables-common.conf deleted file mode 100644 index 37045712..00000000 --- a/config/action.d/nftables-common.conf +++ /dev/null @@ -1,135 +0,0 @@ -# Fail2Ban configuration file -# -# Author: Daniel Black -# Author: Cyril Jaquier -# Modified: Yaroslav O. Halchenko <debian@onerussian.com> -# made active on all ports from original iptables.conf -# Modified: Alexander Belykh <albel727@ngs.ru> -# adapted for nftables -# -# This is a included configuration file and includes the definitions for the nftables -# used in all nftables based actions by default. -# -# The user can override the defaults in nftables-common.local - -[INCLUDES] - -after = nftables-common.local - -[Definition] - -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = <protocol> dport \{ <port> \} - -# Option: actionstart -# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). -# Values: CMD -# -actionstart = <nftables> add set <nftables_family> <nftables_table> <set_name> \{ type <nftables_type>\; \} - <nftables> insert rule <nftables_family> <nftables_table> <chain> %(nftables_mode)s <address_family> saddr @<set_name> <blocktype> - -_nft_list = <nftables> --handle --numeric list chain <nftables_family> <nftables_table> <chain> -_nft_get_handle_id = grep -m1 '<address_family> saddr @<set_name> <blocktype> # handle' | grep -oe ' handle [0-9]*' - -# Option: actionstop -# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) -# Values: CMD -# -actionstop = HANDLE_ID=$(%(_nft_list)s | %(_nft_get_handle_id)s) - <nftables> delete rule <nftables_family> <nftables_table> <chain> $HANDLE_ID - <nftables> delete set <nftables_family> <nftables_table> <set_name> - -# Option: actioncheck -# Notes.: command executed once before each actionban command -# Values: CMD -# -actioncheck = <nftables> list chain <nftables_family> <nftables_table> <chain> | grep -q '@<set_name>[ \t]' - -# Option: actionban -# Notes.: command executed when banning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionban = <nftables> add element <nftables_family> <nftables_table> <set_name> \{ <ip> \} - -# Option: actionunban -# Notes.: command executed when unbanning an IP. Take care that the -# command is executed with Fail2Ban user rights. -# Tags: See jail.conf(5) man page -# Values: CMD -# -actionunban = <nftables> delete element <nftables_family> <nftables_table> <set_name> \{ <ip> \} - -[Init] - -# Option: nftables_type -# Notes.: address type to work with -# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr -# -nftables_type = ipv4_addr - -# Option: nftables_family -# Notes.: address family to work in -# Values: [ip | ip6 | inet] Default: inet -# -nftables_family = inet - -# Option: nftables_table -# Notes.: table in the address family to work in -# Values: STRING Default: filter -# -nftables_table = filter - -# Option: chain -# Notes specifies the nftables chain to which the Fail2Ban rules should be -# added -# Values: STRING Default: input -chain = input - -# Default name of the filtering set -# -name = default - -# Option: port -# Notes.: specifies port to monitor -# Values: [ NUM | STRING ] Default: -# -port = ssh - -# Option: protocol -# Notes.: internally used by config reader for interpolations. -# Values: [ tcp | udp ] Default: tcp -# -protocol = tcp - -# Option: blocktype -# Note: This is what the action does with rules. This can be any jump target -# as per the nftables man page (section 8). Common values are drop -# reject, reject with icmp type host-unreachable -# Values: STRING -blocktype = reject - -# Option: nftables -# Notes.: Actual command to be executed, including common to all calls options -# Values: STRING -nftables = nft - -# Option: set_name -# Notes.: The name of the nft set used to store banned addresses -# Values: STRING -set_name = f2b-<name> - -# Option: address_family -# Notes.: The family of the banned addresses -# Values: [ ip | ip6 ] -address_family = ip - -[Init?family=inet6] - -nftables_type = ipv6_addr -set_name = f2b-<name>6 -address_family = ip6 diff --git a/config/action.d/nftables-multiport.conf b/config/action.d/nftables-multiport.conf index d1afafb3..ba3ec92c 100644 --- a/config/action.d/nftables-multiport.conf +++ b/config/action.d/nftables-multiport.conf @@ -6,17 +6,12 @@ # Modified: Alexander Belykh <albel727@ngs.ru> # adapted for nftables # +# Obsolete: superseded by nftables[type=multiport] [INCLUDES] -before = nftables-common.conf +before = nftables.conf [Definition] -# Option: nftables_mode -# Notes.: additional expressions for nftables filter rule -# Values: nftables expressions -# -nftables_mode = <protocol> dport \{ <port> \} - -[Init] +type = multiport
\ No newline at end of file diff --git a/config/action.d/nftables.conf b/config/action.d/nftables.conf new file mode 100644 index 00000000..77cf3661 --- /dev/null +++ b/config/action.d/nftables.conf @@ -0,0 +1,203 @@ +# Fail2Ban configuration file +# +# Author: Daniel Black +# Author: Cyril Jaquier +# Modified: Yaroslav O. Halchenko <debian@onerussian.com> +# made active on all ports from original iptables.conf +# Modified: Alexander Belykh <albel727@ngs.ru> +# adapted for nftables +# +# This is a included configuration file and includes the definitions for the nftables +# used in all nftables based actions by default. +# +# The user can override the defaults in nftables-common.local +# Example: redirect flow to honeypot +# +# [Init] +# table_family = ip +# chain_type = nat +# chain_hook = prerouting +# chain_priority = -50 +# blocktype = counter redirect to 2222 + +[INCLUDES] + +after = nftables-common.local + +[Definition] + +# Option: type +# Notes.: type of the action. +# Values: [ multiport | allports ] Default: multiport +# +type = multiport + +rule_match-custom = +rule_match-allports = meta l4proto \{ <protocol> \} +rule_match-multiport = $proto dport \{ $(echo '<port>' | sed s/:/-/g) \} +match = <rule_match-<type>> + +# Option: rule_stat +# Notes.: statement for nftables filter rule. +# leaving it empty will block all (include udp and icmp) +# Values: nftables statement +# +rule_stat = %(match)s <addr_family> saddr @<addr_set> <blocktype> + +# optional interator over protocol's: +_nft_for_proto-custom-iter = +_nft_for_proto-custom-done = +_nft_for_proto-allports-iter = +_nft_for_proto-allports-done = +_nft_for_proto-multiport-iter = for proto in $(echo '<protocol>' | sed 's/,/ /g'); do +_nft_for_proto-multiport-done = done + +_nft_list = <nftables> -a list chain <table_family> <table> <chain> +_nft_get_handle_id = grep -oP '@<addr_set>\s+.*\s+\Khandle\s+(\d+)$' + +_nft_add_set = <nftables> add set <table_family> <table> <addr_set> \{ type <addr_type>\; \} + <_nft_for_proto-<type>-iter> + <nftables> add rule <table_family> <table> <chain> %(rule_stat)s + <_nft_for_proto-<type>-done> +_nft_del_set = { %(_nft_list)s | %(_nft_get_handle_id)s; } | while read -r hdl; do + <nftables> delete rule <table_family> <table> <chain> $hdl; done + <nftables> delete set <table_family> <table> <addr_set> + +# Option: _nft_shutdown_table +# Notes.: command executed after the stop in order to delete table (it checks that no sets are available): +# Values: CMD +# +_nft_shutdown_table = { <nftables> list table <table_family> <table> | grep -qP '^\s+set\s+'; } || { + <nftables> delete table <table_family> <table> + } + +# Option: actionstart +# Notes.: command executed on demand at the first ban (or at the start of Fail2Ban if actionstart_on_demand is set to false). +# Values: CMD +# +actionstart = <nftables> add table <table_family> <table> + <nftables> -- add chain <table_family> <table> <chain> \{ type <chain_type> hook <chain_hook> priority <chain_priority> \; \} + %(_nft_add_set)s + +# Option: actionflush +# Notes.: command executed once to flush IPS, by shutdown (resp. by stop of the jail or this action); +# uses `nft flush set ...` and as fallback (e. g. unsupported) recreates the set (with references) +# Values: CMD +# +actionflush = { <nftables> flush set <table_family> <table> <addr_set> 2> /dev/null; } || { + %(_nft_del_set)s + %(_nft_add_set)s + } + +# Option: actionstop +# Notes.: command executed at the stop of jail (or at the end of Fail2Ban) +# Values: CMD +# +actionstop = %(_nft_del_set)s + <_nft_shutdown_table> + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = <nftables> list chain <table_family> <table> <chain> | grep -q '@<addr_set>[ \t]' + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = <nftables> add element <table_family> <table> <addr_set> \{ <ip> \} + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = <nftables> delete element <table_family> <table> <addr_set> \{ <ip> \} + +[Init] + +# Option: table +# Notes.: main table to store chain and sets (automatically created on demand) +# Values: STRING Default: f2b-table +table = f2b-table + +# Option: table_family +# Notes.: address family to work in +# Values: [ip | ip6 | inet] Default: inet +table_family = inet + +# Option: chain +# Notes.: main chain to store rules +# Values: STRING Default: f2b-chain +chain = f2b-chain + +# Option: chain_type +# Notes.: refers to the kind of chain to be created +# Values: [filter | route | nat] Default: filter +# +chain_type = filter + +# Option: chain_hook +# Notes.: refers to the kind of chain to be created +# Values: [ prerouting | input | forward | output | postrouting ] Default: input +# +chain_hook = input + +# Option: chain_priority +# Notes.: priority in the chain. +# Values: NUMBER Default: -1 +# +chain_priority = -1 + +# Option: addr_type +# Notes.: address type to work with +# Values: [ipv4_addr | ipv6_addr] Default: ipv4_addr +# +addr_type = ipv4_addr + +# Default name of the filtering set +# +name = default + +# Option: port +# Notes.: specifies port to monitor +# Values: [ NUM | STRING ] Default: +# +port = ssh + +# Option: protocol +# Notes.: internally used by config reader for interpolations. +# Values: [ tcp | udp ] Default: tcp +# +protocol = tcp + +# Option: blocktype +# Note: This is what the action does with rules. This can be any jump target +# as per the nftables man page (section 8). Common values are drop, +# reject, reject with icmpx type host-unreachable, redirect to 2222 +# Values: STRING +blocktype = reject + +# Option: nftables +# Notes.: Actual command to be executed, including common to all calls options +# Values: STRING +nftables = nft + +# Option: addr_set +# Notes.: The name of the nft set used to store banned addresses +# Values: STRING +addr_set = addr-set-<name> + +# Option: addr_family +# Notes.: The family of the banned addresses +# Values: [ ip | ip6 ] +addr_family = ip + +[Init?family=inet6] +addr_family = ip6 +addr_type = ipv6_addr +addr_set = addr6-set-<name> diff --git a/config/action.d/nginx-block-map.conf b/config/action.d/nginx-block-map.conf index 33c15f9c..0de382bd 100644 --- a/config/action.d/nginx-block-map.conf +++ b/config/action.d/nginx-block-map.conf @@ -84,8 +84,15 @@ srv_cfg_path = /etc/nginx/ #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf srv_cmd = nginx -# first test configuration is correct, hereafter send reload signal: -blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then +# pid file (used to check nginx is running): +srv_pid = /run/nginx.pid + +# command used to check whether nginx is running and configuration is valid: +srv_is_running = [ -f "%(srv_pid)s" ] +srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt + +# first test nginx is running and configuration is correct, hereafter send reload signal: +blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi; fi; @@ -103,6 +110,8 @@ actionstop = %(actionflush)s actioncheck = -actionban = echo "\\\\<fid> 1;" >> '%(blck_lst_file)s'; %(blck_lst_reload)s +_echo_blck_row = printf '\%%s 1;\n' "<fid>" + +actionban = %(_echo_blck_row)s >> '%(blck_lst_file)s'; %(blck_lst_reload)s -actionunban = id=$(echo "<fid>" | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/$id 1;/d" %(blck_lst_file)s; %(blck_lst_reload)s +actionunban = id=$(%(_echo_blck_row)s | sed -e 's/[]\/$*.^|[]/\\&/g'); sed -i "/^$id$/d" %(blck_lst_file)s; %(blck_lst_reload)s diff --git a/config/action.d/sendmail-buffered.conf b/config/action.d/sendmail-buffered.conf index 199c6ce5..13803f8b 100644 --- a/config/action.d/sendmail-buffered.conf +++ b/config/action.d/sendmail-buffered.conf @@ -24,7 +24,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname> The jail <name> has been started successfully.\n Output will be buffered until <lines> lines are available.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -38,7 +38,7 @@ actionstop = if [ -f <tmpfile> ]; then These hosts have been banned by Fail2Ban.\n `cat <tmpfile>` Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> rm <tmpfile> fi printf %%b "Subject: [Fail2Ban] <name>: stopped on <fq-hostname> @@ -47,7 +47,7 @@ actionstop = if [ -f <tmpfile> ]; then Hi,\n The jail <name> has been stopped.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> # Option: actioncheck # Notes.: command executed once before each actionban command @@ -71,7 +71,7 @@ actionban = printf %%b "`date`: <ip> (<failures> failures)\n" >> <tmpfile> These hosts have been banned by Fail2Ban.\n `cat <tmpfile>` Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> rm <tmpfile> fi diff --git a/config/action.d/sendmail-common.conf b/config/action.d/sendmail-common.conf index 9bf15054..1e31fadf 100644 --- a/config/action.d/sendmail-common.conf +++ b/config/action.d/sendmail-common.conf @@ -21,7 +21,7 @@ actionstart = printf %%b "Subject: [Fail2Ban] <name>: started on <fq-hostname> Hi,\n The jail <name> has been started successfully.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -34,7 +34,7 @@ actionstop = printf %%b "Subject: [Fail2Ban] <name>: stopped on <fq-hostname> Hi,\n The jail <name> has been stopped.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> # Option: actioncheck # Notes.: command executed once before each actionban command @@ -60,6 +60,10 @@ actionunban = [Init] +# Your system mail command +# +mailcmd = /usr/sbin/sendmail -f "<sender>" "<dest>" + # Recipient mail address # dest = root diff --git a/config/action.d/sendmail-geoip-lines.conf b/config/action.d/sendmail-geoip-lines.conf index b7c1bf36..b36e49a7 100644 --- a/config/action.d/sendmail-geoip-lines.conf +++ b/config/action.d/sendmail-geoip-lines.conf @@ -37,11 +37,11 @@ actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostn Country:`geoiplookup -f /usr/share/GeoIP/GeoIP.dat "<ip>" | cut -d':' -f2-` AS:`geoiplookup -f /usr/share/GeoIP/GeoIPASNum.dat "<ip>" | cut -d':' -f2-` hostname: <ip-host>\n\n - Lines containing failures of <ip>\n"; + Lines containing failures of <ip> (max <grepmax>)\n"; %(_grep_logs)s; printf %%b "\n Regards,\n - Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" ) | <mailcmd> [Init] diff --git a/config/action.d/sendmail-whois-ipjailmatches.conf b/config/action.d/sendmail-whois-ipjailmatches.conf index 06ea3a3e..7790ec53 100644 --- a/config/action.d/sendmail-whois-ipjailmatches.conf +++ b/config/action.d/sendmail-whois-ipjailmatches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostnam The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n\n Here is more information about <ip> :\n - `/usr/bin/whois <ip>`\n\n + `%(_whois_command)s`\n\n Matches for <name> with <ipjailfailures> failures IP:<ip>\n <ipjailmatches>\n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> [Init] diff --git a/config/action.d/sendmail-whois-ipmatches.conf b/config/action.d/sendmail-whois-ipmatches.conf index 83bff1b4..e4717ca1 100644 --- a/config/action.d/sendmail-whois-ipmatches.conf +++ b/config/action.d/sendmail-whois-ipmatches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostnam The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n\n Here is more information about <ip> :\n - `/usr/bin/whois <ip>`\n\n + `%(_whois_command)s`\n\n Matches with <ipfailures> failures IP:<ip>\n <ipmatches>\n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> [Init] diff --git a/config/action.d/sendmail-whois-lines.conf b/config/action.d/sendmail-whois-lines.conf index 4b947cb2..47ec6ed5 100644 --- a/config/action.d/sendmail-whois-lines.conf +++ b/config/action.d/sendmail-whois-lines.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf helpers-common.conf [Definition] @@ -27,13 +28,13 @@ actionban = ( printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostn Hi,\n The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n\n - Here is more information about <ip> :\n - `/usr/bin/whois <ip> || echo missing whois program`\n\n - Lines containing failures of <ip>\n"; + Here is more information about <ip> :\n" + %(_whois_command)s; + printf %%b "\nLines containing failures of <ip> (max <grepmax>)\n"; %(_grep_logs)s; printf %%b "\n Regards,\n - Fail2Ban" ) | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" ) | <mailcmd> [Init] diff --git a/config/action.d/sendmail-whois-matches.conf b/config/action.d/sendmail-whois-matches.conf index 01520135..08215ea7 100644 --- a/config/action.d/sendmail-whois-matches.conf +++ b/config/action.d/sendmail-whois-matches.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,11 +28,11 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostnam The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n\n Here is more information about <ip> :\n - `/usr/bin/whois <ip>`\n\n + `%(_whois_command)s`\n\n Matches:\n <matches>\n\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> [Init] diff --git a/config/action.d/sendmail-whois.conf b/config/action.d/sendmail-whois.conf index 2fb01ed3..9e93cd32 100644 --- a/config/action.d/sendmail-whois.conf +++ b/config/action.d/sendmail-whois.conf @@ -7,6 +7,7 @@ [INCLUDES] before = sendmail-common.conf + mail-whois-common.conf [Definition] @@ -27,9 +28,9 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostnam The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n\n Here is more information about <ip> :\n - `/usr/bin/whois <ip> || echo missing whois program`\n + `%(_whois_command)s`\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> [Init] diff --git a/config/action.d/sendmail.conf b/config/action.d/sendmail.conf index cf420915..ad9e8d79 100644 --- a/config/action.d/sendmail.conf +++ b/config/action.d/sendmail.conf @@ -27,7 +27,7 @@ actionban = printf %%b "Subject: [Fail2Ban] <name>: banned <ip> from <fq-hostnam The IP <ip> has just been banned by Fail2Ban after <failures> attempts against <name>.\n Regards,\n - Fail2Ban" | /usr/sbin/sendmail -f <sender> <dest> + Fail2Ban" | <mailcmd> [Init] diff --git a/config/action.d/shorewall-ipset-proto6.conf b/config/action.d/shorewall-ipset-proto6.conf index 45be0c0a..eacb53d9 100644 --- a/config/action.d/shorewall-ipset-proto6.conf +++ b/config/action.d/shorewall-ipset-proto6.conf @@ -51,7 +51,7 @@ # Values: CMD # actionstart = if ! ipset -quiet -name list f2b-<name> >/dev/null; - then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-timeout>; + then ipset -quiet -exist create f2b-<name> hash:ip timeout <default-ipsettime>; fi # Option: actionstop @@ -66,9 +66,9 @@ actionstop = ipset flush f2b-<name> # Tags: See jail.conf(5) man page # Values: CMD # -actionban = ipset add f2b-<name> <ip> timeout <bantime> -exist +actionban = ipset add f2b-<name> <ip> timeout <ipsettime> -exist -actionprolong = %(actionban)s +# actionprolong = %(actionban)s # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the @@ -78,8 +78,16 @@ actionprolong = %(actionban)s # actionunban = ipset del f2b-<name> <ip> -exist -# Option: default-timeout +# Option: default-ipsettime # Notes: specifies default timeout in seconds (handled default ipset timeout only) -# Values: [ NUM ] Default: 600 +# Values: [ NUM ] Default: 0 (no timeout, managed by fail2ban by unban) +default-ipsettime = 0 -default-timeout = 600 +# Option: ipsettime +# Notes: specifies ticket timeout (handled ipset timeout only) +# Values: [ NUM ] Default: 0 (managed by fail2ban by unban) +ipsettime = 0 + +# expresion to caclulate timeout from bantime, example: +# banaction = %(known/banaction)s[ipsettime='<timeout-bantime>'] +timeout-bantime = $([ "<bantime>" -le 2147483 ] && echo "<bantime>" || echo 0) diff --git a/config/action.d/shorewall.conf b/config/action.d/shorewall.conf index dcef8829..83d08d99 100644 --- a/config/action.d/shorewall.conf +++ b/config/action.d/shorewall.conf @@ -9,7 +9,7 @@ # connections. So if the attempter goes on trying using the same connection # he could even log in. In order to get the same behavior of the iptable # action (so that the ban is immediate) the /etc/shorewall/shorewall.conf -# file should me modified with "BLACKLISTNEWONLY=No". Note that as of +# file should be modified with "BLACKLISTNEWONLY=No". Note that as of # Shorewall 4.5.13 BLACKLISTNEWONLY is deprecated; however the equivalent # of BLACKLISTNEWONLY=No can now be achieved by setting BLACKLIST="ALL". # diff --git a/config/action.d/smtp.py b/config/action.d/smtp.py index 9cdfe327..5c27d0ff 100644 --- a/config/action.d/smtp.py +++ b/config/action.d/smtp.py @@ -159,25 +159,25 @@ class SMTPAction(ActionBase): try: self._logSys.debug("Connected to SMTP '%s', response: %i: %s", self.host, *smtp.connect(self.host)) - if self.user and self.password: + if self.user and self.password: # pragma: no cover (ATM no tests covering that) smtp.login(self.user, self.password) failed_recipients = smtp.sendmail( self.fromaddr, self.toaddr.split(", "), msg.as_string()) - except smtplib.SMTPConnectError: + except smtplib.SMTPConnectError: # pragma: no cover self._logSys.error("Error connecting to host '%s'", self.host) raise - except smtplib.SMTPAuthenticationError: + except smtplib.SMTPAuthenticationError: # pragma: no cover self._logSys.error( "Failed to authenticate with host '%s' user '%s'", self.host, self.user) raise - except smtplib.SMTPException: + except smtplib.SMTPException: # pragma: no cover self._logSys.error( "Error sending mail to host '%s' from '%s' to '%s'", self.host, self.fromaddr, self.toaddr) raise else: - if failed_recipients: + if failed_recipients: # pragma: no cover self._logSys.warning( "Email to '%s' failed to following recipients: %r", self.toaddr, failed_recipients) @@ -186,7 +186,7 @@ class SMTPAction(ActionBase): try: self._logSys.debug("Disconnected from '%s', response %i: %s", self.host, *smtp.quit()) - except smtplib.SMTPServerDisconnected: + except smtplib.SMTPServerDisconnected: # pragma: no cover pass # Not connected def start(self): diff --git a/config/action.d/symbiosis-blacklist-allports.conf b/config/action.d/symbiosis-blacklist-allports.conf index 6fb7d0af..7208b293 100644 --- a/config/action.d/symbiosis-blacklist-allports.conf +++ b/config/action.d/symbiosis-blacklist-allports.conf @@ -5,7 +5,7 @@ [INCLUDES] -before = iptables-common.conf +before = iptables.conf [Definition] @@ -41,6 +41,11 @@ actionban = echo 'all' >| /etc/symbiosis/firewall/blacklist.d/<ip>.auto actionunban = rm -f /etc/symbiosis/firewall/blacklist.d/<ip>.auto <iptables> -D <chain> -s <ip> -j <blocktype> || : +# [TODO] Flushing is currently not implemented for symbiosis blacklist.d +# +actionflush = + + [Init] # Option: chain diff --git a/config/action.d/ufw.conf b/config/action.d/ufw.conf index d2f731f2..c9ff7f37 100644 --- a/config/action.d/ufw.conf +++ b/config/action.d/ufw.conf @@ -13,16 +13,45 @@ actionstop = actioncheck = -actionban = [ -n "<application>" ] && app="app <application>" - ufw insert <insertpos> <blocktype> from <ip> to <destination> $app +# ufw does "quickly process packets for which we already have a connection" in before.rules, +# therefore all related sockets should be closed +# actionban is using `ss` to do so, this only handles IPv4 and IPv6. -actionunban = [ -n "<application>" ] && app="app <application>" - ufw delete <blocktype> from <ip> to <destination> $app +actionban = if [ -n "<application>" ] && ufw app info "<application>" + then + ufw <add> <blocktype> from <ip> to <destination> app "<application>" comment "<comment>" + else + ufw <add> <blocktype> from <ip> to <destination> comment "<comment>" + fi + <kill> + +actionunban = if [ -n "<application>" ] && ufw app info "<application>" + then + ufw delete <blocktype> from <ip> to <destination> app "<application>" + else + ufw delete <blocktype> from <ip> to <destination> + fi + +# Option: kill-mode +# Notes.: can be set to ss or conntrack (may be extended later with other modes) to immediately drop all connections from banned IP, default empty (no kill) +# Example: banaction = ufw[kill-mode=ss] +kill-mode = + +# intern conditional parameter used to provide killing mode after ban: +_kill_ = +_kill_ss = ss -K dst "[<ip>]" +_kill_conntrack = conntrack -D -s "<ip>" + +# Option: kill +# Notes.: can be used to specify custom killing feature, by default depending on option kill-mode +# Examples: banaction = ufw[kill='ss -K "( sport = :http || sport = :https )" dst "[<ip>]"'] +# banaction = ufw[kill='cutter "<ip>"'] +kill = <_kill_<kill-mode>> [Init] -# Option: insertpos -# Notes.: The position number in the firewall list to insert the block rule -insertpos = 1 +# Option: add +# Notes.: can be set to "insert 1" to insert a rule at certain position (here 1): +add = prepend # Option: blocktype # Notes.: reject or deny @@ -36,6 +65,10 @@ destination = any # Notes.: application from sudo ufw app list application = +# Option: comment +# Notes.: comment for rule added by fail2ban +comment = by Fail2Ban after <failures> attempts against <name> + # DEV NOTES: # # Author: Guilhem Lettron diff --git a/config/action.d/xarf-login-attack.conf b/config/action.d/xarf-login-attack.conf index 2b135c43..f348b2c4 100644 --- a/config/action.d/xarf-login-attack.conf +++ b/config/action.d/xarf-login-attack.conf @@ -41,7 +41,12 @@ actionstop = actioncheck = -actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(dig +short -t txt -q $4.$3.$2.$1.abuse-contacts.abusix.org); IFS=${oifs} +actionban = oifs=${IFS}; + RESOLVER_ADDR="%(addr_resolver)s" + if [ "<debug>" -gt 0 ]; then echo "try to resolve $RESOLVER_ADDR"; fi + ADDRESSES=$(dig +short -t txt -q $RESOLVER_ADDR | tr -d '"') + IFS=,; ADDRESSES=$(echo $ADDRESSES) + IFS=${oifs} IP=<ip> FROM=<sender> SERVICE=<service> @@ -51,26 +56,37 @@ actionban = oifs=${IFS}; IFS=.;SEP_IP=( <ip> ); set -- ${SEP_IP}; ADDRESSES=$(di PORT=<port> DATE=`LC_ALL=C date --date=@<time> +"%%a, %%d %%h %%Y %%T %%z"` if [ ! -z "$ADDRESSES" ]; then + oifs=${IFS}; IFS=,; ADDRESSES=$(echo $ADDRESSES) + IFS=${oifs} (printf -- %%b "<header>\n<message>\n<report>\n\n"; date '+Note: Local timezone is %%z (%%Z)'; - printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> ${ADDRESSES//,/\" \"} + printf -- %%b "\n<ipmatches>\n\n<footer>") | <mailcmd> <mailargs> $ADDRESSES fi actionunban = -[Init] +# Server as resolver used in dig command +# +addr_resolver = <ip-rev>abuse-contacts.abusix.org + +# Option: boundary +# Notes: This can be overwritten to be safe for possible predictions +boundary = bfbb0f920793ac03cb8634bde14d8a1e + +_boundary = Abuse<time>-<boundary> + # Option: header # Notes: This is really a fixed value -header = Subject: abuse report about $IP - $DATE\nAuto-Submitted: auto-generated\nX-XARF: PLAIN\nContent-Transfer-Encoding: 7bit\nContent-Type: multipart/mixed; charset=utf8;\n boundary=Abuse-bfbb0f920793ac03cb8634bde14d8a1e;\n\n--Abuse-bfbb0f920793ac03cb8634bde14d8a1e\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf-8;\n +header = Subject: abuse report about $IP - $DATE\nAuto-Submitted: auto-generated\nX-XARF: PLAIN\nContent-Transfer-Encoding: 7bit\nContent-Type: multipart/mixed; charset=utf8;\n boundary=%(_boundary)s;\n\n--%(_boundary)s\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf-8;\n # Option: footer # Notes: This is really a fixed value and needs to match the report and header # mime delimiters -footer = \n\n--Abuse-bfbb0f920793ac03cb8634bde14d8a1e-- +footer = \n\n--%(_boundary)s-- # Option: report # Notes: Intended to be fixed -report = --Abuse-bfbb0f920793ac03cb8634bde14d8a1e\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf-8; name=\"report.txt\";\n\n---\nReported-From: $FROM\nCategory: abuse\nReport-ID: $REPORTID\nReport-Type: login-attack\nService: $SERVICE\nVersion: 0.2\nUser-Agent: Fail2ban v0.9\nDate: $DATE\nSource-Type: ip-address\nSource: $IP\nPort: $PORT\nSchema-URL: http://www.x-arf.org/schema/abuse_login-attack_0.1.2.json\nAttachment: text/plain\nOccurances: $FAILURES\nTLP: $TLP\n\n\n--Abuse-bfbb0f920793ac03cb8634bde14d8a1e\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf8; name=\"logfile.log\"; +report = --%(_boundary)s\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf-8; name=\"report.txt\";\n\n---\nReported-From: $FROM\nCategory: abuse\nReport-ID: $REPORTID\nReport-Type: login-attack\nService: $SERVICE\nVersion: 0.2\nUser-Agent: Fail2ban v0.9\nDate: $DATE\nSource-Type: ip-address\nSource: $IP\nPort: $PORT\nSchema-URL: http://www.x-arf.org/schema/abuse_login-attack_0.1.2.json\nAttachment: text/plain\nOccurances: $FAILURES\nTLP: $TLP\n\n\n--%(_boundary)s\nMIME-Version: 1.0\nContent-Transfer-Encoding: 7bit\nContent-Type: text/plain; charset=utf8; name=\"logfile.log\"; # Option: Message # Notes: This can be modified by the users diff --git a/config/fail2ban.conf b/config/fail2ban.conf index 52e47187..fd6baebf 100644 --- a/config/fail2ban.conf +++ b/config/fail2ban.conf @@ -5,11 +5,11 @@ # Changes: in most of the cases you should not modify this # file, but provide customizations in fail2ban.local file, e.g.: # -# [Definition] +# [DEFAULT] # loglevel = DEBUG # -[Definition] +[DEFAULT] # Option: loglevel # Notes.: Set the log level output. @@ -19,18 +19,18 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # loglevel = INFO # Option: logtarget -# Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. +# Notes.: Set the log target. This could be a file, SYSTEMD-JOURNAL, SYSLOG, STDERR or STDOUT. # Only one log target can be specified. # If you change logtarget from the default value and you are # using logrotate -- also adjust or disable rotation in the # corresponding configuration file # (e.g. /etc/logrotate.d/fail2ban on Debian systems) -# Values: [ STDOUT | STDERR | SYSLOG | SYSOUT | FILE ] Default: STDERR +# Values: [ STDOUT | STDERR | SYSLOG | SYSOUT | SYSTEMD-JOURNAL | FILE ] Default: STDERR # logtarget = /var/log/fail2ban.log @@ -55,6 +55,12 @@ socket = /var/run/fail2ban/fail2ban.sock # pidfile = /var/run/fail2ban/fail2ban.pid +# Option: allowipv6 +# Notes.: Allows IPv6 interface: +# Default: auto +# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto +#allowipv6 = auto + # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. # A value of ":memory:" means database is only stored in memory @@ -67,3 +73,20 @@ dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) dbpurgeage = 1d + +# Options: dbmaxmatches +# Notes.: Number of matches stored in database per ticket (resolvable via +# tags <ipmatches>/<ipjailmatches> in actions) +# Values: [ INT ] Default: 10 +dbmaxmatches = 10 + +[Definition] + + +[Thread] + +# Options: stacksize +# Notes.: Specifies the stack size (in KiB) to be used for subsequently created threads, +# and must be 0 or a positive integer value of at least 32. +# Values: [ SIZE ] Default: 0 (use platform or configured default) +#stacksize = 0 diff --git a/config/filter.d/apache-auth.conf b/config/filter.d/apache-auth.conf index 91c89b26..40f6d6e3 100644 --- a/config/filter.d/apache-auth.conf +++ b/config/filter.d/apache-auth.conf @@ -9,6 +9,16 @@ before = apache-common.conf [Definition] +# Mode for filter: normal (default) and aggressive (allows DDoS & brute force detection of mod_evasive) +mode = normal + +# ignore messages of mod_evasive module: +apache-pref-ign-normal = (?!evasive) +# allow "denied by server configuration" from all modules: +apache-pref-ign-aggressive = +# mode related ignore prefix for common _apache_error_client substitution: +apache-pref-ignore = <apache-pref-ign-<mode>> + prefregex = ^%(_apache_error_client)s (?:AH\d+: )?<F-CONTENT>.+</F-CONTENT>$ # auth_type = ((?:Digest|Basic): )? diff --git a/config/filter.d/apache-common.conf b/config/filter.d/apache-common.conf index 3eec83d0..6577fe7d 100644 --- a/config/filter.d/apache-common.conf +++ b/config/filter.d/apache-common.conf @@ -27,7 +27,9 @@ _daemon = (?:apache\d*|httpd(?:/\w+)?) apache-prefix = <apache-prefix-<logging>> -_apache_error_client = <apache-prefix>\[(:?error|\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\] +apache-pref-ignore = + +_apache_error_client = <apache-prefix>\[(:?error|<apache-pref-ignore>\S+:\S+)\]( \[pid \d+(:\S+ \d+)?\])? \[client <HOST>(:\d{1,5})?\] datepattern = {^LN-BEG} diff --git a/config/filter.d/apache-fakegooglebot.conf b/config/filter.d/apache-fakegooglebot.conf index 729410ad..ee23656a 100644 --- a/config/filter.d/apache-fakegooglebot.conf +++ b/config/filter.d/apache-fakegooglebot.conf @@ -2,11 +2,11 @@ [Definition] -failregex = ^<HOST> .*Googlebot.*$ +failregex = ^\s*<HOST> \S+ \S+(?: \S+)?\s+\S+ "[A-Z]+ /\S* [^"]*" \d+ \d+ \"[^"]*\" "[^"]*\bGooglebot/[^"]*" ignoreregex = -datepattern = ^[^\[]*\[({DATE}) +datepattern = ^[^\[]*(\[{DATE}\s*\]) {^LN-BEG} # DEV Notes: diff --git a/config/filter.d/apache-modsecurity.conf b/config/filter.d/apache-modsecurity.conf index e296227a..f7600acf 100644 --- a/config/filter.d/apache-modsecurity.conf +++ b/config/filter.d/apache-modsecurity.conf @@ -10,7 +10,7 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s ModSecurity:\s+(?:\[(?:\w+ \"[^\"]*\"|[^\]]*)\]\s*)*Access denied with code [45]\d\d +failregex = ^%(_apache_error_client)s(?: \[client [^\]]+\])? ModSecurity:\s+(?:\[(?:\w+ \"[^\"]*\"|[^\]]*)\]\s*)*Access denied with code [45]\d\d ignoreregex = diff --git a/config/filter.d/apache-noscript.conf b/config/filter.d/apache-noscript.conf index 1963d1dd..dd9452a9 100644 --- a/config/filter.d/apache-noscript.conf +++ b/config/filter.d/apache-noscript.conf @@ -17,13 +17,13 @@ before = apache-common.conf [Definition] -script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl) +script = /\S*(?:php(?:[45]|[.-]cgi)?|\.asp|\.exe|\.pl|\bcgi-bin/) -prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$ +prefregex = ^%(_apache_error_client)s (?:AH0(?:01(?:28|30)|1(?:264|071)|2811): )?(?:(?:[Ff]ile|script|[Gg]ot) )<F-CONTENT>.+</F-CONTENT>$ failregex = ^(?:does not exist|not found or unable to stat): <script>\b ^'<script>\S*' not found or unable to stat - ^error '[Pp]rimary script unknown\\n' + ^error '[Pp]rimary script unknown(?:\\n)?' ignoreregex = diff --git a/config/filter.d/apache-overflows.conf b/config/filter.d/apache-overflows.conf index 02a2ef20..0f54da11 100644 --- a/config/filter.d/apache-overflows.conf +++ b/config/filter.d/apache-overflows.conf @@ -8,7 +8,7 @@ before = apache-common.conf [Definition] -failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) +failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) ignoreregex = diff --git a/config/filter.d/asterisk.conf b/config/filter.d/asterisk.conf index 6f7ae5d5..e15d7bfe 100644 --- a/config/filter.d/asterisk.conf +++ b/config/filter.d/asterisk.conf @@ -21,12 +21,12 @@ log_prefix= (?:NOTICE|SECURITY|WARNING)%(__pid_re)s:?(?:\[C-[\da-f]*\])?:? [^:]+ prefregex = ^%(__prefix_line)s%(log_prefix)s <F-CONTENT>.+</F-CONTENT>$ failregex = ^Registration from '[^']*' failed for '<HOST>(:\d+)?' - (?:Wrong password|Username/auth name mismatch|No matching peer found|Not a local domain|Device does not match ACL|Peer is not supposed to register|ACL error \(permit/deny\)|Not a local domain)$ - ^Call from '[^']*' \(<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context + ^Call from '[^']*' \((?:(?:TCP|UDP):)?<HOST>:\d+\) to extension '[^']*' rejected because extension not found in context ^(?:Host )?<HOST> (?:failed (?:to authenticate\b|MD5 authentication\b)|tried to authenticate with nonexistent user\b) ^No registration for peer '[^']*' \(from <HOST>\)$ ^hacking attempt detected '<HOST>'$ - ^SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/(UDP|TCP|WS)/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$ - ^"Rejecting unknown SIP connection from <HOST>"$ + ^SecurityEvent="(?:FailedACL|InvalidAccountID|ChallengeResponseFailed|InvalidPassword)"(?:(?:,(?!RemoteAddress=)\w+="[^"]*")*|.*?),RemoteAddress="IPV[46]/[^/"]+/<HOST>/\d+"(?:,(?!RemoteAddress=)\w+="[^"]*")*$ + ^"Rejecting unknown SIP connection from <HOST>(?::\d+)?"$ ^Request (?:'[^']*' )?from '(?:[^']*|.*?)' failed for '<HOST>(?::\d+)?'\s\(callid: [^\)]*\) - (?:No matching endpoint found|Not match Endpoint(?: Contact)? ACL|(?:Failed|Error) to authenticate)\s*$ # FreePBX (todo: make optional in v.0.10): @@ -44,3 +44,12 @@ datepattern = {^LN-BEG} # First regex: channels/chan_sip.c # # main/logger.c:ast_log_vsyslog - "in {functionname}:" only occurs in syslog + +journalmatch = _SYSTEMD_UNIT=asterisk.service + + +[lt_journal] + +# asterisk can log timestamp if logs into systemd-journal (optional part matching this timestamp, gh-2383): +__extra_timestamp = (?:\[[^\]]+\]\s+)? +__prefix_line = %(known/__prefix_line)s%(__extra_timestamp)s diff --git a/config/filter.d/bitwarden.conf b/config/filter.d/bitwarden.conf new file mode 100644 index 00000000..b0651c8e --- /dev/null +++ b/config/filter.d/bitwarden.conf @@ -0,0 +1,13 @@ +# Fail2Ban filter for Bitwarden +# Detecting failed login attempts +# Logged in bwdata/logs/identity/Identity/log.txt + +[INCLUDES] +before = common.conf + +[Definition] +_daemon = Bitwarden-Identity +failregex = ^%(__prefix_line)s\s*\[(?:W(?:RN|arning)|Bit\.Core\.[^\]]+)\]\s+Failed login attempt(?:, 2FA invalid)?\. <ADDR>$ + +# DEV Notes: +# __prefix_line can result to an empty string, so it can support syslog and non-syslog at once. diff --git a/config/filter.d/centreon.conf b/config/filter.d/centreon.conf new file mode 100644 index 00000000..fd3c8482 --- /dev/null +++ b/config/filter.d/centreon.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for Centreon Web +# Detecting unauthorized access to the Centreon Web portal +# typically logged in /var/log/centreon/login.log + +[Init] +datepattern = ^%%Y-%%m-%%d %%H:%%M:%%S + +[Definition] +failregex = ^(?:\|-?\d+){3}\|\[[^\]]*\] \[<HOST>\] Authentication failed for '<F-USER>[^']+</F-USER>' diff --git a/config/filter.d/common.conf b/config/filter.d/common.conf index a8cba188..e6b3c641 100644 --- a/config/filter.d/common.conf +++ b/config/filter.d/common.conf @@ -10,6 +10,9 @@ after = common.local [DEFAULT] +# Type of log-file resp. log-format (file, short, journal, rfc5424): +logtype = file + # Daemon definition is to be specialized (if needed) in .conf file _daemon = \S* @@ -22,7 +25,7 @@ __pid_re = (?:\[\d+\]) # Daemon name (with optional source_file:line or whatever) # EXAMPLES: pam_rhosts_auth, [sshd], pop(pam_unix) -__daemon_re = [\[\(]?%(_daemon)s(?:\(\S+\))?[\]\)]?:? +__daemon_re = [\[\(]?<_daemon>(?:\(\S+\))?[\]\)]?:? # extra daemon info # EXAMPLE: [ID 800047 auth.info] @@ -30,11 +33,11 @@ __daemon_extra_re = \[ID \d+ \S+\] # Combinations of daemon name and PID # EXAMPLES: sshd[31607], pop(pam_unix)[4920] -__daemon_combs_re = (?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:?) +__daemon_combs_re = (?:<__pid_re>?:\s+<__daemon_re>|<__daemon_re><__pid_re>?:?) # Some messages have a kernel prefix with a timestamp # EXAMPLES: kernel: [769570.846956] -__kernel_prefix = kernel: \[ *\d+\.\d+\] +__kernel_prefix = kernel:\s?\[ *\d+\.\d+\]:? __hostname = \S+ @@ -55,13 +58,32 @@ __date_ambit = (?:\[\]) # [bsdverbose]? [hostname] [vserver tag] daemon_id spaces # # This can be optional (for instance if we match named native log files) -__prefix_line = %(__date_ambit)s?\s*(?:%(__bsd_syslog_verbose)s\s+)?(?:%(__hostname)s\s+)?(?:%(__kernel_prefix)s\s+)?(?:%(__vserver)s\s+)?(?:%(__daemon_combs_re)s\s+)?(?:%(__daemon_extra_re)s\s+)? +__prefix_line = <lt_<logtype>/__prefix_line> # PAM authentication mechanism check for failures, e.g.: pam_unix, pam_sss, # pam_ldap __pam_auth = pam_unix # standardly all formats using prefix have line-begin anchored date: +datepattern = <lt_<logtype>/datepattern> + +[lt_file] +# Common line prefixes for logtype "file": +__prefix_line = <__date_ambit>?\s*(?:<__bsd_syslog_verbose>\s+)?(?:<__hostname>\s+)?(?:<__kernel_prefix>\s+)?(?:<__vserver>\s+)?(?:<__daemon_combs_re>\s+)?(?:<__daemon_extra_re>\s+)? datepattern = {^LN-BEG} -# Author: Yaroslav Halchenko +[lt_short] +# Common (short) line prefix for logtype "journal" (corresponds output of formatJournalEntry): +__prefix_line = \s*(?:<__hostname>\s+)?(?:<_daemon><__pid_re>?:?\s+)?(?:<__kernel_prefix>\s+)? +datepattern = %(lt_file/datepattern)s +[lt_journal] +__prefix_line = %(lt_short/__prefix_line)s +datepattern = %(lt_short/datepattern)s + +[lt_rfc5424] +# RFC 5424 log-format, see gh-2309: +#__prefix_line = \s*<__hostname> <__daemon_re> \d+ \S+ \S+\s+ +__prefix_line = \s*<__hostname> <__daemon_re> \d+ \S+ (?:[^\[\]\s]+|(?:\[(?:[^\]"]*|"[^"]*")*\])+)\s+ +datepattern = ^<\d+>\d+\s+{DATE} + +# Author: Yaroslav Halchenko, Sergey G. Brester (aka sebres) diff --git a/config/filter.d/courier-auth.conf b/config/filter.d/courier-auth.conf index 1ac33736..d5ba9c50 100644 --- a/config/filter.d/courier-auth.conf +++ b/config/filter.d/courier-auth.conf @@ -11,7 +11,7 @@ before = common.conf _daemon = (?:courier)?(?:imapd?|pop3d?)(?:login)?(?:-ssl)? -failregex = ^%(__prefix_line)sLOGIN FAILED, (?:user|method)=.*, ip=\[<HOST>\]$ +failregex = ^%(__prefix_line)sLOGIN FAILED, (?:(?!ip=)(?:user=<F-USER>[^,]*</F-USER>|\w+=[^,]*), )*ip=\[<HOST>\] ignoreregex = diff --git a/config/filter.d/courier-smtp.conf b/config/filter.d/courier-smtp.conf index 888753c4..4b2b8d87 100644 --- a/config/filter.d/courier-smtp.conf +++ b/config/filter.d/courier-smtp.conf @@ -12,7 +12,7 @@ before = common.conf _daemon = courieresmtpd -prefregex = ^%(__prefix_line)serror,relay=<HOST>,<F-CONTENT>.+</F-CONTENT>$ +prefregex = ^%(__prefix_line)serror,relay=<HOST>,(?:port=\d+,)?<F-CONTENT>.+</F-CONTENT>$ failregex = ^[^:]*: 550 User (<.*> )?unknown\.?$ ^msg="535 Authentication failed\.",cmd:( AUTH \S+)?( [0-9a-zA-Z\+/=]+)?(?: \S+)$ diff --git a/config/filter.d/dante.conf b/config/filter.d/dante.conf new file mode 100644 index 00000000..e3f6f7b2 --- /dev/null +++ b/config/filter.d/dante.conf @@ -0,0 +1,16 @@ +# Fail2Ban filter for dante +# +# Make sure you have "log: error" set in your "client pass" directive +# + +[INCLUDES] +before = common.conf + +[Definition] +_daemon = danted + +failregex = ^%(__prefix_line)sinfo: block\(1\): tcp/accept \]: <HOST>\.\d+ [\d.]+: error after reading \d+ bytes? in \d+ seconds?: (?:could not access |system password authentication failed for )user "<F-USER>[^"]+</F-USER>" + +[Init] +journalmatch = _SYSTEMD_UNIT=danted.service + diff --git a/config/filter.d/domino-smtp.conf b/config/filter.d/domino-smtp.conf index cdc17736..638cd7c5 100644 --- a/config/filter.d/domino-smtp.conf +++ b/config/filter.d/domino-smtp.conf @@ -35,9 +35,12 @@ # 08-09-2014 06:14:27 smtp: postmaster [1.2.3.4] authentication failure using internet password # 08-09-2014 06:14:27 SMTP Server: Authentication failed for user postmaster ; connecting host 1.2.3.4 -__prefix = (?:\[[^\]]+\])?\s+ -failregex = ^%(__prefix)sSMTP Server: Authentication failed for user .*? \; connecting host <HOST>$ - ^%(__prefix)ssmtp: (?:[^\[]+ )*\[<HOST>\] authentication failure using internet password\s*$ +__prefix = (?:\[[^\]]+\])?\s* +__opt_data = (?::|\s+\[[^\]]+\]) +failregex = ^%(__prefix)sSMTP Server%(__opt_data)s Authentication failed for user .*? \; connecting host \[?<HOST>\]?$ + ^%(__prefix)ssmtp: (?:[^\[]+ )*\[?<HOST>\]? authentication failure using internet password\s*$ + ^%(__prefix)sSMTP Server%(__opt_data)s Connection from \[?<HOST>\]? rejected for policy reasons\. + # Option: ignoreregex # Notes.: regex to ignore. If this regex matches, the line is ignored. # Values: TEXT diff --git a/config/filter.d/dovecot.conf b/config/filter.d/dovecot.conf index f0481e06..dc3ebbcd 100644 --- a/config/filter.d/dovecot.conf +++ b/config/filter.d/dovecot.conf @@ -7,18 +7,21 @@ before = common.conf [Definition] -_auth_worker = (?:dovecot: )?auth(?:-worker)? _daemon = (?:dovecot(?:-auth)?|auth) -prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap)-login: )?(?:Info: )?<F-CONTENT>.+</F-CONTENT>$ +_auth_worker = (?:dovecot: )?auth(?:-worker)? +_auth_worker_info = (?:conn \w+:auth(?:-worker)? \([^\)]+\): auth(?:-worker)?<\d+>: )? +_bypass_reject_reason = (?:: (?:\w+\([^\):]*\) \w+|[^\(]+))* + +prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s<F-CONTENT>.+</F-CONTENT>$ failregex = ^authentication failure; logname=<F-ALT_USER1>\S*</F-ALT_USER1> uid=\S* euid=\S* tty=dovecot ruser=<F-USER>\S*</F-USER> rhost=<HOST>(?:\s+user=<F-ALT_USER>\S*</F-ALT_USER>)?\s*$ - ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$ - ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$ - ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch)\s*$ + ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<<F-USER>[^>]*</F-USER>>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$ + ^pam\(\S+,<HOST>(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$ + ^[a-z\-]{3,15}\(\S*,<HOST>(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch) <mdre-<mode>> -mdre-aggressive = ^(?:Aborted login|Disconnected)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$ +mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)%(_bypass_reject_reason)s \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=<HOST>(?:[^>]*(?:, session=<\S+>)?)\s*$ mdre-normal = diff --git a/config/filter.d/drupal-auth.conf b/config/filter.d/drupal-auth.conf index b60abe3e..2404cc6d 100644 --- a/config/filter.d/drupal-auth.conf +++ b/config/filter.d/drupal-auth.conf @@ -14,7 +14,7 @@ before = common.conf [Definition] -failregex = ^%(__prefix_line)s(https?:\/\/)([\da-z\.-]+)\.([a-z\.]{2,6})(\/[\w\.-]+)*\|\d{10}\|user\|<HOST>\|.+\|.+\|\d\|.*\|Login attempt failed for .+\.$ +failregex = ^%(__prefix_line)s(?:https?:\/\/)[^|]+\|[^|]+\|[^|]+\|<ADDR>\|(?:[^|]*\|)*Login attempt failed (?:for|from) <F-USER>[^|]+</F-USER>\.$ ignoreregex = diff --git a/config/filter.d/exim-common.conf b/config/filter.d/exim-common.conf index b3b25750..36644e94 100644 --- a/config/filter.d/exim-common.conf +++ b/config/filter.d/exim-common.conf @@ -12,7 +12,7 @@ after = exim-common.local host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)? host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s -pid = (?: \[\d+\])? +pid = (?: \[\d+\]| \w+ exim\[\d+\]:)? # DEV Notes: # From exim source code: ./src/receive.c:add_host_info_for_log diff --git a/config/filter.d/gitlab.conf b/config/filter.d/gitlab.conf new file mode 100644 index 00000000..0c614ae5 --- /dev/null +++ b/config/filter.d/gitlab.conf @@ -0,0 +1,6 @@ +# Fail2Ban filter for Gitlab +# Detecting unauthorized access to the Gitlab Web portal +# typically logged in /var/log/gitlab/gitlab-rails/application.log + +[Definition] +failregex = ^: Failed Login: username=<F-USER>.+</F-USER> ip=<HOST>$ diff --git a/config/filter.d/grafana.conf b/config/filter.d/grafana.conf new file mode 100644 index 00000000..e7f0f420 --- /dev/null +++ b/config/filter.d/grafana.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for Grafana +# Detecting unauthorized access +# Typically logged in /var/log/grafana/grafana.log + +[Init] +datepattern = ^t=%%Y-%%m-%%dT%%H:%%M:%%S%%z + +[Definition] +failregex = ^(?: lvl=err?or)? msg="Invalid username or password"(?: uname=(?:"<F-ALT_USER>[^"]+</F-ALT_USER>"|<F-USER>\S+</F-USER>)| error="<F-ERROR>[^"]+</F-ERROR>"| \S+=(?:\S*|"[^"]+"))* remote_addr=<ADDR>$ diff --git a/config/filter.d/guacamole.conf b/config/filter.d/guacamole.conf index 09b4e7b0..bc6dbea9 100644 --- a/config/filter.d/guacamole.conf +++ b/config/filter.d/guacamole.conf @@ -5,21 +5,47 @@ [Definition] -# Option: failregex -# Notes.: regex to match the password failures messages in the logfile. -# Values: TEXT -# -failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" failed\.$ +logging = catalina +failregex = <L_<logging>/failregex> +maxlines = <L_<logging>/maxlines> +datepattern = <L_<logging>/datepattern> -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# -ignoreregex = +[L_catalina] + +failregex = ^.*\nWARNING: Authentication attempt from <HOST> for user "[^"]*" failed\.$ -# "maxlines" is number of log lines to buffer for multi-line regex searches maxlines = 2 datepattern = ^%%b %%d, %%ExY %%I:%%M:%%S %%p ^WARNING:()** - {^LN-BEG}
\ No newline at end of file + {^LN-BEG} + +[L_webapp] + +failregex = ^ \[\S+\] WARN \S+ - Authentication attempt from <HOST> for user "<F-USER>[^"]+</F-USER>" failed. + +maxlines = 1 + +datepattern = ^%%H:%%M:%%S.%%f + +# DEV Notes: +# +# failregex is based on the default pattern given in Guacamole documentation : +# https://guacamole.apache.org/doc/gug/configuring-guacamole.html#webapp-logging +# +# The following logback.xml Guacamole configuration file can then be used accordingly : +# <configuration> +# <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> +# <file>/var/log/guacamole.log</file> +# <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> +# <fileNamePattern>/var/log/guacamole.%d.log.gz</fileNamePattern> +# <maxHistory>32</maxHistory> +# </rollingPolicy> +# <encoder> +# <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern> +# </encoder> +# </appender> +# <root level="info"> +# <appender-ref ref="FILE" /> +# </root> +# </configuration> diff --git a/config/filter.d/ignorecommands/apache-fakegooglebot b/config/filter.d/ignorecommands/apache-fakegooglebot index 3c443251..8351efa2 100755 --- a/config/filter.d/ignorecommands/apache-fakegooglebot +++ b/config/filter.d/ignorecommands/apache-fakegooglebot @@ -6,32 +6,43 @@ # import sys from fail2ban.server.ipdns import DNSUtils, IPAddr +from threading import Thread def process_args(argv): - if len(argv) != 2: - raise ValueError("Please provide a single IP as an argument. Got: %s\n" - % (argv[1:])) + if len(argv) - 1 not in (1, 2): + raise ValueError("Usage %s ip ?timeout?. Got: %s\n" + % (argv[0], argv[1:])) ip = argv[1] if not IPAddr(ip).isValid: raise ValueError("Argument must be a single valid IP. Got: %s\n" % ip) - return ip + return argv[1:] google_ips = None -def is_googlebot(ip): +def is_googlebot(ip, timeout=55): import re - host = DNSUtils.ipToName(ip) - if not host or not re.match('.*\.google(bot)?\.com$', host): + timeout = float(timeout or 0) + if timeout: + def ipToNameTO(host, ip, timeout): + host[0] = DNSUtils.ipToName(ip) + host = [None] + th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start() + th.join(timeout) + host = host[0] + else: + host = DNSUtils.ipToName(ip) + + if not host or not re.match(r'.*\.google(bot)?\.com$', host): return False host_ips = DNSUtils.dnsToIp(host) return (ip in host_ips) if __name__ == '__main__': # pragma: no cover try: - ret = is_googlebot(process_args(sys.argv)) + ret = is_googlebot(*process_args(sys.argv)) except ValueError as e: sys.stderr.write(str(e)) sys.exit(2) diff --git a/config/filter.d/lighttpd-auth.conf b/config/filter.d/lighttpd-auth.conf index a68f4f4d..dcf19d3e 100644 --- a/config/filter.d/lighttpd-auth.conf +++ b/config/filter.d/lighttpd-auth.conf @@ -3,7 +3,7 @@ [Definition] -failregex = ^: \((?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match .* username: .*|digest: auth failed for .*: wrong password|get_password failed), IP: <HOST>\s*$ +failregex = ^\s*(?:: )?\(?(?:http|mod)_auth\.c\.\d+\) (?:password doesn\'t match for (?:\S+|.*?) username:\s+<F-USER>(?:\S+|.*?)</F-USER>\s*|digest: auth failed(?: for\s+<F-ALT_USER>(?:\S+|.*?)</F-ALT_USER>\s*)?: (?:wrong password|uri mismatch \([^\)]*\))|get_password failed),? IP: <HOST>\s*$ ignoreregex = diff --git a/config/filter.d/monit.conf b/config/filter.d/monit.conf index b652a1f4..fdaee9c3 100644 --- a/config/filter.d/monit.conf +++ b/config/filter.d/monit.conf @@ -8,13 +8,17 @@ # common.local before = common.conf +# [DEFAULT] +# logtype = short + [Definition] _daemon = monit +_prefix = Warning|HttpRequest + # Regexp for previous (accessing monit httpd) and new (access denied) versions -failregex = ^\[\s*\]\s*error\s*:\s*Warning:\s+Client '<HOST>' supplied (?:unknown user '[^']+'|wrong password for user '[^']*') accessing monit httpd$ - ^%(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '[^']+'|wrong password for user '[^']*'|empty password)$ +failregex = ^%(__prefix_line)s(?:error\s*:\s+)?(?:%(_prefix)s):\s+(?:access denied\s+--\s+)?[Cc]lient '?<HOST>'?(?:\s+supplied|\s*:)\s+(?:unknown user '<F-ALT_USER>[^']+</F-ALT_USER>'|wrong password for user '<F-USER>[^']*</F-USER>'|empty password) # Ignore login with empty user (first connect, no user specified) # ignoreregex = %(__prefix_line)s\w+: access denied -- client <HOST>: (?:unknown user '') diff --git a/config/filter.d/monitorix.conf b/config/filter.d/monitorix.conf new file mode 100644 index 00000000..ff69f1bc --- /dev/null +++ b/config/filter.d/monitorix.conf @@ -0,0 +1,25 @@ +# Fail2Ban filter for Monitorix (HTTP built-in server) +# + +[INCLUDES] + +before = common.conf + +[Definition] + +_daemon = monitorix-httpd + +# Option: failregex +# Notes.: regex to match the password failures messages in the logfile. The +# host must be matched by a group named "host". The tag "<HOST>" can +# be used for standard IP/hostname matching and is only an alias for +# (?:::f{4,6}:)?(?P<host>\S+) +# Values: TEXT +# +failregex = ^(?:\s+-)?\s*(?:NOTEXIST|AUTHERR|NOTALLOWED) - <ADDR>\b + +# Option: ignoreregex +# Notes.: regex to ignore. If this regex matches, the line is ignored. +# Values: TEXT +# +ignoreregex = diff --git a/config/filter.d/mssql-auth.conf b/config/filter.d/mssql-auth.conf new file mode 100644 index 00000000..65bbd917 --- /dev/null +++ b/config/filter.d/mssql-auth.conf @@ -0,0 +1,15 @@ +# Fail2Ban filter for failed MSSQL Server authentication attempts + +[Definition] + +failregex = ^\s*Logon\s+Login failed for user '<F-USER>(?:[^']*|.*)</F-USER>'\. [^'\[]+\[CLIENT: <ADDR>\]$ + + +# DEV Notes: +# Tested with SQL Server 2019 on Ubuntu 18.04 +# +# Example: +# 2020-02-24 14:48:55.12 Logon Login failed for user 'root'. Reason: Could not find a login matching the name provided. [CLIENT: 127.0.0.1] +# +# Author: Rüdiger Olschewsky +#
\ No newline at end of file diff --git a/config/filter.d/mysqld-auth.conf b/config/filter.d/mysqld-auth.conf index 31bd2056..930c9b5a 100644 --- a/config/filter.d/mysqld-auth.conf +++ b/config/filter.d/mysqld-auth.conf @@ -3,7 +3,7 @@ # # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld]: # log-error=/var/log/mysqld.log -# log-warning = 2 +# log-warnings = 2 # # If using mysql syslog [mysql_safe] has syslog in /etc/my.cnf @@ -17,7 +17,7 @@ before = common.conf _daemon = mysqld -failregex = ^%(__prefix_line)s(?:\d+ |\d{6} \s?\d{1,2}:\d{2}:\d{2} )?\[\w+\] Access denied for user '[^']+'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ +failregex = ^%(__prefix_line)s(?:(?:\d{6}|\d{4}-\d{2}-\d{2})[ T]\s?\d{1,2}:\d{2}:\d{2} )?(?:\d+ )?\[\w+\] (?:\[[^\]]+\] )*Access denied for user '<F-USER>[^']+</F-USER>'@'<HOST>' (to database '[^']*'|\(using password: (YES|NO)\))*\s*$ ignoreregex = diff --git a/config/filter.d/named-refused.conf b/config/filter.d/named-refused.conf index 2e14d442..798f66e6 100644 --- a/config/filter.d/named-refused.conf +++ b/config/filter.d/named-refused.conf @@ -22,7 +22,7 @@ [Definition] # Daemon name -_daemon=named +_daemon=named(?:-\w+)? # Shortcuts for easier comprehension of the failregex @@ -30,15 +30,18 @@ __pid_re=(?:\[\d+\]) __daemon_re=\(?%(_daemon)s(?:\(\S+\))?\)?:? __daemon_combs_re=(?:%(__pid_re)s?:\s+%(__daemon_re)s|%(__daemon_re)s%(__pid_re)s?:) +_category = (?!error|info)[\w-]+ +_category_re = (?:%(_category)s: )? + # hostname daemon_id spaces # this can be optional (for instance if we match named native log files) -__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? +__line_prefix=\s*(?:\S+ %(__daemon_combs_re)s\s+)?%(_category_re)s -prefregex = ^%(__line_prefix)s( error:)?\s*client <HOST>#\S+( \([\S.]+\))?: <F-CONTENT>.+</F-CONTENT>$ +prefregex = ^%(__line_prefix)s(?:(?:error|info):\s*)?client(?: @\S*)? <HOST>#\S+(?: \([\S.]+\))?: <F-CONTENT>.+</F-CONTENT>\s(?:denied|\(NOTAUTH\))\s*$ -failregex = ^(view (internal|external): )?query(?: \(cache\))? '.*' denied\s*$ - ^zone transfer '\S+/AXFR/\w+' denied\s*$ - ^bad zone transfer request: '\S+/IN': non-authoritative zone \(NOTAUTH\)\s*$ +failregex = ^(?:view (?:internal|external): )?query(?: \(cache\))? + ^zone transfer + ^bad zone transfer request: '\S+/IN': non-authoritative zone ignoreregex = diff --git a/config/filter.d/nginx-bad-request.conf b/config/filter.d/nginx-bad-request.conf new file mode 100644 index 00000000..12c14ab7 --- /dev/null +++ b/config/filter.d/nginx-bad-request.conf @@ -0,0 +1,16 @@ +# Fail2Ban filter to match bad requests to nginx +# + +[Definition] + +# The request often doesn't contain a method, only some encoded garbage +# This will also match requests that are entirely empty +failregex = ^<HOST> - \S+ \[\] "[^"]*" 400 + +datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,]%%f)?(?:\s*%%z)? + ^[^\[]*\[({DATE}) + {^LN-BEG} + +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + +# Author: Jan Przybylak diff --git a/config/filter.d/nginx-botsearch.conf b/config/filter.d/nginx-botsearch.conf index 0be895b2..2bd23072 100644 --- a/config/filter.d/nginx-botsearch.conf +++ b/config/filter.d/nginx-botsearch.conf @@ -17,7 +17,9 @@ datepattern = {^LN-BEG}%%ExY(?P<_sep>[-/.])%%m(?P=_sep)%%d[T ]%%H:%%M:%%S(?:[.,] ^[^\[]*\[({DATE}) {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV Notes: # Based on apache-botsearch filter # -# Author: Frantisek Sumsal
\ No newline at end of file +# Author: Frantisek Sumsal diff --git a/config/filter.d/nginx-http-auth.conf b/config/filter.d/nginx-http-auth.conf index 93341cd2..71806e85 100644 --- a/config/filter.d/nginx-http-auth.conf +++ b/config/filter.d/nginx-http-auth.conf @@ -3,15 +3,32 @@ [Definition] +mode = normal -failregex = ^ \[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$ +mdre-auth = ^\s*\[error\] \d+#\d+: \*\d+ user "(?:[^"]+|.*?)":? (?:password mismatch|was not found in "[^\"]*"), client: <HOST>, server: \S*, request: "\S+ \S+ HTTP/\d+\.\d+", host: "\S+"(?:, referrer: "\S+")?\s*$ +mdre-fallback = ^\s*\[crit\] \d+#\d+: \*\d+ SSL_do_handshake\(\) failed \(SSL: error:\S+(?: \S+){1,3} too (?:long|short)\)[^,]*, client: <HOST> + +mdre-normal = %(mdre-auth)s +mdre-aggressive = %(mdre-auth)s + %(mdre-fallback)s + +failregex = <mdre-<mode>> ignoreregex = datepattern = {^LN-BEG} +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + # DEV NOTES: +# mdre-auth: # Based on samples in https://github.com/fail2ban/fail2ban/pull/43/files # Extensive search of all nginx auth failures not done yet. # # Author: Daniel Black + +# mdre-fallback: +# Ban people checking for TLS_FALLBACK_SCSV repeatedly +# https://stackoverflow.com/questions/28010492/nginx-critical-error-with-ssl-handshaking/28010608#28010608 +# Author: Stephan Orlowsky + diff --git a/config/filter.d/nginx-limit-req.conf b/config/filter.d/nginx-limit-req.conf index e23548ab..2f45e831 100644 --- a/config/filter.d/nginx-limit-req.conf +++ b/config/filter.d/nginx-limit-req.conf @@ -44,3 +44,6 @@ failregex = ^\s*\[[a-z]+\] \d+#\d+: \*\d+ limiting requests, excess: [\d\.]+ by ignoreregex = datepattern = {^LN-BEG} + +journalmatch = _SYSTEMD_UNIT=nginx.service + _COMM=nginx + diff --git a/config/filter.d/nsd.conf b/config/filter.d/nsd.conf index bfd99544..0589c16c 100644 --- a/config/filter.d/nsd.conf +++ b/config/filter.d/nsd.conf @@ -22,10 +22,10 @@ _daemon = nsd # (?:::f{4,6}:)?(?P<host>[\w\-.^_]+) # Values: TEXT -failregex = ^%(__prefix_line)sinfo: ratelimit block .* query <HOST> TYPE255$ - ^%(__prefix_line)sinfo: .* <HOST> refused, no acl matches\.$ +failregex = ^%(__prefix_line)sinfo: ratelimit block .* query <ADDR> TYPE255$ + ^%(__prefix_line)sinfo: .* from(?: client)? <ADDR> refused, no acl matches\.?$ ignoreregex = datepattern = {^LN-BEG}Epoch - {^LN-BEG}
\ No newline at end of file + {^LN-BEG} diff --git a/config/filter.d/phpmyadmin-syslog.conf b/config/filter.d/phpmyadmin-syslog.conf index 5b0862bb..4378bedb 100644 --- a/config/filter.d/phpmyadmin-syslog.conf +++ b/config/filter.d/phpmyadmin-syslog.conf @@ -1,4 +1,4 @@ -# Fail2Ban fitler for the phpMyAdmin-syslog +# Fail2Ban filter for the phpMyAdmin-syslog # [INCLUDES] diff --git a/config/filter.d/postfix.conf b/config/filter.d/postfix.conf index d1505e32..b374f472 100644 --- a/config/filter.d/postfix.conf +++ b/config/filter.d/postfix.conf @@ -12,16 +12,15 @@ before = common.conf _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])? _port = (?::\d+)? +_pref = [A-Z]{4} prefregex = ^%(__prefix_line)s<mdpr-<mode>> <F-CONTENT>.+</F-CONTENT>$ -mdpr-normal = (?:NOQUEUE: reject:|improper command pipelining after \S+) -mdre-normal=^RCPT from [^[]*\[<HOST>\]%(_port)s: 55[04] 5\.7\.1\s - ^RCPT from [^[]*\[<HOST>\]%(_port)s: 45[04] 4\.7\.1 (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b) - ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.7\.1 (<[^>]*>)?: Helo command rejected: Host not found\b - ^EHLO from [^[]*\[<HOST>\]%(_port)s: 504 5\.5\.2 (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b - ^VRFY from [^[]*\[<HOST>\]%(_port)s: 550 5\.1\.1\s - ^RCPT from [^[]*\[<HOST>\]%(_port)s: 450 4\.1\.8 (<[^>]*>)?: Sender address rejected: Domain not found\b +# Extended RE for normal mode to match reject by unknown users or undeliverable address, can be set to empty to avoid this: +exre-user = |[Uu](?:ser unknown|ndeliverable address) + +mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+) +mdre-normal=^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match%(exre-user)s)\b ^from [^[]*\[<HOST>\]%(_port)s:? mdpr-auth = warning: @@ -31,13 +30,15 @@ mdre-auth2= ^[^[]*\[<HOST>\]%(_port)s: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5 # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only: mdpr-rbl = %(mdpr-normal)s -mdre-rbl = ^RCPT from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b +mdre-rbl = ^%(_pref)s from [^[]*\[<HOST>\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b # Mode "rbl" currently included in mode "normal" (within 1st rule) mdpr-more = %(mdpr-normal)s mdre-more = %(mdre-normal)s -mdpr-ddos = lost connection after(?! DATA) [A-Z]+ +# Includes some of the log messages described in +# <http://www.postfix.org/POSTSCREEN_README.html>. +mdpr-ddos = (?:lost connection after(?! DATA) [A-Z]+|disconnect(?= from \S+(?: \S+=\d+)* auth=0/(?:[1-9]|\d\d+))|(?:PREGREET \d+|HANGUP) after \S+|COMMAND (?:TIME|COUNT|LENGTH) LIMIT) mdre-ddos = ^from [^[]*\[<HOST>\]%(_port)s:? mdpr-extra = (?:%(mdpr-auth)s|%(mdpr-normal)s) @@ -48,6 +49,8 @@ mdpr-aggressive = (?:%(mdpr-auth)s|%(mdpr-normal)s|%(mdpr-ddos)s) mdre-aggressive = %(mdre-auth2)s %(mdre-normal)s +mdpr-errors = too many errors after \S+ +mdre-errors = ^from [^[]*\[<HOST>\]%(_port)s$ failregex = <mdre-<mode>> @@ -56,10 +59,17 @@ failregex = <mdre-<mode>> # Usage example (for jail.local): # [postfix] # mode = aggressive +# # # or another jail (rewrite filter parameters of jail): # [postfix-rbl] # filter = postfix[mode=rbl] # +# # jail to match "too many errors", related postconf `smtpd_hard_error_limit`: +# # (normally included in other modes (normal, more, extra, aggressive), but this jail'd allow to ban on the first message) +# [postfix-many-errors] +# filter = postfix[mode=errors] +# maxretry = 1 +# mode = more ignoreregex = diff --git a/config/filter.d/proftpd.conf b/config/filter.d/proftpd.conf index a7bd2837..71f2ba73 100644 --- a/config/filter.d/proftpd.conf +++ b/config/filter.d/proftpd.conf @@ -1,4 +1,4 @@ -# Fail2Ban fitler for the Proftpd FTP daemon +# Fail2Ban filter for the Proftpd FTP daemon # # Set "UseReverseDNS off" in proftpd.conf to avoid the need for DNS. # See: http://www.proftpd.org/docs/howto/DNS.html @@ -14,16 +14,15 @@ before = common.conf _daemon = proftpd -__suffix_failed_login = (User not authorized for login|No such user found|Incorrect password|Password expired|Account disabled|Invalid shell: '\S+'|User in \S+|Limit (access|configuration) denies login|Not a UserAlias|maximum login length exceeded).? +__suffix_failed_login = ([uU]ser not authorized for login|[nN]o such user found|[iI]ncorrect password|[pP]assword expired|[aA]ccount disabled|[iI]nvalid shell: '\S+'|[uU]ser in \S+|[lL]imit (access|configuration) denies login|[nN]ot a UserAlias|[mM]aximum login length exceeded) -prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum).+</F-CONTENT>$ +prefregex = ^%(__prefix_line)s%(__hostname)s \(\S+\[<HOST>\]\)[: -]+ <F-CONTENT>(?:USER|SECURITY|Maximum) .+</F-CONTENT>$ -failregex = ^USER .*: no such user found from \S+ \[\S+\] to \S+:\S+ *$ - ^USER .* \(Login failed\): %(__suffix_failed_login)s\s*$ - ^SECURITY VIOLATION: .* login attempted\. *$ - ^Maximum login attempts \(\d+\) exceeded *$ +failregex = ^USER <F-USER>\S+|.*?</F-USER>(?: \(Login failed\))?: %(__suffix_failed_login)s + ^SECURITY VIOLATION: <F-USER>\S+|.*?</F-USER> login attempted + ^Maximum login attempts \(\d+\) exceeded ignoreregex = diff --git a/config/filter.d/scanlogd.conf b/config/filter.d/scanlogd.conf new file mode 100644 index 00000000..d3fe78b0 --- /dev/null +++ b/config/filter.d/scanlogd.conf @@ -0,0 +1,17 @@ +# Fail2Ban filter for port scans detected by scanlogd + +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = common.conf + +[Definition] + +_daemon = scanlogd + +failregex = ^%(__prefix_line)s<ADDR>(?::<F-PORT/>)? to \S+ ports\b + +ignoreregex = + +# Author: Mike Gabriel <mike.gabriel@das-netzwerkteam.de> diff --git a/config/filter.d/selinux-common.conf b/config/filter.d/selinux-common.conf index b3e0ae4f..dc9616d2 100644 --- a/config/filter.d/selinux-common.conf +++ b/config/filter.d/selinux-common.conf @@ -14,7 +14,7 @@ [Definition] -failregex = ^type=%(_type)s msg=audit\(:\d+\): (user )?pid=\d+ uid=%(_uid)s auid=%(_auid)s ses=\d+ subj=%(_subj)s msg='%(_msg)s'$ +failregex = ^type=%(_type)s msg=audit\(:\d+\): (?:user )?pid=\d+ uid=%(_uid)s auid=%(_auid)s ses=\d+ subj=%(_subj)s msg='%(_msg)s'(?:\x1D|$) ignoreregex = diff --git a/config/filter.d/selinux-ssh.conf b/config/filter.d/selinux-ssh.conf index 6955094f..0e38eb11 100644 --- a/config/filter.d/selinux-ssh.conf +++ b/config/filter.d/selinux-ssh.conf @@ -15,7 +15,9 @@ _subj = (?:unconfined_u|system_u):system_r:sshd_t:s0-s0:c0\.c1023 _exe =/usr/sbin/sshd _terminal = ssh -_msg = op=\S+ acct=(?P<_quote_acct>"?)\S+(?P=_quote_acct) exe="%(_exe)s" hostname=(\?|(\d+\.){3}\d+) addr=<HOST> terminal=%(_terminal)s res=failed +_anygrp = (?!acct=|exe=|addr=|terminal=|res=)\w+=(?:"[^"]+"|\S*) + +_msg = (?:%(_anygrp)s )*acct=(?:"<F-USER>[^"]+</F-USER>"|<F-ALT_USER>\S+</F-ALT_USER>) exe="%(_exe)s" (?:%(_anygrp)s )*addr=<ADDR> terminal=%(_terminal)s res=failed # DEV Notes: # diff --git a/config/filter.d/sendmail-auth.conf b/config/filter.d/sendmail-auth.conf index a370eea2..3fa3c701 100644 --- a/config/filter.d/sendmail-auth.conf +++ b/config/filter.d/sendmail-auth.conf @@ -8,9 +8,14 @@ before = common.conf [Definition] _daemon = (?:sendmail|sm-(?:mta|acceptingconnections)) +# "\w{14,20}" will give support for IDs from 14 up to 20 characters long +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:IPv6:<IP6>|<IP4>) -failregex = ^%(__prefix_line)s\w{14}: (\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ +prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$ +failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ + ^AUTH failure \([^\)]+\):(?: [^:]+:)? (?:authentication failure|user not found): [^,]*, (?:user=<F-USER>(?:\S+|.*?)</F-USER>, )?relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff --git a/config/filter.d/sendmail-reject.conf b/config/filter.d/sendmail-reject.conf index 985eac8b..41035e5f 100644 --- a/config/filter.d/sendmail-reject.conf +++ b/config/filter.d/sendmail-reject.conf @@ -20,19 +20,21 @@ before = common.conf [Definition] _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? +addr = (?:(?:IPv6:)?<IP6>|<IP4>) -prefregex = ^<F-MLFID>%(__prefix_line)s(?:\w{14}: )?</F-MLFID><F-CONTENT>.+</F-CONTENT>$ +prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID><F-CONTENT>.+</F-CONTENT>$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P<dom>\S+), arg2=(?:IPv6:<IP6>|<IP4>), relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ - ^rejecting commands from (\S* )?\[(?:IPv6:<IP6>|<IP4>)\] due to pre-greeting traffic after \d+ seconds$ - ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<email><\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ + ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ + ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ - ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[(?:IPv6:<IP6>|<IP4>)\]$ + ^<F-NOFAIL>from=<[^@]+@[^>]+></F-NOFAIL>, size=\d+, class=\d+, nrcpts=\d+, bodytype=\w+, proto=E?SMTP, daemon=MTA, relay=\S+ \[%(addr)s\]$ mdre-normal = -mdre-extra = ^(?:\S+ )?\[(?:IPv6:<IP6>|<IP4>)\](?: \(may be forged\))? did not issue (?:[A-Z]{4}[/ ]?)+during connection to M(?:TA|SP)(?:-\w+)?$ +mdre-extra = ^(?:\S+ )?\[%(addr)s\](?: \(may be forged\))? did not issue \S+ during connection mdre-aggressive = %(mdre-extra)s @@ -48,7 +50,7 @@ mode = normal ignoreregex = -journalmatch = _SYSTEMD_UNIT=sendmail.service +journalmatch = SYSLOG_IDENTIFIER=sm-mta + _SYSTEMD_UNIT=sendmail.service # DEV NOTES: # diff --git a/config/filter.d/softethervpn.conf b/config/filter.d/softethervpn.conf new file mode 100644 index 00000000..f7e7c0c3 --- /dev/null +++ b/config/filter.d/softethervpn.conf @@ -0,0 +1,9 @@ +# Fail2Ban filter for SoftEtherVPN +# Detecting unauthorized access to SoftEtherVPN +# typically logged in /usr/local/vpnserver/security_log/*/sec.log, or in syslog, depending on configuration + +[INCLUDES] +before = common.conf + +[Definition] +failregex = ^%(__prefix_line)s(?:(?:\([\d\-]+ [\d:.]+\) )?<SECURITY_LOG>: )?Connection "[^"]+": User authentication failed. The user name that has been provided was "<F-USER>(?:[^"]+|.+)</F-USER>", from <ADDR>\.$ diff --git a/config/filter.d/sogo-auth.conf b/config/filter.d/sogo-auth.conf index 48221dc0..4155f89e 100644 --- a/config/filter.d/sogo-auth.conf +++ b/config/filter.d/sogo-auth.conf @@ -4,7 +4,7 @@ [Definition] -failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>' for user '.*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$ +failregex = ^ sogod \[\d+\]: SOGoRootPage Login from '<HOST>(?:,[^']*)?' for user '[^']*' might not have worked( - password policy: \d* grace: -?\d* expire: -?\d* bound: -?\d*)?\s*$ ignoreregex = "^<ADDR>" diff --git a/config/filter.d/sshd.conf b/config/filter.d/sshd.conf index 60efead7..d5d189b0 100644 --- a/config/filter.d/sshd.conf +++ b/config/filter.d/sshd.conf @@ -25,7 +25,7 @@ __pref = (?:(?:error|fatal): (?:PAM: )?)? __suff = (?: (?:port \d+|on \S+|\[preauth\])){0,3}\s* __on_port_opt = (?: (?:port \d+|on \S+)){0,2} # close by authenticating user: -__authng_user = (?: authenticating user <F-USER>\S+|.+?</F-USER>)? +__authng_user = (?: (?:invalid|authenticating) user <F-USER>\S+|.*?</F-USER>)? # for all possible (also future) forms of "no matching (cipher|mac|MAC|compression method|key exchange method|host key type) found", # see ssherr.c for all possible SSH_ERR_..._ALG_MATCH errors. @@ -40,45 +40,68 @@ prefregex = ^<F-MLFID>%(__prefix_line)s</F-MLFID>%(__pref)s<F-CONTENT>.+</F-CONT cmnfailre = ^[aA]uthentication (?:failure|error|failed) for <F-USER>.*</F-USER> from <HOST>( via \S+)?%(__suff)s$ ^User not known to the underlying authentication module for <F-USER>.*</F-USER> from <HOST>%(__suff)s$ - ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^Failed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + <cmnfailre-failed-pub-<publickey>> + ^Failed <cmnfailed> for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^<F-USER>ROOT</F-USER> LOGIN REFUSED FROM <HOST> ^[iI](?:llegal|nvalid) user <F-USER>.*?</F-USER> from <HOST>%(__suff)s$ - ^User <F-USER>.+</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$ - ^User <F-USER>.+</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$ - ^User <F-USER>.+</F-USER> from <HOST> not allowed because not in any group%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not listed in AllowUsers%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because listed in DenyUsers%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because not in any group%(__suff)s$ ^refused connect from \S+ \(<HOST>\) ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*3: .*: Auth fail%(__suff)s$ - ^User <F-USER>.+</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$ - ^User <F-USER>.+</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because a group is listed in DenyGroups%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> from <HOST> not allowed because none of user's groups are listed in AllowGroups%(__suff)s$ ^<F-NOFAIL>%(__pam_auth)s\(sshd:auth\):\s+authentication failure;</F-NOFAIL>(?:\s+(?:(?:logname|e?uid|tty)=\S*)){0,4}\s+ruser=<F-ALT_USER>\S*</F-ALT_USER>\s+rhost=<HOST>(?:\s+user=<F-USER>\S*</F-USER>)?%(__suff)s$ - ^(error: )?maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ - ^User <F-USER>.+</F-USER> not allowed because account is locked%(__suff)s - ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>: Too many authentication failures(?: for <F-USER>.+?</F-USER>)?%(__suff)s$ + ^maximum authentication attempts exceeded for <F-USER>.*</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?%(__suff)s$ + ^User <F-USER>\S+|.*?</F-USER> not allowed because account is locked%(__suff)s + ^<F-MLFFORGET>Disconnecting</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s:\s*Change of username or service not allowed:\s*.*\[preauth\]\s*$ + ^Disconnecting: Too many authentication failures(?: for <F-USER>\S+|.*?</F-USER>)?%(__suff)s$ ^<F-NOFAIL>Received <F-MLFFORGET>disconnect</F-MLFFORGET></F-NOFAIL> from <HOST>%(__on_port_opt)s:\s*11: - ^<F-NOFAIL>Connection <F-MLFFORGET>closed</F-MLFFORGET></F-NOFAIL> by%(__authng_user)s <HOST><mdrp-<mode>-suff-onclosed> - ^<F-MLFFORGET><F-NOFAIL>Accepted \w+</F-NOFAIL></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$) + <mdre-<mode>-other> + ^<F-MLFFORGET><F-MLFGAINED>Accepted \w+</F-MLFGAINED></F-MLFFORGET> for <F-USER>\S+</F-USER> from <HOST>(?:\s|$) + +cmnfailed-any = \S+ +cmnfailed-ignore = \b(?!publickey)\S+ +cmnfailed-invalid = <cmnfailed-ignore> +cmnfailed-nofail = (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+) +cmnfailed = <cmnfailed-<publickey>> mdre-normal = # used to differentiate "connection closed" with and without `[preauth]` (fail/nofail cases in ddos mode) -mdrp-normal-suff-onclosed = (?:%(__suff)s|\s*)$ +mdre-normal-other = ^<F-NOFAIL><F-MLFFORGET>(Connection (?:closed|reset)|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__suff)s|\s*)$ mdre-ddos = ^Did not receive identification string from <HOST> - ^Connection <F-MLFFORGET>reset</F-MLFFORGET> by <HOST> - ^Connection <F-MLFFORGET>closed</F-MLFFORGET> by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$ + ^kex_exchange_identification: (?:read: )?(?:[Cc]lient sent invalid protocol identifier|[Cc]onnection (?:closed by remote host|reset by peer)) + ^Bad protocol version identification '.*' from <HOST> ^<F-NOFAIL>SSH: Server;Ltype:</F-NOFAIL> (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+: ^Read from socket failed: Connection <F-MLFFORGET>reset</F-MLFFORGET> by peer -mdrp-ddos-suff-onclosed = %(__on_port_opt)s\s*$ + ^banner exchange: Connection from <HOST><__on_port_opt>: invalid format +# same as mdre-normal-other, but as failure (without <F-NOFAIL> with [preauth] and with <F-NOFAIL> on no preauth phase as helper to identify address): +mdre-ddos-other = ^<F-MLFFORGET>(Connection (?:closed|reset)|Disconnected)</F-MLFFORGET> (?:by|from)%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$ + ^<F-NOFAIL><F-MLFFORGET>(Connection (?:closed|reset)|Disconnected)</F-MLFFORGET></F-NOFAIL> (?:by|from)%(__authng_user)s <HOST>(?:%(__on_port_opt)s|\s*)$ -mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^Received <F-MLFFORGET>disconnect</F-MLFFORGET> from <HOST>%(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^Unable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found. ^Unable to negotiate a <__alg_match> ^no matching <__alg_match> found: -mdrp-extra-suff-onclosed = %(mdrp-normal-suff-onclosed)s +# part of mdre-ddos-other, but user name is supplied (invalid/authenticating) on [preauth] phase only: +mdre-extra-other = ^<F-MLFFORGET>Disconnected</F-MLFFORGET>(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+|.*?</F-USER> <HOST>%(__on_port_opt)s \[preauth\]\s*$ mdre-aggressive = %(mdre-ddos)s %(mdre-extra)s -mdrp-aggressive-suff-onclosed = %(mdrp-ddos-suff-onclosed)s +# mdre-extra-other is fully included within mdre-ddos-other: +mdre-aggressive-other = %(mdre-ddos-other)s + +# Parameter "publickey": nofail (default), invalid, any, ignore +publickey = nofail +# consider failed publickey for invalid users only: +cmnfailre-failed-pub-invalid = ^Failed publickey for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) +# consider failed publickey for valid users too (don't need RE, see cmnfailed): +cmnfailre-failed-pub-any = +# same as invalid, but consider failed publickey for valid users too, just as no failure (helper to get IP and user-name only, see cmnfailed): +cmnfailre-failed-pub-nofail = <cmnfailre-failed-pub-invalid> +# don't consider failed publickey as failures (don't need RE, see cmnfailed): +cmnfailre-failed-pub-ignore = cfooterre = ^<F-NOFAIL>Connection from</F-NOFAIL> <HOST> @@ -104,8 +127,6 @@ maxlines = 1 journalmatch = _SYSTEMD_UNIT=sshd.service + _COMM=sshd -datepattern = {^LN-BEG} - # DEV Notes: # # "Failed \S+ for .*? from <HOST>..." failregex uses non-greedy catch-all because diff --git a/config/filter.d/traefik-auth.conf b/config/filter.d/traefik-auth.conf new file mode 100644 index 00000000..8022fee1 --- /dev/null +++ b/config/filter.d/traefik-auth.conf @@ -0,0 +1,76 @@ +# Fail2ban filter configuration for traefik :: auth +# used to ban hosts, that were failed through traefik +# +# Author: CrazyMax +# +# To use 'traefik-auth' filter you have to configure your Traefik instance to write +# the access logs as describe in https://docs.traefik.io/configuration/logs/#access-logs +# into a log file on host and specifiy users for Basic Authentication +# https://docs.traefik.io/configuration/entrypoints/#basic-authentication +# +# Example: +# +# version: "3.2" +# +# services: +# traefik: +# image: traefik:latest +# command: +# - "--loglevel=INFO" +# - "--accesslog=true" +# - "--accessLog.filePath=/var/log/access.log" +# # - "--accessLog.filters.statusCodes=400-499" +# - "--defaultentrypoints=http,https" +# - "--entryPoints=Name:http Address::80" +# - "--entryPoints=Name:https Address::443 TLS" +# - "--docker.domain=example.com" +# - "--docker.watch=true" +# - "--docker.exposedbydefault=false" +# - "--api=true" +# - "--api.dashboard=true" +# ports: +# - target: 80 +# published: 80 +# protocol: tcp +# mode: host +# - target: 443 +# published: 443 +# protocol: tcp +# mode: host +# labels: +# - "traefik.enable=true" +# - "traefik.port=8080" +# - "traefik.backend=traefik" +# - "traefik.frontend.rule=Host:traefik.example.com" +# - "traefik.frontend.auth.basic.users=test:$$apr1$$H6uskkkW$$IgXLP6ewTrSuBkTrqE8wj/" +# volumes: +# - "/var/log/traefik:/var/log" +# - "/var/run/docker.sock:/var/run/docker.sock" +# restart: always +# + +[Definition] + +# Parameter "method" can be used to specifiy request method +req-method = \S+ +# Usage example (for jail.local): +# filter = traefik-auth[req-method="GET|POST|HEAD"] + +failregex = ^<HOST> \- <usrre-<mode>> \[\] \"(?:<req-method>) [^\"]+\" 401\b + +ignoreregex = + +# Parameter "mode": normal (default), ddos or aggressive +# Usage example (for jail.local): +# [traefik-auth] +# mode = aggressive +# # or another jail (rewrite filter parameters of jail): +# [traefik-auth-ddos] +# filter = traefik-auth[mode=ddos] +# +mode = normal + +# part of failregex matches user name (must be available in normal mode, must be empty in ddos mode, and both for aggressive mode): +usrre-normal = (?!- )<F-USER>\S+</F-USER> +usrre-ddos = - +usrre-aggressive = <F-USER>\S+</F-USER>
\ No newline at end of file diff --git a/config/filter.d/znc-adminlog.conf b/config/filter.d/znc-adminlog.conf new file mode 100644 index 00000000..8faa25e3 --- /dev/null +++ b/config/filter.d/znc-adminlog.conf @@ -0,0 +1,34 @@ +# Fail2Ban filter for ZNC (requires adminlog module) +# +# to use this module, enable the adminlog module from within ZNC and point +# logpath to its logfile (e.g. /var/lib/znc/moddata/adminlog/znc.log). + +[DEFAULT] + +logtype = file + +[Definition] + +_daemon = znc + +# Prefix for different logtype (file, journal): +# +__prefix_file = (?:\[\]\s+)? +__prefix_short = (?:\S+\s+%(_daemon)s\[\d+\]:)\s+ +__prefix_journal = %(__prefix_short)s + +__prefix_line = <__prefix_<logtype>> + +failregex = ^%(__prefix_line)s\[[^]]+\] failed to login from <ADDR> + +ignoreregex = + +journalmatch = _SYSTEMD_UNIT=znc.service + _COMM=znc + +# DEV Notes: +# Log format is: [<DATE+TIME>] [<USERNAME>] <ACTION> from <ADDR> +# [2018-10-27 01:40:17] [girst] connected to ZNC from 1.2.3.4 +# [2018-10-27 01:40:21] [girst] disconnected from ZNC from 1.2.3.4 +# [2018-10-27 01:40:55] [girst] failed to login from 1.2.3.4 +# +# Author: Tobias Girstmair (//gir.st/) diff --git a/config/filter.d/zoneminder.conf b/config/filter.d/zoneminder.conf index cc82755a..8e8ed432 100644 --- a/config/filter.d/zoneminder.conf +++ b/config/filter.d/zoneminder.conf @@ -5,17 +5,23 @@ before = apache-common.conf [Definition] -# pattern: [Wed Apr 27 23:12:07.736196 2016] [:error] [pid 2460] [client 10.1.1.1:47296] WAR [Login denied for user "test"], referer: https://zoneminderurl/index.php -# +# patterns: [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ +# [Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user test details], referer: https://zm/ +# [Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "john"], referer: https://zm/ # # Option: failregex -# Notes.: regex to match the password failure messages in the logfile. +# Notes.: regex to match the login failure and non-existent user error messages in the logfile. + +prefregex = ^%(_apache_error_client)s (?:ERR|WAR) <F-CONTENT>\[(?:Login denied|Could not retrieve).*</F-CONTENT>$ -failregex = ^%(_apache_error_client)s WAR \[Login denied for user "[^"]*"\] +failregex = ^\[Login denied for user "<F-USER>[^"]*</F-USER>"\] + ^\[Could not retrieve user <F-USER>\S*</F-USER> ignoreregex = # Notes: -# Tested on Zoneminder 1.29.0 +# Tested on Zoneminder 1.29 and 1.35.21 +# +# Zoneminder versions > 1.3x use "ERR" and < 1.3x use "WAR" level logs, so i've kept both for compatibility reasons # # Author: John Marzella diff --git a/config/jail.conf b/config/jail.conf index a6f2ac5a..b2fb7ec0 100644 --- a/config/jail.conf +++ b/config/jail.conf @@ -52,7 +52,7 @@ before = paths-debian.conf # to prevent "clever" botnets calculate exact time IP can be unbanned again: #bantime.rndtime = -# "bantime.maxtime" is the max number of seconds using the ban time can reach (don't grows further) +# "bantime.maxtime" is the max number of seconds using the ban time can reach (doesn't grow further) #bantime.maxtime = # "bantime.factor" is a coefficient to calculate exponent growing of the formula or common multiplier, @@ -60,14 +60,14 @@ before = paths-debian.conf # grows by 1, 2, 4, 8, 16 ... #bantime.factor = 1 -# "bantime.formula" used by default to calculate next value of ban time, default value bellow, +# "bantime.formula" used by default to calculate next value of ban time, default value below, # the same ban time growing will be reached by multipliers 1, 2, 4, 8, 16, 32... #bantime.formula = ban.Time * (1<<(ban.Count if ban.Count<20 else 20)) * banFactor # # more aggressive example of formula has the same values only for factor "2.0 / 2.885385" : #bantime.formula = ban.Time * math.exp(float(ban.Count+1)*banFactor)/math.exp(1*banFactor) -# "bantime.multipliers" used to calculate next value of ban time instead of formula, coresponding +# "bantime.multipliers" used to calculate next value of ban time instead of formula, corresponding # previously ban count and given "bantime.factor" (for multipliers default is 1); # following example grows ban time by 1, 2, 4, 8, 16 ... and if last ban count greater as multipliers count, # always used last multiplier (64 in example), for factor '1' and original ban time 600 - 10.6 hours @@ -77,7 +77,7 @@ before = paths-debian.conf #bantime.multipliers = 1 5 30 60 300 720 1440 2880 # "bantime.overalljails" (if true) specifies the search of IP in the database will be executed -# cross over all jails, if false (dafault), only current jail of the ban IP will be searched +# cross over all jails, if false (default), only current jail of the ban IP will be searched #bantime.overalljails = false # -------------------- @@ -85,6 +85,7 @@ before = paths-debian.conf # "ignoreself" specifies whether the local resp. own IP addresses should be ignored # (default is true). Fail2ban will not ban a host which matches such addresses. #ignoreself = true + # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban # will not ban a host which matches an address in this list. Several addresses # can be defined using space (and/or comma) separator. @@ -106,6 +107,9 @@ findtime = 10m # "maxretry" is the number of failures before a host get banned. maxretry = 5 +# "maxmatches" is the number of matches stored in ticket (resolvable via tag <matches> in actions). +maxmatches = %(maxretry)s + # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling", "systemd" and "auto". # This option can be overridden in each jail as well. @@ -205,28 +209,37 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] +action_mwl = %(action_)s + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath=%(logpath)s, port="%(port)s"] +action_xarf = %(action_)s + xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] + +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +# action = %(action_)s +# apprise # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # @@ -236,21 +249,7 @@ action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] # in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in # corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] - -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. -# -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] # Report ban via abuseipdb.com. # @@ -347,7 +346,7 @@ maxretry = 2 port = http,https logpath = %(apache_access_log)s maxretry = 1 -ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot <ip> +ignorecommand = %(fail2ban_confpath)s/filter.d/ignorecommands/apache-fakegooglebot <ip> [apache-modsecurity] @@ -367,12 +366,15 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log +# To use more aggressive http-auth modes set filter parameter "mode" in jail.local: +# normal (default), aggressive (combines all), auth or fallback +# See "tests/files/logs/nginx-http-auth" or "filter.d/nginx-http-auth.conf" for usage example and details. [nginx-http-auth] - +# mode = normal port = http,https logpath = %(nginx_error_log)s @@ -388,13 +390,14 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_error_log)s -maxretry = 2 -[nginx-forbidden] +[nginx-bad-request] +port = http,https +logpath = %(nginx_access_log)s +[nginx-forbidden] port = http,https logpath = %(nginx_error_log)s -maxretry = 10 # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year @@ -479,11 +482,13 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login port = 2812 logpath = /var/log/monit + /var/log/monit.log [webmin-auth] @@ -744,8 +749,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -756,9 +761,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -766,16 +770,22 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 +# enable adminlog; it will log to a file inside znc's directory by default. +[znc-adminlog] + +port = 6667 +logpath = /var/lib/znc/moddata/adminlog/znc.log + + # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld] or # equivalent section: -# log-warning = 2 +# log-warnings = 2 # # for syslog (daemon facility) # [mysqld_safe] @@ -791,6 +801,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation @@ -846,11 +864,31 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + +[bitwarden] +port = http,https +logpath = /home/*/bwdata/logs/identity/Identity/log.txt + +[centreon] +port = http,https +logpath = /var/log/centreon/login.log # consider low maxretry and a long bantime # nobody except your own Nagios server should ever probe nrpe @@ -884,7 +922,8 @@ filter = apache-pass[knocking_url="%(knocking_url)s"] logpath = %(apache_access_log)s blocktype = RETURN returntype = DROP -action = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s] +action = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s, + actionstart_on_demand=false, actionrepair_on_unban=true] bantime = 1h maxretry = 1 findtime = 1 @@ -893,8 +932,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log @@ -930,3 +969,21 @@ backend = %(syslog_backend)s port = http,https logpath = %(apache_error_log)s +[traefik-auth] +# to use 'traefik-auth' filter you have to configure your Traefik instance, +# see `filter.d/traefik-auth.conf` for details and service example. +port = http,https +logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s + +[monitorix] +port = 8080 +logpath = /var/log/monitorix-httpd + +[dante] +port = 1080 +logpath = %(syslog_daemon)s + diff --git a/config/paths-common.conf b/config/paths-common.conf index 7383cafe..4f6a5f71 100644 --- a/config/paths-common.conf +++ b/config/paths-common.conf @@ -91,6 +91,3 @@ mysql_log = %(syslog_daemon)s mysql_backend = %(default_backend)s roundcube_errors_log = /var/log/roundcube/errors - -# Directory with ignorecommand scripts -ignorecommands_dir = /etc/fail2ban/filter.d/ignorecommands diff --git a/config/paths-debian.conf b/config/paths-debian.conf index e096f972..1f5ea37d 100644 --- a/config/paths-debian.conf +++ b/config/paths-debian.conf @@ -26,3 +26,5 @@ exim_main_log = /var/log/exim4/mainlog # was in debian squeezy but not in wheezy # /etc/proftpd/proftpd.conf (SystemLog) proftpd_log = /var/log/proftpd/proftpd.log + +roundcube_errors_log = /var/log/roundcube/errors.log diff --git a/fail2ban/__init__.py b/fail2ban/__init__.py index 317f53e7..61789a45 100644 --- a/fail2ban/__init__.py +++ b/fail2ban/__init__.py @@ -79,3 +79,10 @@ logging.handlers.SysLogHandler.priority_map['NOTICE'] = 'notice' from time import strptime # strptime thread safety hack-around - http://bugs.python.org/issue7980 strptime("2012", "%Y") + +# short names for pure numeric log-level ("Level 25" could be truncated by short formats): +def _init(): + for i in range(50): + if logging.getLevelName(i).startswith('Level'): + logging.addLevelName(i, '#%02d-Lev.' % i) +_init() diff --git a/fail2ban/client/actionreader.py b/fail2ban/client/actionreader.py index 3ed8204c..88b0aca1 100644 --- a/fail2ban/client/actionreader.py +++ b/fail2ban/client/actionreader.py @@ -38,26 +38,32 @@ class ActionReader(DefinitionInitConfigReader): _configOpts = { "actionstart": ["string", None], - "actionstart_on_demand": ["string", None], + "actionstart_on_demand": ["bool", None], "actionstop": ["string", None], "actionflush": ["string", None], "actionreload": ["string", None], "actioncheck": ["string", None], "actionrepair": ["string", None], + "actionrepair_on_unban": ["bool", None], "actionban": ["string", None], "actionprolong": ["string", None], + "actionreban": ["string", None], "actionunban": ["string", None], - "norestored": ["string", None], + "norestored": ["bool", None], } def __init__(self, file_, jailName, initOpts, **kwargs): + # always supply jail name as name parameter if not specified in options: + n = initOpts.get("name") + if n is None: + initOpts["name"] = n = jailName actname = initOpts.get("actname") if actname is None: actname = file_ + # ensure we've unique action name per jail: + if n != jailName: + actname += n[len(jailName):] if n.startswith(jailName) else '-' + n initOpts["actname"] = actname - # always supply jail name as name parameter if not specified in options: - if initOpts.get("name") is None: - initOpts["name"] = jailName self._name = actname DefinitionInitConfigReader.__init__( self, file_, jailName, initOpts, **kwargs) @@ -78,11 +84,6 @@ class ActionReader(DefinitionInitConfigReader): def convert(self): opts = self.getCombined( ignore=CommandAction._escapedTags | set(('timeout', 'bantime'))) - # type-convert only after combined (otherwise boolean converting prevents substitution): - for o in ('norestored', 'actionstart_on_demand'): - if opts.get(o): - opts[o] = self._convert_to_boolean(opts[o]) - # stream-convert: head = ["set", self._jailName] stream = list() diff --git a/fail2ban/client/beautifier.py b/fail2ban/client/beautifier.py index 4d9e549f..97cd38b2 100644 --- a/fail2ban/client/beautifier.py +++ b/fail2ban/client/beautifier.py @@ -180,6 +180,12 @@ class Beautifier: msg = "The jail %s action %s has the following " \ "methods:\n" % (inC[1], inC[3]) msg += ", ".join(response) + elif inC[2] == "banip" and inC[0] == "get": + if isinstance(response, list): + sep = " " if len(inC) <= 3 else inC[3] + if sep == "--with-time": + sep = "\n" + msg = sep.join(response) except Exception: logSys.warning("Beautifier error. Please report the error") logSys.error("Beautify %r with %r failed", response, self.__inputCmd, diff --git a/fail2ban/client/configparserinc.py b/fail2ban/client/configparserinc.py index 70dfd91b..cc4ada0a 100644 --- a/fail2ban/client/configparserinc.py +++ b/fail2ban/client/configparserinc.py @@ -29,7 +29,7 @@ import re import sys from ..helpers import getLogger -if sys.version_info >= (3,2): +if sys.version_info >= (3,): # pragma: 2.x no cover # SafeConfigParser deprecated from Python 3.2 (renamed to ConfigParser) from configparser import ConfigParser as SafeConfigParser, BasicInterpolation, \ @@ -61,7 +61,7 @@ if sys.version_info >= (3,2): return super(BasicInterpolationWithName, self)._interpolate_some( parser, option, accum, rest, section, map, *args, **kwargs) -else: # pragma: no cover +else: # pragma: 3.x no cover from ConfigParser import SafeConfigParser, \ InterpolationMissingOptionError, NoOptionError, NoSectionError @@ -73,6 +73,17 @@ else: # pragma: no cover return self._cp_interpolate_some(option, accum, rest, section, map, *args, **kwargs) SafeConfigParser._interpolate_some = _interpolate_some +def _expandConfFilesWithLocal(filenames): + """Expands config files with local extension. + """ + newFilenames = [] + for filename in filenames: + newFilenames.append(filename) + localname = os.path.splitext(filename)[0] + '.local' + if localname not in filenames and os.path.isfile(localname): + newFilenames.append(localname) + return newFilenames + # Gets the instance of the logger. logSys = getLogger(__name__) logLevel = 7 @@ -245,6 +256,7 @@ after = 1.conf def _getIncludes(self, filenames, seen=[]): if not isinstance(filenames, list): filenames = [ filenames ] + filenames = _expandConfFilesWithLocal(filenames) # retrieve or cache include paths: if self._cfg_share: # cache/share include list: @@ -360,7 +372,8 @@ after = 1.conf s2 = alls.get(n) if isinstance(s2, dict): # save previous known values, for possible using in local interpolations later: - self.merge_section('KNOWN/'+n, s2, '') + self.merge_section('KNOWN/'+n, + dict(filter(lambda i: i[0] in s, s2.iteritems())), '') # merge section s2.update(s) else: diff --git a/fail2ban/client/configreader.py b/fail2ban/client/configreader.py index b03daca9..c7f965ce 100644 --- a/fail2ban/client/configreader.py +++ b/fail2ban/client/configreader.py @@ -34,6 +34,30 @@ from ..helpers import getLogger, _as_bool, _merge_dicts, substituteRecursiveTags # Gets the instance of the logger. logSys = getLogger(__name__) +CONVERTER = { + "bool": _as_bool, + "int": int, +} +def _OptionsTemplateGen(options): + """Iterator over the options template with default options. + + Each options entry is composed of an array or tuple with: + [[type, name, ?default?], ...] + Or it is a dict: + {name: [type, default], ...} + """ + if isinstance(options, (list,tuple)): + for optname in options: + if len(optname) > 2: + opttype, optname, optvalue = optname + else: + (opttype, optname), optvalue = optname, None + yield opttype, optname, optvalue + else: + for optname in options: + opttype, optvalue = options[optname] + yield opttype, optname, optvalue + class ConfigReader(): """Generic config reader class. @@ -120,6 +144,13 @@ class ConfigReader(): except AttributeError: return False + def has_option(self, sec, opt, withDefault=True): + return self._cfg.has_option(sec, opt) if withDefault \ + else opt in self._cfg._sections.get(sec, {}) + + def merge_defaults(self, d): + self._cfg.get_defaults().update(d) + def merge_section(self, section, *args, **kwargs): try: return self._cfg.merge_section(section, *args, **kwargs) @@ -221,29 +252,22 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # Or it is a dict: # {name: [type, default], ...} - def getOptions(self, sec, options, pOptions=None, shouldExist=False): + def getOptions(self, sec, options, pOptions=None, shouldExist=False, convert=True): values = dict() if pOptions is None: pOptions = {} # Get only specified options: - for optname in options: - if isinstance(options, (list,tuple)): - if len(optname) > 2: - opttype, optname, optvalue = optname - else: - (opttype, optname), optvalue = optname, None - else: - opttype, optvalue = options[optname] + for opttype, optname, optvalue in _OptionsTemplateGen(options): if optname in pOptions: continue try: - if opttype == "bool": - v = self.getboolean(sec, optname) - elif opttype == "int": - v = self.getint(sec, optname) - else: - v = self.get(sec, optname, vars=pOptions) + v = self.get(sec, optname, vars=pOptions) values[optname] = v + if convert: + conv = CONVERTER.get(opttype) + if conv: + if v is None: continue + values[optname] = conv(v) except NoSectionError as e: if shouldExist: raise @@ -253,11 +277,11 @@ class ConfigReaderUnshared(SafeConfigParserWithIncludes): # TODO: validate error handling here. except NoOptionError: if not optvalue is None: - logSys.warning("'%s' not defined in '%s'. Using default one: %r" + logSys.debug("'%s' not defined in '%s'. Using default one: %r" % (optname, sec, optvalue)) values[optname] = optvalue - elif logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) + # elif logSys.getEffectiveLevel() <= logLevel: + # logSys.log(logLevel, "Non essential option '%s' not defined in '%s'.", optname, sec) except ValueError: logSys.warning("Wrong value for '" + optname + "' in '" + sec + "'. Using default one: '" + repr(optvalue) + "'") @@ -315,8 +339,9 @@ class DefinitionInitConfigReader(ConfigReader): pOpts = dict() if self._initOpts: pOpts = _merge_dicts(pOpts, self._initOpts) + # type-convert only in combined (otherwise int/bool converting prevents substitution): self._opts = ConfigReader.getOptions( - self, "Definition", self._configOpts, pOpts) + self, "Definition", self._configOpts, pOpts, convert=False) self._pOpts = pOpts if self.has_section("Init"): # get only own options (without options from default): @@ -337,10 +362,21 @@ class DefinitionInitConfigReader(ConfigReader): if opt == '__name__' or opt in self._opts: continue self._opts[opt] = self.get("Definition", opt) + def convertOptions(self, opts, configOpts): + """Convert interpolated combined options to expected type. + """ + for opttype, optname, optvalue in _OptionsTemplateGen(configOpts): + conv = CONVERTER.get(opttype) + if conv: + v = opts.get(optname) + if v is None: continue + try: + opts[optname] = conv(v) + except ValueError: + logSys.warning("Wrong %s value %r for %r. Using default one: %r", + opttype, v, optname, optvalue) + opts[optname] = optvalue - def _convert_to_boolean(self, value): - return _as_bool(value) - def getCombOption(self, optname): """Get combined definition option (as string) using pre-set and init options as preselection (values with higher precedence as specified in section). @@ -375,6 +411,8 @@ class DefinitionInitConfigReader(ConfigReader): ignore=ignore, addrepl=self.getCombOption) if not opts: raise ValueError('recursive tag definitions unable to be resolved') + # convert options after all interpolations: + self.convertOptions(opts, self._configOpts) return opts def convert(self): diff --git a/fail2ban/client/csocket.py b/fail2ban/client/csocket.py index ab3e294b..88795674 100644 --- a/fail2ban/client/csocket.py +++ b/fail2ban/client/csocket.py @@ -48,7 +48,8 @@ class CSocket: def send(self, msg, nonblocking=False, timeout=None): # Convert every list member to string obj = dumps(map(CSocket.convert, msg), HIGHEST_PROTOCOL) - self.__csock.send(obj + CSPROTO.END) + self.__csock.send(obj) + self.__csock.send(CSPROTO.END) return self.receive(self.__csock, nonblocking, timeout) def settimeout(self, timeout): @@ -81,9 +82,12 @@ class CSocket: msg = CSPROTO.EMPTY if nonblocking: sock.setblocking(0) if timeout: sock.settimeout(timeout) - while msg.rfind(CSPROTO.END) == -1: - chunk = sock.recv(512) - if chunk in ('', b''): # python 3.x may return b'' instead of '' - raise RuntimeError("socket connection broken") + bufsize = 1024 + while msg.rfind(CSPROTO.END, -32) == -1: + chunk = sock.recv(bufsize) + if not len(chunk): + raise socket.error(104, 'Connection reset by peer') + if chunk == CSPROTO.END: break msg = msg + chunk + if bufsize < 32768: bufsize <<= 1 return loads(msg) diff --git a/fail2ban/client/fail2banclient.py b/fail2ban/client/fail2banclient.py index 73abacb2..f3b0f7b2 100755 --- a/fail2ban/client/fail2banclient.py +++ b/fail2ban/client/fail2banclient.py @@ -168,30 +168,20 @@ class Fail2banClient(Fail2banCmdLine, Thread): if not ret: return None - # verify that directory for the socket file exists - socket_dir = os.path.dirname(self._conf["socket"]) - if not os.path.exists(socket_dir): - logSys.error( - "There is no directory %s to contain the socket file %s." - % (socket_dir, self._conf["socket"])) - return None - if not os.access(socket_dir, os.W_OK | os.X_OK): # pragma: no cover - logSys.error( - "Directory %s exists but not accessible for writing" - % (socket_dir,)) - return None - # Check already running if not self._conf["force"] and os.path.exists(self._conf["socket"]): logSys.error("Fail2ban seems to be in unexpected state (not running but the socket exists)") return None - stream.append(['server-status']) - return stream + return [["server-stream", stream], ['server-status']] + + def _set_server(self, s): + self._server = s ## def __startServer(self, background=True): from .fail2banserver import Fail2banServer + # read configuration here (in client only, in server we do that in the config-thread): stream = self.__prepareStartServer() self._alive = True if not stream: @@ -206,16 +196,19 @@ class Fail2banClient(Fail2banCmdLine, Thread): return False else: # In foreground mode we should make server/client communication in different threads: - th = Thread(target=Fail2banClient.__processStartStreamAfterWait, args=(self, stream, False)) - th.daemon = True - th.start() + phase = dict() + self.configureServer(phase=phase, stream=stream) # Mark current (main) thread as daemon: - self.setDaemon(True) + self.daemon = True # Start server direct here in main thread (not fork): - self._server = Fail2banServer.startServerDirect(self._conf, False) - + self._server = Fail2banServer.startServerDirect(self._conf, False, self._set_server) + if not phase.get('done', False): + if self._server: # pragma: no cover + self._server.quit() + self._server = None + exit(255) except ExitException: # pragma: no cover - pass + raise except Exception as e: # pragma: no cover output("") logSys.error("Exception while starting server " + ("background" if background else "foreground")) @@ -228,23 +221,39 @@ class Fail2banClient(Fail2banCmdLine, Thread): return True ## - def configureServer(self, nonsync=True, phase=None): + def configureServer(self, nonsync=True, phase=None, stream=None): # if asynchronous start this operation in the new thread: if nonsync: - th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase)) + if phase is not None: + # event for server ready flag: + def _server_ready(): + phase['start-ready'] = True + logSys.log(5, ' server phase %s', phase) + # notify waiting thread if server really ready + self._conf['onstart'] = _server_ready + th = Thread(target=Fail2banClient.configureServer, args=(self, False, phase, stream)) th.daemon = True - return th.start() + th.start() + # if we need to read configuration stream: + if stream is None and phase is not None: + # wait, do not continue if configuration is not 100% valid: + Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001) + logSys.log(5, ' server phase %s', phase) + if not phase.get('start', False): + raise ServerExecutionException('Async configuration of server failed') + return True # prepare: read config, check configuration is valid, etc.: if phase is not None: phase['start'] = True logSys.log(5, ' client phase %s', phase) - stream = self.__prepareStartServer() + if stream is None: + stream = self.__prepareStartServer() if phase is not None: phase['ready'] = phase['start'] = (True if stream else False) logSys.log(5, ' client phase %s', phase) if not stream: return False - # wait a litle bit for phase "start-ready" before enter active waiting: + # wait a little bit for phase "start-ready" before enter active waiting: if phase is not None: Utils.wait_for(lambda: phase.get('start-ready', None) is not None, 0.5, 0.001) phase['configure'] = (True if stream else False) @@ -335,13 +344,14 @@ class Fail2banClient(Fail2banCmdLine, Thread): def __processStartStreamAfterWait(self, *args): + ret = False try: # Wait for the server to start if not self.__waitOnServer(): # pragma: no cover logSys.error("Could not find server, waiting failed") return False # Configure the server - self.__processCmd(*args) + ret = self.__processCmd(*args) except ServerExecutionException as e: # pragma: no cover if self._conf["verbose"] > 1: logSys.exception(e) @@ -350,10 +360,11 @@ class Fail2banClient(Fail2banCmdLine, Thread): "remove " + self._conf["socket"] + ". If " "you used fail2ban-client to start the " "server, adding the -x option will do it") - if self._server: - self._server.quit() - return False - return True + + if not ret and self._server: # stop on error (foreground, config read in another thread): + self._server.quit() + self._server = None + return ret def __waitOnServer(self, alive=True, maxtime=None): if maxtime is None: diff --git a/fail2ban/client/fail2bancmdline.py b/fail2ban/client/fail2bancmdline.py index 1268ee9f..c2f6d0be 100644 --- a/fail2ban/client/fail2bancmdline.py +++ b/fail2ban/client/fail2bancmdline.py @@ -27,15 +27,20 @@ import sys from ..version import version, normVersion from ..protocol import printFormatted -from ..helpers import getLogger, str2LogLevel, getVerbosityFormat +from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, BrokenPipeError # Gets the instance of the logger. logSys = getLogger("fail2ban") def output(s): # pragma: no cover - print(s) + try: + print(s) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise -CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket",) +# Config parameters required to start fail2ban which can be also set via command line (overwrite fail2ban.conf), +CONFIG_PARAMS = ("socket", "pidfile", "logtarget", "loglevel", "syslogsocket") # Used to signal - we are in test cases (ex: prevents change logging params, log capturing, etc) PRODUCTION = True @@ -94,9 +99,10 @@ class Fail2banCmdLine(): output("and bans the corresponding IP addresses using firewall rules.") output("") output("Options:") - output(" -c <DIR> configuration directory") - output(" -s <FILE> socket path") - output(" -p <FILE> pidfile path") + output(" -c, --conf <DIR> configuration directory") + output(" -s, --socket <FILE> socket path") + output(" -p, --pidfile <FILE> pidfile path") + output(" --pname <NAME> name of the process (main thread) to identify instance (default fail2ban-server)") output(" --loglevel <LEVEL> logging level") output(" --logtarget <TARGET> logging target, use file-name or stdout, stderr, syslog or sysout.") output(" --syslogsocket auto|<FILE>") @@ -129,17 +135,15 @@ class Fail2banCmdLine(): """ for opt in optList: o = opt[0] - if o == "-c": + if o in ("-c", "--conf"): self._conf["conf"] = opt[1] - elif o == "-s": + elif o in ("-s", "--socket"): self._conf["socket"] = opt[1] - elif o == "-p": + elif o in ("-p", "--pidfile"): self._conf["pidfile"] = opt[1] - elif o.startswith("--log") or o.startswith("--sys"): - self._conf[ o[2:] ] = opt[1] - elif o in ["-d", "--dp", "--dump-pretty"]: + elif o in ("-d", "--dp", "--dump-pretty"): self._conf["dump"] = True if o == "-d" else 2 - elif o == "-t" or o == "--test": + elif o in ("-t", "--test"): self.cleanConfOnly = True self._conf["test"] = True elif o == "-v": @@ -163,12 +167,14 @@ class Fail2banCmdLine(): from ..server.mytime import MyTime output(MyTime.str2seconds(opt[1])) return True - elif o in ["-h", "--help"]: + elif o in ("-h", "--help"): self.dispUsage() return True - elif o in ["-V", "--version"]: + elif o in ("-V", "--version"): self.dispVersion(o == "-V") return True + elif o.startswith("--"): # other long named params (see also resetConf) + self._conf[ o[2:] ] = opt[1] return None def initCmdLine(self, argv): @@ -185,7 +191,8 @@ class Fail2banCmdLine(): try: cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'conf=', 'pidfile=', 'pname=', 'socket=', + 'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() @@ -227,7 +234,8 @@ class Fail2banCmdLine(): if not conf: self.configurator.readEarly() conf = self.configurator.getEarlyOptions() - self._conf[o] = conf[o] + if o in conf: + self._conf[o] = conf[o] logSys.info("Using socket file %s", self._conf["socket"]) @@ -304,18 +312,24 @@ class Fail2banCmdLine(): # since method is also exposed in API via globally bound variable @staticmethod def _exit(code=0): - if hasattr(os, '_exit') and os._exit: - os._exit(code) - else: - sys.exit(code) + # implicit flush without to produce broken pipe error (32): + sys.stderr.close() + try: + sys.stdout.flush() + # exit: + if hasattr(sys, 'exit') and sys.exit: + sys.exit(code) + else: + os._exit(code) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno != 32: # closed / broken pipe + raise @staticmethod def exit(code=0): logSys.debug("Exit with code %s", code) # because of possible buffered output in python, we should flush it before exit: logging.shutdown() - sys.stdout.flush() - sys.stderr.flush() # exit Fail2banCmdLine._exit(code) diff --git a/fail2ban/client/fail2banreader.py b/fail2ban/client/fail2banreader.py index c81d585e..1f135cf8 100644 --- a/fail2ban/client/fail2banreader.py +++ b/fail2ban/client/fail2banreader.py @@ -53,20 +53,30 @@ class Fail2banReader(ConfigReader): opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], + ["string", "allowipv6", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], + ["int", "dbmaxmatches", None], ["string", "dbpurgeage", "1d"]] self.__opts = ConfigReader.getOptions(self, "Definition", opts) if updateMainOpt: self.__opts.update(updateMainOpt) # check given log-level: str2LogLevel(self.__opts.get('loglevel', 0)) - + # thread options: + opts = [["int", "stacksize", ], + ] + if self.has_section("Thread"): + thopt = ConfigReader.getOptions(self, "Thread", opts) + if thopt: + self.__opts['thread'] = thopt + def convert(self): # Ensure logtarget/level set first so any db errors are captured # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return - order = {"syslogsocket":0, "loglevel":1, "logtarget":2, - "dbfile":50, "dbpurgeage":51} + order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, + "allowipv6": 14, + "dbfile":50, "dbmaxmatches":51, "dbpurgeage":51} stream = list() for opt in self.__opts: if opt in order: diff --git a/fail2ban/client/fail2banregex.py b/fail2ban/client/fail2banregex.py index 29723dfb..b1795588 100644 --- a/fail2ban/client/fail2banregex.py +++ b/fail2ban/client/fail2banregex.py @@ -21,20 +21,25 @@ Fail2Ban reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. This tools can test regular expressions for "fail2ban". - """ __author__ = "Fail2Ban Developers" -__copyright__ = "Copyright (c) 2004-2008 Cyril Jaquier, 2012-2014 Yaroslav Halchenko" +__copyright__ = """Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors +Copyright of modifications held by their respective authors. +Licensed under the GNU General Public License v2 (GPL). + +Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. +Many contributions by Yaroslav O. Halchenko, Steven Hiscocks, Sergey G. Brester (sebres).""" + __license__ = "GPL" import getopt import logging +import re import os import shlex import sys import time -import time import urllib from optparse import OptionParser, Option @@ -47,7 +52,7 @@ except ImportError: from ..version import version, normVersion from .filterreader import FilterReader -from ..server.filter import Filter, FileContainer +from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ @@ -97,33 +102,37 @@ def dumpNormVersion(*args): output(normVersion()) sys.exit(0) -def get_opt_parser(): - # use module docstring for help output - p = OptionParser( - usage="%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]\n" % sys.argv[0] + __doc__ - + """ +usage = lambda: "%s [OPTIONS] <LOG> <REGEX> [IGNOREREGEX]" % sys.argv[0] + +class _f2bOptParser(OptionParser): + def format_help(self, *args, **kwargs): + """ Overwritten format helper with full ussage.""" + self.usage = '' + return "Usage: " + usage() + "\n" + __doc__ + """ LOG: - string a string representing a log line - filename path to a log file (/var/log/auth.log) - "systemd-journal" search systemd journal (systemd-python required) + string a string representing a log line + filename path to a log file (/var/log/auth.log) + systemd-journal search systemd journal (systemd-python required), + optionally with backend parameters, see `man jail.conf` + for usage and examples (systemd-journal[journalflags=1]). REGEX: - string a string representing a 'failregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing a 'failregex' + filter name of filter, optionally with options (sshd[mode=aggressive]) + filename path to a filter file (filter.d/sshd.conf) IGNOREREGEX: - string a string representing an 'ignoreregex' - filename path to a filter file (filter.d/sshd.conf) + string a string representing an 'ignoreregex' + filename path to a filter file (filter.d/sshd.conf) +\n""" + OptionParser.format_help(self, *args, **kwargs) + """\n +Report bugs to https://github.com/fail2ban/fail2ban/issues\n +""" + __copyright__ + "\n" -Copyright (c) 2004-2008 Cyril Jaquier, 2008- Fail2Ban Contributors -Copyright of modifications held by their respective authors. -Licensed under the GNU General Public License v2 (GPL). - -Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. -Many contributions by Yaroslav O. Halchenko and Steven Hiscocks. -Report bugs to https://github.com/fail2ban/fail2ban/issues -""", +def get_opt_parser(): + # use module docstring for help output + p = _f2bOptParser( + usage=usage(), version="%prog " + version) p.add_options([ @@ -160,6 +169,10 @@ Report bugs to https://github.com/fail2ban/fail2ban/issues help="Verbose date patterns/regex in output"), Option("-D", "--debuggex", action='store_true', help="Produce debuggex.com urls for debugging there"), + Option("--no-check-all", action="store_false", dest="checkAllRegex", default=True, + help="Disable check for all regex's"), + Option("-o", "--out", action="store", dest="out", default=None, + help="Set token to print failure information only (row, id, ip, msg, host, ip4, ip6, dns, matches, ...)"), Option("--print-no-missed", action='store_true', help="Do not print any missed lines"), Option("--print-no-ignored", action='store_true', @@ -234,12 +247,15 @@ class Fail2banRegex(object): def __init__(self, opts): # set local protected members from given options: self.__dict__.update(dict(('_'+o,v) for o,v in opts.__dict__.iteritems())) + self._opts = opts self._maxlines_set = False # so we allow to override maxlines in cmdline self._datepattern_set = False self._journalmatch = None self.share_config=dict() self._filter = Filter(None) + self._prefREMatched = 0 + self._prefREGroups = list() self._ignoreregex = list() self._failregex = list() self._time_elapsed = None @@ -253,17 +269,25 @@ class Fail2banRegex(object): self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.timezone: self._filter.setLogTimeZone(opts.timezone) + self._filter.checkFindTime = False + if True: # not opts.out: + MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns) + from ..server.strptime import _updateTimeRE + _updateTimeRE() if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: self._filter.setUseDns(opts.usedns) self._filter.returnRawHost = opts.raw - self._filter.checkFindTime = False - self._filter.checkAllRegex = True - self._opts = opts + self._filter.checkAllRegex = opts.checkAllRegex and not opts.out + # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) + self._filter.ignorePending = bool(opts.out) + # callback to increment ignored RE's by index (during process): + self._filter.onIgnoreRegex = self._onIgnoreRegex + self._backend = 'auto' - def decode_line(self, line): - return FileContainer.decode_line('<LOG>', self._encoding, line) + def output(self, line): + if not self._opts.out: output(line) def encode_line(self, line): return line.encode(self._encoding, 'ignore') @@ -273,43 +297,63 @@ class Fail2banRegex(object): self._filter.setDatePattern(pattern) self._datepattern_set = True if pattern is not None: - output( "Use datepattern : %s" % ( - self._filter.getDatePattern()[1], ) ) + self.output( "Use datepattern : %s : %s" % ( + pattern, self._filter.getDatePattern()[1], ) ) def setMaxLines(self, v): if not self._maxlines_set: self._filter.setMaxLines(int(v)) self._maxlines_set = True - output( "Use maxlines : %d" % self._filter.getMaxLines() ) + self.output( "Use maxlines : %d" % self._filter.getMaxLines() ) def setJournalMatch(self, v): self._journalmatch = v + def _dumpRealOptions(self, reader, fltOpt): + realopts = {} + combopts = reader.getCombined() + # output all options that are specified in filter-argument as well as some special (mostly interested): + for k in ['logtype', 'datepattern'] + fltOpt.keys(): + # combined options win, but they contain only a sub-set in filter expected keys, + # so get the rest from definition section: + try: + realopts[k] = combopts[k] if k in combopts else reader.get('Definition', k) + except NoOptionError: # pragma: no cover + pass + self.output("Real filter options : %r" % realopts) + def readRegex(self, value, regextype): assert(regextype in ('fail', 'ignore')) regex = regextype + 'regex' # try to check - we've case filter?[options...]?: basedir = self._opts.config + fltName = value fltFile = None fltOpt = {} if regextype == 'fail': - fltName, fltOpt = extractOptions(value) - if fltName is not None: - if "." in fltName[~5:]: - tryNames = (fltName,) - else: - tryNames = (fltName, fltName + '.conf', fltName + '.local') - for fltFile in tryNames: - if not "/" in fltFile: - if os.path.basename(basedir) == 'filter.d': - fltFile = os.path.join(basedir, fltFile) - else: - fltFile = os.path.join(basedir, 'filter.d', fltFile) + if re.search(r'(?ms)^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): + try: + fltName, fltOpt = extractOptions(value) + if "." in fltName[~5:]: + tryNames = (fltName,) else: - basedir = os.path.dirname(fltFile) - if os.path.isfile(fltFile): - break - fltFile = None + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + except Exception as e: + output("ERROR: Wrong filter name or options: %s" % (str(e),)) + output(" while parsing: %s" % (value,)) + if self._verbose: raise(e) + return False # if it is filter file: if fltFile is not None: if (basedir == self._opts.config @@ -320,13 +364,15 @@ class Fail2banRegex(object): if os.path.basename(basedir) == 'filter.d': basedir = os.path.dirname(basedir) fltName = os.path.splitext(os.path.basename(fltName))[0] - output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) + self.output( "Use %11s filter file : %s, basedir: %s" % (regex, fltName, basedir) ) else: ## foreign file - readexplicit this file and includes if possible: - output( "Use %11s file : %s" % (regex, fltName) ) + self.output( "Use %11s file : %s" % (regex, fltName) ) basedir = None + if not os.path.isabs(fltName): # avoid join with "filter.d" inside FilterReader + fltName = os.path.abspath(fltName) if fltOpt: - output( "Use filter options : %r" % fltOpt ) + self.output( "Use filter options : %r" % fltOpt ) reader = FilterReader(fltName, 'fail2ban-regex-jail', fltOpt, share_config=self.share_config, basedir=basedir) ret = None try: @@ -342,7 +388,14 @@ class Fail2banRegex(object): if not ret: output( "ERROR: failed to load filter %s" % value ) return False + # set backend-related options (logtype): + reader.applyAutoOptions(self._backend) + # get, interpolate and convert options: reader.getOptions(None) + # show real options if expected: + if self._verbose > 1 or logSys.getEffectiveLevel()<=logging.DEBUG: + self._dumpRealOptions(reader, fltOpt) + # to stream: readercommands = reader.convert() regex_values = {} @@ -384,7 +437,7 @@ class Fail2banRegex(object): return False else: - output( "Use %11s line : %s" % (regex, shortstr(value)) ) + self.output( "Use %11s line : %s" % (regex, shortstr(value)) ) regex_values = {regextype: [RegexStat(value)]} for regextype, regex_values in regex_values.iteritems(): @@ -396,71 +449,144 @@ class Fail2banRegex(object): 'add%sRegex' % regextype.title())(regex.getFailRegex()) return True - def testIgnoreRegex(self, line): - found = False - try: - ret = self._filter.ignoreLine([(line, "", "")]) - if ret is not None: - found = True - regex = self._ignoreregex[ret].inc() - except RegexException as e: # pragma: no cover - output( 'ERROR: %s' % e ) - return False - return found + def _onIgnoreRegex(self, idx, ignoreRegex): + self._lineIgnored = True + self._ignoreregex[idx].inc() def testRegex(self, line, date=None): orgLineBuffer = self._filter._Filter__lineBuffer + # duplicate line buffer (list can be changed inplace during processLine): + if self._filter.getMaxLines() > 1: + orgLineBuffer = orgLineBuffer[:] fullBuffer = len(orgLineBuffer) >= self._filter.getMaxLines() - is_ignored = False + is_ignored = self._lineIgnored = False try: found = self._filter.processLine(line, date) lines = [] - line = self._filter.processedLine() ret = [] for match in found: - # Append True/False flag depending if line was matched by - # more than one regex - match.append(len(ret)>1) - regex = self._failregex[match[0]] - regex.inc() - regex.appendIP(match) + if not self._opts.out: + # Append True/False flag depending if line was matched by + # more than one regex + match.append(len(ret)>1) + regex = self._failregex[match[0]] + regex.inc() + regex.appendIP(match) if not match[3].get('nofail'): ret.append(match) else: is_ignored = True + if self._opts.out: # (formated) output - don't need stats: + return None, ret, None + # prefregex stats: + if self._filter.prefRegex: + pre = self._filter.prefRegex + if pre.hasMatched(): + self._prefREMatched += 1 + if self._verbose: + if len(self._prefREGroups) < self._maxlines: + self._prefREGroups.append(pre.getGroups()) + else: + if len(self._prefREGroups) == self._maxlines: + self._prefREGroups.append('...') except RegexException as e: # pragma: no cover output( 'ERROR: %s' % e ) - return False - for bufLine in orgLineBuffer[int(fullBuffer):]: - if bufLine not in self._filter._Filter__lineBuffer: - try: - self._line_stats.missed_lines.pop( - self._line_stats.missed_lines.index("".join(bufLine))) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.pop( - self._line_stats.missed_lines_timeextracted.index( - "".join(bufLine[::2]))) - except ValueError: - pass - # if buffering - add also another lines from match: - if self._print_all_matched: - if not self._debuggex: - self._line_stats.matched_lines.append("".join(bufLine)) - else: - lines.append(bufLine[0] + bufLine[2]) - self._line_stats.matched += 1 - self._line_stats.missed -= 1 + return None, 0, None + if self._filter.getMaxLines() > 1: + for bufLine in orgLineBuffer[int(fullBuffer):]: + if bufLine not in self._filter._Filter__lineBuffer: + try: + self._line_stats.missed_lines.pop( + self._line_stats.missed_lines.index("".join(bufLine))) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.pop( + self._line_stats.missed_lines_timeextracted.index( + "".join(bufLine[::2]))) + except ValueError: + pass + # if buffering - add also another lines from match: + if self._print_all_matched: + if not self._debuggex: + self._line_stats.matched_lines.append("".join(bufLine)) + else: + lines.append(bufLine[0] + bufLine[2]) + self._line_stats.matched += 1 + self._line_stats.missed -= 1 if lines: # pre-lines parsed in multiline mode (buffering) - lines.append(line) + lines.append(self._filter.processedLine()) line = "\n".join(lines) - return line, ret, is_ignored + return line, ret, (is_ignored or self._lineIgnored) + + def _prepaireOutput(self): + """Prepares output- and fetch-function corresponding given '--out' option (format)""" + ofmt = self._opts.out + if ofmt in ('id', 'fid'): + def _out(ret): + for r in ret: + output(r[1]) + elif ofmt == 'ip': + def _out(ret): + for r in ret: + output(r[3].get('ip', r[1])) + elif ofmt == 'msg': + def _out(ret): + for r in ret: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + output(r) + elif ofmt == 'row': + def _out(ret): + for r in ret: + output('[%r,\t%r,\t%r],' % (r[1],r[2],dict((k,v) for k, v in r[3].iteritems() if k != 'matches'))) + elif '<' not in ofmt: + def _out(ret): + for r in ret: + output(r[3].get(ofmt)) + else: # extended format with tags substitution: + from ..server.actions import Actions, CommandAction, BanTicket + def _escOut(t, v): + # use safe escape (avoid inject on pseudo tag "\x00msg\x00"): + if t not in ('msg',): + return v.replace('\x00', '\\x00') + return v + def _out(ret): + rows = [] + wrap = {'NL':0} + for r in ret: + ticket = BanTicket(r[1], time=r[2], data=r[3]) + aInfo = Actions.ActionInfo(ticket) + # if msg tag is used - output if single line (otherwise let it as is to wrap multilines later): + def _get_msg(self): + if not wrap['NL'] and len(r[3].get('matches', [])) <= 1: + return self['matches'] + else: # pseudo tag for future replacement: + wrap['NL'] = 1 + return "\x00msg\x00" + aInfo['msg'] = _get_msg + # not recursive interpolation (use safe escape): + v = CommandAction.replaceDynamicTags(ofmt, aInfo, escapeVal=_escOut) + if wrap['NL']: # contains multiline tags (msg): + rows.append((r, v)) + continue + output(v) + # wrap multiline tag (msg) interpolations to single line: + for r, v in rows: + for r in r[3].get('matches'): + if not isinstance(r, basestring): + r = ''.join(r for r in r) + r = v.replace("\x00msg\x00", r) + output(r) + return _out + def process(self, test_lines): t0 = time.time() + if self._opts.out: # get out function + out = self._prepaireOutput() for line in test_lines: if isinstance(line, tuple): - line_datetimestripped, ret, is_ignored = self.testRegex( - line[0], line[1]) + line_datetimestripped, ret, is_ignored = self.testRegex(line[0], line[1]) line = "".join(line[0]) else: line = line.rstrip('\r\n') @@ -468,8 +594,10 @@ class Fail2banRegex(object): # skip comment and empty lines continue line_datetimestripped, ret, is_ignored = self.testRegex(line) - if not is_ignored: - is_ignored = self.testIgnoreRegex(line_datetimestripped) + + if self._opts.out: # (formated) output: + if len(ret) > 0 and not is_ignored: out(ret) + continue if is_ignored: self._line_stats.ignored += 1 @@ -477,28 +605,25 @@ class Fail2banRegex(object): self._line_stats.ignored_lines.append(line) if self._debuggex: self._line_stats.ignored_lines_timeextracted.append(line_datetimestripped) - - if len(ret) > 0: - assert(not is_ignored) + elif len(ret) > 0: self._line_stats.matched += 1 if self._print_all_matched: self._line_stats.matched_lines.append(line) if self._debuggex: self._line_stats.matched_lines_timeextracted.append(line_datetimestripped) else: - if not is_ignored: - self._line_stats.missed += 1 - if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): - self._line_stats.missed_lines.append(line) - if self._debuggex: - self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) + self._line_stats.missed += 1 + if not self._print_no_missed and (self._print_all_missed or self._line_stats.missed <= self._maxlines + 1): + self._line_stats.missed_lines.append(line) + if self._debuggex: + self._line_stats.missed_lines_timeextracted.append(line_datetimestripped) self._line_stats.tested += 1 self._time_elapsed = time.time() - t0 def printLines(self, ltype): lstats = self._line_stats - assert(self._line_stats.missed == lstats.tested - (lstats.matched + lstats.ignored)) + assert(lstats.missed == lstats.tested - (lstats.matched + lstats.ignored)) lines = lstats[ltype] l = lstats[ltype + '_lines'] multiline = self._filter.getMaxLines() > 1 @@ -528,6 +653,7 @@ class Fail2banRegex(object): "to print all %d lines" % (header, ltype, lines) ) def printStats(self): + if self._opts.out: return True output( "" ) output( "Results" ) output( "=======" ) @@ -555,7 +681,18 @@ class Fail2banRegex(object): pprint_list(out, " #) [# of hits] regular expression") return total - # Print title + # Print prefregex: + if self._filter.prefRegex: + #self._filter.prefRegex.hasMatched() + pre = self._filter.prefRegex + out = [pre.getRegex()] + if self._verbose: + for grp in self._prefREGroups: + out.append(" %s" % (grp,)) + output( "\n%s: %d total" % ("Prefregex", self._prefREMatched) ) + pprint_list(out) + + # Print regex's: total = print_failregexes("Failregex", self._failregex) _ = print_failregexes("Ignoreregex", self._ignoreregex) @@ -587,14 +724,13 @@ class Fail2banRegex(object): return True - def file_lines_gen(self, hdlr): - for line in hdlr: - yield self.decode_line(line) - def start(self, args): cmd_log, cmd_regex = args[:2] + if cmd_log.startswith("systemd-journal"): # pragma: no cover + self._backend = 'systemd' + try: if not self.readRegex(cmd_regex, 'fail'): # pragma: no cover return False @@ -606,10 +742,10 @@ class Fail2banRegex(object): if os.path.isfile(cmd_log): try: - hdlr = open(cmd_log, 'rb') - output( "Use log file : %s" % cmd_log ) - output( "Use encoding : %s" % self._encoding ) - test_lines = self.file_lines_gen(hdlr) + test_lines = FileContainer(cmd_log, self._encoding, doOpen=True) + + self.output( "Use log file : %s" % cmd_log ) + self.output( "Use encoding : %s" % self._encoding ) except IOError as e: # pragma: no cover output( e ) return False @@ -617,8 +753,8 @@ class Fail2banRegex(object): if not FilterSystemd: output( "Error: systemd library not found. Exiting..." ) return False - output( "Use systemd journal" ) - output( "Use encoding : %s" % self._encoding ) + self.output( "Use systemd journal" ) + self.output( "Use encoding : %s" % self._encoding ) backend, beArgs = extractOptions(cmd_log) flt = FilterSystemd(None, **beArgs) flt.setLogEncoding(self._encoding) @@ -627,23 +763,23 @@ class Fail2banRegex(object): self.setDatePattern(None) if journalmatch: flt.addJournalMatch(journalmatch) - output( "Use journal match : %s" % " ".join(journalmatch) ) + self.output( "Use journal match : %s" % " ".join(journalmatch) ) test_lines = journal_lines_gen(flt, myjournal) else: # if single line parsing (without buffering) - if self._filter.getMaxLines() <= 1: - output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) + if self._filter.getMaxLines() <= 1 and '\n' not in cmd_log: + self.output( "Use single line : %s" % shortstr(cmd_log.replace("\n", r"\n")) ) test_lines = [ cmd_log ] - else: # multi line parsing (with buffering) + else: # multi line parsing (with and without buffering) test_lines = cmd_log.split("\n") - output( "Use multi line : %s line(s)" % len(test_lines) ) + self.output( "Use multi line : %s line(s)" % len(test_lines) ) for i, l in enumerate(test_lines): if i >= 5: - output( "| ..." ); break - output( "| %2.2s: %s" % (i+1, shortstr(l)) ) - output( "`-" ) + self.output( "| ..." ); break + self.output( "| %2.2s: %s" % (i+1, shortstr(l)) ) + self.output( "`-" ) - output( "" ) + self.output( "" ) self.process(test_lines) @@ -654,6 +790,7 @@ class Fail2banRegex(object): def exec_command_line(*args): + logging.exitOnIOError = True parser = get_opt_parser() (opts, args) = parser.parse_args(*args) errors = [] @@ -666,14 +803,15 @@ def exec_command_line(*args): if not len(args) in (2, 3): errors.append("ERROR: provide both <LOG> and <REGEX>.") if errors: - sys.stderr.write("\n".join(errors) + "\n\n") parser.print_help() + sys.stderr.write("\n" + "\n".join(errors) + "\n") sys.exit(255) - output( "" ) - output( "Running tests" ) - output( "=============" ) - output( "" ) + if not opts.out: + output( "" ) + output( "Running tests" ) + output( "=============" ) + output( "" ) # Log level (default critical): opts.log_level = str2LogLevel(opts.log_level) @@ -694,6 +832,14 @@ def exec_command_line(*args): stdout.setFormatter(Formatter(getVerbosityFormat(opts.verbose, fmt))) logSys.addHandler(stdout) - fail2banRegex = Fail2banRegex(opts) + try: + fail2banRegex = Fail2banRegex(opts) + except Exception as e: + if opts.verbose or logSys.getEffectiveLevel()<=logging.DEBUG: + logSys.critical(e, exc_info=True) + else: + output( 'ERROR: %s' % e ) + sys.exit(255) + if not fail2banRegex.start(args): sys.exit(255) diff --git a/fail2ban/client/fail2banserver.py b/fail2ban/client/fail2banserver.py index d94d13ff..eee78d5f 100644 --- a/fail2ban/client/fail2banserver.py +++ b/fail2ban/client/fail2banserver.py @@ -44,7 +44,7 @@ class Fail2banServer(Fail2banCmdLine): # Start the Fail2ban server in background/foreground (daemon mode or not). @staticmethod - def startServerDirect(conf, daemon=True): + def startServerDirect(conf, daemon=True, setServer=None): logSys.debug(" direct starting of server in %s, deamon: %s", os.getpid(), daemon) from ..server.server import Server server = None @@ -52,6 +52,10 @@ class Fail2banServer(Fail2banCmdLine): # Start it in foreground (current thread, not new process), # server object will internally fork self if daemon is True server = Server(daemon) + # notify caller - set server handle: + if setServer: + setServer(server) + # run: server.start(conf["socket"], conf["pidfile"], conf["force"], conf=conf) @@ -63,6 +67,10 @@ class Fail2banServer(Fail2banCmdLine): if conf["verbose"] > 1: logSys.exception(e2) raise + finally: + # notify waiting thread server ready resp. done (background execution, error case, etc): + if conf.get('onstart'): + conf['onstart']() return server @@ -179,27 +187,15 @@ class Fail2banServer(Fail2banCmdLine): # Start new thread with client to read configuration and # transfer it to the server: cli = self._Fail2banClient() + cli._conf = self._conf phase = dict() logSys.debug('Configure via async client thread') cli.configureServer(phase=phase) - # wait, do not continue if configuration is not 100% valid: - Utils.wait_for(lambda: phase.get('ready', None) is not None, self._conf["timeout"], 0.001) - logSys.log(5, ' server phase %s', phase) - if not phase.get('start', False): - raise ServerExecutionException('Async configuration of server failed') - # event for server ready flag: - def _server_ready(): - phase['start-ready'] = True - logSys.log(5, ' server phase %s', phase) - # notify waiting thread if server really ready - self._conf['onstart'] = _server_ready # Start server, daemonize it, etc. pid = os.getpid() - server = Fail2banServer.startServerDirect(self._conf, background) - # notify waiting thread server ready resp. done (background execution, error case, etc): - if not nonsync: - _server_ready() + server = Fail2banServer.startServerDirect(self._conf, background, + cli._set_server if cli else None) # If forked - just exit other processes if pid != os.getpid(): # pragma: no cover os._exit(0) diff --git a/fail2ban/client/filterreader.py b/fail2ban/client/filterreader.py index 9edeb2f3..24341014 100644 --- a/fail2ban/client/filterreader.py +++ b/fail2ban/client/filterreader.py @@ -37,9 +37,10 @@ logSys = getLogger(__name__) class FilterReader(DefinitionInitConfigReader): _configOpts = { + "usedns": ["string", None], "prefregex": ["string", None], "ignoreregex": ["string", None], - "failregex": ["string", ""], + "failregex": ["string", None], "maxlines": ["int", None], "datepattern": ["string", None], "journalmatch": ["string", None], @@ -52,35 +53,48 @@ class FilterReader(DefinitionInitConfigReader): def getFile(self): return self.__file + def applyAutoOptions(self, backend): + # set init option to backend-related logtype, considering + # that the filter settings may be overwritten in its local: + if (not self._initOpts.get('logtype') and + not self.has_option('Definition', 'logtype', False) + ): + self._initOpts['logtype'] = ['file','journal'][int(backend.startswith("systemd"))] + def convert(self): stream = list() opts = self.getCombined() if not len(opts): return stream + return FilterReader._fillStream(stream, opts, self._jailName) + + @staticmethod + def _fillStream(stream, opts, jailName): + prio0idx = 0 for opt, value in opts.iteritems(): + # Do not send a command if the value is not set (empty). + if value is None: continue if opt in ("failregex", "ignoreregex"): - if value is None: continue multi = [] for regex in value.split('\n'): # Do not send a command if the rule is empty. if regex != '': multi.append(regex) if len(multi) > 1: - stream.append(["multi-set", self._jailName, "add" + opt, multi]) + stream.append(["multi-set", jailName, "add" + opt, multi]) elif len(multi): - stream.append(["set", self._jailName, "add" + opt, multi[0]]) - elif opt in ('maxlines', 'prefregex'): - # Be sure we set this options first. - stream.insert(0, ["set", self._jailName, opt, value]) + stream.append(["set", jailName, "add" + opt, multi[0]]) + elif opt in ('usedns', 'maxlines', 'prefregex'): + # Be sure we set this options first, and usedns is before all regex(s). + stream.insert(0 if opt == 'usedns' else prio0idx, + ["set", jailName, opt, value]) + prio0idx += 1 elif opt in ('datepattern'): - stream.append(["set", self._jailName, opt, value]) - # Do not send a command if the match is empty. + stream.append(["set", jailName, opt, value]) elif opt == 'journalmatch': - if value is None: continue for match in value.split("\n"): if match == '': continue stream.append( - ["set", self._jailName, "addjournalmatch"] + - shlex.split(match)) + ["set", jailName, "addjournalmatch"] + shlex.split(match)) return stream diff --git a/fail2ban/client/jailreader.py b/fail2ban/client/jailreader.py index 4ee97317..37746d4c 100644 --- a/fail2ban/client/jailreader.py +++ b/fail2ban/client/jailreader.py @@ -33,7 +33,7 @@ from .configreader import ConfigReaderUnshared, ConfigReader from .filterreader import FilterReader from .actionreader import ActionReader from ..version import version -from ..helpers import getLogger, extractOptions, splitwords +from ..helpers import getLogger, extractOptions, splitWithOptions, splitwords # Gets the instance of the logger. logSys = getLogger(__name__) @@ -86,43 +86,54 @@ class JailReader(ConfigReader): logSys.warning("File %s is a dangling link, thus cannot be monitored" % p) return pathList + _configOpts1st = { + "enabled": ["bool", False], + "backend": ["string", "auto"], + "filter": ["string", ""] + } + _configOpts = { + "enabled": ["bool", False], + "backend": ["string", "auto"], + "maxretry": ["int", None], + "maxmatches": ["int", None], + "findtime": ["string", None], + "bantime": ["string", None], + "bantime.increment": ["bool", None], + "bantime.factor": ["string", None], + "bantime.formula": ["string", None], + "bantime.multipliers": ["string", None], + "bantime.maxtime": ["string", None], + "bantime.rndtime": ["string", None], + "bantime.overalljails": ["bool", None], + "ignorecommand": ["string", None], + "ignoreself": ["bool", None], + "ignoreip": ["string", None], + "ignorecache": ["string", None], + "filter": ["string", ""], + "logtimezone": ["string", None], + "logencoding": ["string", None], + "logpath": ["string", None], + "action": ["string", ""] + } + _configOpts.update(FilterReader._configOpts) + + _ignoreOpts = set(['action', 'filter', 'enabled'] + FilterReader._configOpts.keys()) + def getOptions(self): - opts1st = [["bool", "enabled", False], - ["string", "filter", ""]] - opts = [["bool", "enabled", False], - ["string", "backend", "auto"], - ["int", "maxretry", None], - ["string", "findtime", None], - ["string", "bantime", None], - ["bool", "bantime.increment", None], - ["string", "bantime.factor", None], - ["string", "bantime.formula", None], - ["string", "bantime.multipliers", None], - ["string", "bantime.maxtime", None], - ["string", "bantime.rndtime", None], - ["bool", "bantime.overalljails", None], - ["string", "usedns", None], # be sure usedns is before all regex(s) in stream - ["string", "failregex", None], - ["string", "ignoreregex", None], - ["string", "ignorecommand", None], - ["bool", "ignoreself", None], - ["string", "ignoreip", None], - ["string", "ignorecache", None], - ["string", "filter", ""], - ["string", "datepattern", None], - ["string", "logtimezone", None], - ["string", "logencoding", None], - ["string", "logpath", None], # logpath after all log-related data (backend, date-pattern, etc) - ["string", "action", ""]] + + basedir = self.getBaseDir() # Before interpolation (substitution) add static options always available as default: - defsec = self._cfg.get_defaults() - defsec["fail2ban_version"] = version + self.merge_defaults({ + "fail2ban_version": version, + "fail2ban_confpath": basedir + }) try: # Read first options only needed for merge defaults ('known/...' from filter): - self.__opts = ConfigReader.getOptions(self, self.__name, opts1st, shouldExist=True) + self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts1st, + shouldExist=True) if not self.__opts: # pragma: no cover raise JailDefError("Init jail options failed") @@ -132,15 +143,18 @@ class JailReader(ConfigReader): # Read filter flt = self.__opts["filter"] if flt: - filterName, filterOpt = extractOptions(flt) - if not filterName: - raise JailDefError("Invalid filter definition %r" % flt) + try: + filterName, filterOpt = extractOptions(flt) + except ValueError as e: + raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) self.__filter = FilterReader( filterName, self.__name, filterOpt, - share_config=self.share_config, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=basedir) ret = self.__filter.read() if not ret: raise JailDefError("Unable to read the filter %r" % filterName) + # set backend-related options (logtype): + self.__filter.applyAutoOptions(self.__opts.get('backend', '')) # merge options from filter as 'known/...' (all options unfiltered): self.__filter.getOptions(self.__opts, all=True) ConfigReader.merge_section(self, self.__name, self.__filter.getCombined(), 'known/') @@ -149,7 +163,7 @@ class JailReader(ConfigReader): logSys.warning("No filter set for jail %s" % self.__name) # Read second all options (so variables like %(known/param) can be interpolated): - self.__opts = ConfigReader.getOptions(self, self.__name, opts) + self.__opts = ConfigReader.getOptions(self, self.__name, self._configOpts) if not self.__opts: # pragma: no cover raise JailDefError("Read jail options failed") @@ -158,13 +172,16 @@ class JailReader(ConfigReader): self.__filter.getOptions(self.__opts) # Read action - for act in self.__opts["action"].split('\n'): + for act in splitWithOptions(self.__opts["action"]): try: + act = act.strip() if not act: # skip empty actions continue - actName, actOpt = extractOptions(act) - if not actName: - raise JailDefError("Invalid action definition %r" % act) + # join with previous line if needed (consider possible new-line): + try: + actName, actOpt = extractOptions(act) + except ValueError as e: + raise JailDefError("Invalid action definition %r: %s" % (act, e)) if actName.endswith(".py"): self.__actions.append([ "set", @@ -172,13 +189,13 @@ class JailReader(ConfigReader): "addaction", actOpt.pop("actname", os.path.splitext(actName)[0]), os.path.join( - self.getBaseDir(), "action.d", actName), + basedir, "action.d", actName), json.dumps(actOpt), ]) else: action = ActionReader( actName, self.__name, actOpt, - share_config=self.share_config, basedir=self.getBaseDir()) + share_config=self.share_config, basedir=basedir) ret = action.read() if ret: action.getOptions(self.__opts) @@ -213,15 +230,19 @@ class JailReader(ConfigReader): """ stream = [] + stream2 = [] e = self.__opts.get('config-error') if e: stream.extend([['config-error', "Jail '%s' skipped, because of wrong configuration: %s" % (self.__name, e)]]) return stream + # fill jail with filter options, using filter (only not overriden in jail): if self.__filter: stream.extend(self.__filter.convert()) + # and using options from jail: + FilterReader._fillStream(stream, self.__opts, self.__name) for opt, value in self.__opts.iteritems(): if opt == "logpath": - if self.__opts.get('backend', None).startswith("systemd"): continue + if self.__opts.get('backend', '').startswith("systemd"): continue found_files = 0 for path in value.split("\n"): path = path.rsplit(" ", 1) @@ -231,33 +252,22 @@ class JailReader(ConfigReader): logSys.notice("No file(s) found for glob %s" % path) for p in pathList: found_files += 1 - stream.append( + # logpath after all log-related data (backend, date-pattern, etc) + stream2.append( ["set", self.__name, "addlogpath", p, tail]) if not found_files: msg = "Have not found any log file for %s jail" % self.__name if not allow_no_files: raise ValueError(msg) logSys.warning(msg) - - elif opt == "logencoding": - stream.append(["set", self.__name, "logencoding", value]) elif opt == "backend": backend = value elif opt == "ignoreip": - for ip in splitwords(value): - stream.append(["set", self.__name, "addignoreip", ip]) - elif opt in ("failregex", "ignoreregex"): - multi = [] - for regex in value.split('\n'): - # Do not send a command if the rule is empty. - if regex != '': - multi.append(regex) - if len(multi) > 1: - stream.append(["multi-set", self.__name, "add" + opt, multi]) - elif len(multi): - stream.append(["set", self.__name, "add" + opt, multi[0]]) - elif opt not in ('action', 'filter', 'enabled'): + stream.append(["set", self.__name, "addignoreip"] + splitwords(value)) + elif opt not in JailReader._ignoreOpts: stream.append(["set", self.__name, opt, value]) + # consider options order (after other options): + if stream2: stream += stream2 for action in self.__actions: if isinstance(action, (ConfigReaderUnshared, ConfigReader)): stream.extend(action.convert()) diff --git a/fail2ban/helpers.py b/fail2ban/helpers.py index 49a5ca47..5c1750a6 100644 --- a/fail2ban/helpers.py +++ b/fail2ban/helpers.py @@ -32,6 +32,13 @@ from threading import Lock from .server.mytime import MyTime +try: + import ctypes + _libcap = ctypes.CDLL('libcap.so.2') +except: + _libcap = None + + PREFER_ENC = locale.getpreferredencoding() # correct preferred encoding if lang not set in environment: if PREFER_ENC.startswith('ANSI_'): # pragma: no cover @@ -201,6 +208,26 @@ class FormatterWithTraceBack(logging.Formatter): return logging.Formatter.format(self, record) +logging.exitOnIOError = False +def __stopOnIOError(logSys=None, logHndlr=None): # pragma: no cover + if logSys and len(logSys.handlers): + logSys.removeHandler(logSys.handlers[0]) + if logHndlr: + logHndlr.close = lambda: None + logging.StreamHandler.flush = lambda self: None + #sys.excepthook = lambda *args: None + if logging.exitOnIOError: + try: + sys.stderr.close() + except: + pass + sys.exit(0) + +try: + BrokenPipeError = BrokenPipeError +except NameError: # pragma: 3.x no cover + BrokenPipeError = IOError + __origLog = logging.Logger._log def __safeLog(self, level, msg, args, **kwargs): """Safe log inject to avoid possible errors by unsafe log-handlers, @@ -216,6 +243,10 @@ def __safeLog(self, level, msg, args, **kwargs): try: # if isEnabledFor(level) already called... __origLog(self, level, msg, args, **kwargs) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(self) + raise except Exception as e: # pragma: no cover - unreachable if log-handler safe in this python-version try: for args in ( @@ -230,6 +261,18 @@ def __safeLog(self, level, msg, args, **kwargs): pass logging.Logger._log = __safeLog +__origLogFlush = logging.StreamHandler.flush +def __safeLogFlush(self): + """Safe flush inject stopping endless logging on closed streams (redirected pipe). + """ + try: + __origLogFlush(self) + except (BrokenPipeError, IOError) as e: # pragma: no cover + if e.errno == 32: # closed / broken pipe + __stopOnIOError(None, self) + raise +logging.StreamHandler.flush = __safeLogFlush + def getLogger(name): """Get logging.Logger instance with Fail2Ban logger name convention """ @@ -260,7 +303,7 @@ def getVerbosityFormat(verbosity, fmt=' %(message)s', addtime=True, padding=True if addtime: fmt = ' %(asctime)-15s' + fmt else: # default (not verbose): - fmt = "%(name)-23.23s [%(process)d]: %(levelname)-7s" + fmt + fmt = "%(name)-24s[%(process)d]: %(levelname)-7s" + fmt if addtime: fmt = "%(asctime)s " + fmt # remove padding if not needed: @@ -284,7 +327,7 @@ def splitwords(s): """ if not s: return [] - return filter(bool, map(str.strip, re.split('[ ,\n]+', s))) + return filter(bool, map(lambda v: v.strip(), re.split('[ ,\n]+', s))) if sys.version_info >= (3,5): eval(compile(r'''if 1: @@ -328,30 +371,42 @@ OPTION_CRE = re.compile(r"^([^\[]+)(?:\[(.*)\])?\s*$", re.DOTALL) # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` OPTION_EXTRACT_CRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) + r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P<wrngA>.+))|,?\s*$|(?P<wrngB>.+)', re.DOTALL) +# split by new-line considering possible new-lines within options [...]: +OPTION_SPLIT_CRE = re.compile( + r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) def extractOptions(option): match = OPTION_CRE.match(option) if not match: - # TODO proper error handling - return None, None + raise ValueError("unexpected option syntax") option_name, optstr = match.groups() option_opts = dict() if optstr: for optmatch in OPTION_EXTRACT_CRE.finditer(optstr): + if optmatch.group("wrngA"): + raise ValueError("unexpected syntax at %d after option %r: %s" % ( + optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25])) + if optmatch.group("wrngB"): + raise ValueError("expected option, wrong syntax at %d: %s" % ( + optmatch.start("wrngB"), optmatch.group("wrngB")[0:25])) opt = optmatch.group(1) + if not opt: continue value = [ val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() return option_name, option_opts +def splitWithOptions(option): + return OPTION_SPLIT_CRE.findall(option) + # # Following facilities used for safe recursive interpolation of # tags (<tag>) in tagged options. # -# max tag replacement count: -MAX_TAG_REPLACE_COUNT = 10 +# max tag replacement count (considering tag X in tag Y repeat): +MAX_TAG_REPLACE_COUNT = 25 # compiled RE for tag name (replacement name) TAG_CRE = re.compile(r'<([^ <>]+)>') @@ -379,13 +434,13 @@ def substituteRecursiveTags(inptags, conditional='', """ #logSys = getLogger("fail2ban") tre_search = TAG_CRE.search - # copy return tags dict to prevent modifying of inptags: - tags = inptags.copy() + tags = inptags # init: ignore = set(ignore) done = set() noRecRepl = hasattr(tags, "getRawItem") # repeat substitution while embedded-recursive (repFlag is True) + repCounts = {} while True: repFlag = False # substitute each value: @@ -397,7 +452,7 @@ def substituteRecursiveTags(inptags, conditional='', value = orgval = uni_string(tags[tag]) # search and replace all tags within value, that can be interpolated using other tags: m = tre_search(value) - refCounts = {} + rplc = repCounts.get(tag, {}) #logSys.log(5, 'TAG: %s, value: %s' % (tag, value)) while m: # found replacement tag: @@ -407,13 +462,13 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue #logSys.log(5, 'found: %s' % rtag) - if rtag == tag or refCounts.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: + if rtag == tag or rplc.get(rtag, 1) > MAX_TAG_REPLACE_COUNT: # recursive definitions are bad #logSys.log(5, 'recursion fail tag: %s value: %s' % (tag, value) ) raise ValueError( "properties contain self referencing definitions " "and cannot be resolved, fail tag: %s, found: %s in %s, value: %s" % - (tag, rtag, refCounts, value)) + (tag, rtag, rplc, value)) repl = None if conditional: repl = tags.get(rtag + '?' + conditional) @@ -429,11 +484,11 @@ def substituteRecursiveTags(inptags, conditional='', m = tre_search(value, m.end()) continue # if calling map - be sure we've string: - if noRecRepl: repl = uni_string(repl) + if not isinstance(repl, basestring): repl = uni_string(repl) value = value.replace('<%s>' % rtag, repl) #logSys.log(5, 'value now: %s' % value) # increment reference count: - refCounts[rtag] = refCounts.get(rtag, 0) + 1 + rplc[rtag] = rplc.get(rtag, 0) + 1 # the next match for replace: m = tre_search(value, m.start()) #logSys.log(5, 'TAG: %s, newvalue: %s' % (tag, value)) @@ -441,7 +496,11 @@ def substituteRecursiveTags(inptags, conditional='', if orgval != value: # check still contains any tag - should be repeated (possible embedded-recursive substitution): if tre_search(value): + repCounts[tag] = rplc repFlag = True + # copy return tags dict to prevent modifying of inptags: + if id(tags) == id(inptags): + tags = inptags.copy() tags[tag] = value # no more sub tags (and no possible composite), add this tag to done set (just to be faster): if '<' not in value: done.add(tag) @@ -451,6 +510,25 @@ def substituteRecursiveTags(inptags, conditional='', return tags +if _libcap: + def prctl_set_th_name(name): + """Helper to set real thread name (used for identification and diagnostic purposes). + + Side effect: name can be silently truncated to 15 bytes (16 bytes with NTS zero) + """ + try: + if sys.version_info >= (3,): # pragma: 2.x no cover + name = name.encode() + else: # pragma: 3.x no cover + name = bytes(name) + _libcap.prctl(15, name) # PR_SET_NAME = 15 + except: # pragma: no cover + pass +else: # pragma: no cover + def prctl_set_th_name(name): + pass + + class BgService(object): """Background servicing diff --git a/fail2ban/protocol.py b/fail2ban/protocol.py index b21ab848..a81c6657 100644 --- a/fail2ban/protocol.py +++ b/fail2ban/protocol.py @@ -55,6 +55,8 @@ protocol = [ ["stop", "stops all jails and terminate the server"], ["unban --all", "unbans all IP addresses (in all jails and database)"], ["unban <IP> ... <IP>", "unbans <IP> (in all jails and database)"], +["banned", "return jails with banned IPs as dictionary"], +["banned <IP> ... <IP>]", "return list(s) of jails where given IP(s) are banned"], ["status", "gets the current status of the server"], ["ping", "tests if the server is alive"], ["echo", "for internal usage, returns back and outputs a given string"], @@ -64,7 +66,7 @@ protocol = [ ["set loglevel <LEVEL>", "sets logging level to <LEVEL>. Levels: CRITICAL, ERROR, WARNING, NOTICE, INFO, " "DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5)"], ["get loglevel", "gets the logging level"], -["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG or a file"], +["set logtarget <TARGET>", "sets logging target to <TARGET>. Can be STDOUT, STDERR, SYSLOG, SYSTEMD-JOURNAL or a file"], ["get logtarget", "gets logging target"], ["set syslogsocket auto|<SOCKET>", "sets the syslog socket path to auto or <SOCKET>. Only used if logtarget is SYSLOG"], ["get syslogsocket", "gets syslog socket path"], @@ -72,6 +74,8 @@ protocol = [ ['', "DATABASE", ""], ["set dbfile <FILE>", "set the location of fail2ban persistent datastore. Set to \"None\" to disable"], ["get dbfile", "get the location of fail2ban persistent datastore"], +["set dbmaxmatches <INT>", "sets the max number of matches stored in database per ticket"], +["get dbmaxmatches", "gets the max number of matches stored in database per ticket"], ["set dbpurgeage <SECONDS>", "sets the max age in <SECONDS> that history of bans will be kept"], ["get dbpurgeage", "gets the max age in seconds that history of bans will be kept"], ['', "JAIL CONTROL", ""], @@ -99,9 +103,11 @@ protocol = [ ["set <JAIL> bantime <TIME>", "sets the number of seconds <TIME> a host will be banned for <JAIL>"], ["set <JAIL> datepattern <PATTERN>", "sets the <PATTERN> used to match date/times for <JAIL>"], ["set <JAIL> usedns <VALUE>", "sets the usedns mode for <JAIL>"], -["set <JAIL> banip <IP>", "manually Ban <IP> for <JAIL>"], -["set <JAIL> unbanip <IP>", "manually Unban <IP> in <JAIL>"], +["set <JAIL> attempt <IP> [<failure1> ... <failureN>]", "manually notify about <IP> failure"], +["set <JAIL> banip <IP> ... <IP>", "manually Ban <IP> for <JAIL>"], +["set <JAIL> unbanip [--report-absent] <IP> ... <IP>", "manually Unban <IP> in <JAIL>"], ["set <JAIL> maxretry <RETRY>", "sets the number of failures <RETRY> before banning the host for <JAIL>"], +["set <JAIL> maxmatches <INT>", "sets the max number of matches stored in memory per ticket in <JAIL>"], ["set <JAIL> maxlines <LINES>", "sets the number of <LINES> to buffer for regex search for <JAIL>"], ["set <JAIL> addaction <ACT>[ <PYTHONFILE> <JSONKWARGS>]", "adds a new action named <ACT> for <JAIL>. Optionally for a Python based action, a <PYTHONFILE> and <JSONKWARGS> can be specified, else will be a Command Action"], ["set <JAIL> delaction <ACT>", "removes the action <ACT> from <JAIL>"], @@ -116,6 +122,8 @@ protocol = [ ["set <JAIL> action <ACT> <PROPERTY> <VALUE>", "sets the <VALUE> of <PROPERTY> for the action <ACT> for <JAIL>"], ["set <JAIL> action <ACT> <METHOD>[ <JSONKWARGS>]", "calls the <METHOD> with <JSONKWARGS> for the action <ACT> for <JAIL>"], ['', "JAIL INFORMATION", ""], +["get <JAIL> banned", "return banned IPs of <JAIL>"], +["get <JAIL> banned <IP> ... <IP>]", "return 1 if IP is banned in <JAIL> otherwise 0, or a list of 1/0 for multiple IPs"], ["get <JAIL> logpath", "gets the list of the monitored files for <JAIL>"], ["get <JAIL> logencoding", "gets the encoding of the log files for <JAIL>"], ["get <JAIL> journalmatch", "gets the journal filter match for <JAIL>"], @@ -126,9 +134,11 @@ protocol = [ ["get <JAIL> ignoreregex", "gets the list of regular expressions which matches patterns to ignore for <JAIL>"], ["get <JAIL> findtime", "gets the time for which the filter will look back for failures for <JAIL>"], ["get <JAIL> bantime", "gets the time a host is banned for <JAIL>"], -["get <JAIL> datepattern", "gets the patern used to match date/times for <JAIL>"], +["get <JAIL> datepattern", "gets the pattern used to match date/times for <JAIL>"], ["get <JAIL> usedns", "gets the usedns setting for <JAIL>"], +["get <JAIL> banip [<SEP>|--with-time]", "gets the list of of banned IP addresses for <JAIL>. Optionally the separator character ('<SEP>', default is space) or the option '--with-time' (printing the times of ban) may be specified. The IPs are ordered by end of ban."], ["get <JAIL> maxretry", "gets the number of failures allowed for <JAIL>"], +["get <JAIL> maxmatches", "gets the max number of matches stored in memory per ticket in <JAIL>"], ["get <JAIL> maxlines", "gets the number of lines to buffer for <JAIL>"], ["get <JAIL> actions", "gets a list of actions for <JAIL>"], ["", "COMMAND ACTION INFORMATION",""], diff --git a/fail2ban/server/action.py b/fail2ban/server/action.py index ec9ad19a..16ff6621 100644 --- a/fail2ban/server/action.py +++ b/fail2ban/server/action.py @@ -30,13 +30,17 @@ import tempfile import threading import time from abc import ABCMeta -from collections import MutableMapping +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from .failregex import mapTag2Opt -from .ipdns import asip, DNSUtils +from .ipdns import DNSUtils from .mytime import MyTime from .utils import Utils -from ..helpers import getLogger, _merge_copy_dicts, uni_string, substituteRecursiveTags, TAG_CRE, MAX_TAG_REPLACE_COUNT +from ..helpers import getLogger, _merge_copy_dicts, \ + splitwords, substituteRecursiveTags, uni_string, TAG_CRE, MAX_TAG_REPLACE_COUNT # Gets the instance of the logger. logSys = getLogger(__name__) @@ -44,13 +48,14 @@ logSys = getLogger(__name__) # Create a lock for running system commands _cmd_lock = threading.Lock() -# Todo: make it configurable resp. automatically set, ex.: `[ -f /proc/net/if_inet6 ] && echo 'yes' || echo 'no'`: -allowed_ipv6 = True +# Specifies whether IPv6 subsystem is available: +allowed_ipv6 = DNSUtils.IPv6IsAllowed # capture groups from filter for map to ticket data: FCUSTAG_CRE = re.compile(r'<F-([A-Z0-9_\-]+)>'); # currently uppercase only -CONDITIONAL_FAM_RE = re.compile(r"^(\w+)\?(family)=") +COND_FAMILIES = ('inet4', 'inet6') +CONDITIONAL_FAM_RE = re.compile(r"^(\w+)\?(family)=(.*)$") # Special tags: DYN_REPL_TAGS = { @@ -173,7 +178,7 @@ class CallingMap(MutableMapping, object): def __len__(self): return len(self.data) - def copy(self): # pragma: no cover + def copy(self): return self.__class__(_merge_copy_dicts(self.data, self.storage)) @@ -215,6 +220,7 @@ class ActionBase(object): "start", "stop", "ban", + "reban", "unban", ) for method in required: @@ -248,6 +254,17 @@ class ActionBase(object): """ pass + def reban(self, aInfo): # pragma: no cover - abstract + """Executed when a ban occurs. + + Parameters + ---------- + aInfo : dict + Dictionary which includes information in relation to + the ban. + """ + return self.ban(aInfo) + @property def _prolongable(self): # pragma: no cover - abstract return False @@ -288,6 +305,7 @@ class CommandAction(ActionBase): ---------- actionban actioncheck + actionreban actionreload actionrepair actionstart @@ -308,6 +326,7 @@ class CommandAction(ActionBase): self.actionstart = '' ## Command executed when ticket gets banned. self.actionban = '' + self.actionreban = '' ## Command executed when ticket gets removed. self.actionunban = '' ## Command executed in order to check requirements. @@ -343,7 +362,7 @@ class CommandAction(ActionBase): if wrp == 'ignore': # ignore (filter) dynamic parameters return elif wrp == 'str2seconds': - value = str(MyTime.str2seconds(value)) + value = MyTime.str2seconds(value) # parameters changed - clear properties and substitution cache: self.__properties = None self.__substCache.clear() @@ -352,6 +371,8 @@ class CommandAction(ActionBase): # set: self.__dict__[name] = value + __setitem__ = __setattr__ + def __delattr__(self, name): if not name.startswith('_'): # parameters changed - clear properties and substitution cache: @@ -375,8 +396,8 @@ class CommandAction(ActionBase): self.__properties = dict( (key, getattr(self, key)) for key in dir(self) - if not key.startswith("_") and not callable(getattr(self, key))) - # + if not key.startswith("_") and not callable(getattr(self, key)) + ) return self.__properties @property @@ -384,10 +405,43 @@ class CommandAction(ActionBase): return self.__substCache def _getOperation(self, tag, family): - return self.replaceTag(tag, self._properties, - conditional=('family=' + family), cache=self.__substCache) + # replace operation tag (interpolate all values), be sure family is enclosed as conditional value + # (as lambda in addrepl so only if not overwritten in action): + cmd = self.replaceTag(tag, self._properties, + conditional=('family='+family if family else ''), + cache=self.__substCache) + if not family or '<' not in cmd: return cmd + # replace family as dynamic tags, important - don't cache, no recursion and auto-escape here: + cmd = self.replaceDynamicTags(cmd, {'family':family}) + return cmd + + def _operationExecuted(self, tag, family, *args): + """ Get, set or delete command of operation considering family. + """ + key = ('__eOpCmd',tag) + if not len(args): # get + if not callable(family): # pragma: no cover + return self.__substCache.get(key, {}).get(family) + # family as expression - use it to filter values: + return [v for f, v in self.__substCache.get(key, {}).iteritems() if family(f)] + cmd = args[0] + if cmd: # set: + try: + famd = self.__substCache[key] + except KeyError: + famd = self.__substCache[key] = {} + famd[family] = cmd + else: # delete (given family and all other with same command): + try: + famd = self.__substCache[key] + cmd = famd.pop(family) + for family, v in famd.items(): + if v == cmd: + del famd[family] + except KeyError: # pragma: no cover + pass - def _executeOperation(self, tag, operation, family=[]): + def _executeOperation(self, tag, operation, family=[], afterExec=None): """Executes the operation commands (like "actionstart", "actionstop", etc). Replace the tags in the action command with actions properties @@ -395,54 +449,106 @@ class CommandAction(ActionBase): """ # check valid tags in properties (raises ValueError if self recursion, etc.): res = True - try: - # common (resp. ipv4): - startCmd = None - if not family or 'inet4' in family: - startCmd = self._getOperation(tag, 'inet4') - if startCmd: - res &= self.executeCmd(startCmd, self.timeout) - # start ipv6 actions if available: - if allowed_ipv6 and (not family or 'inet6' in family): - startCmd6 = self._getOperation(tag, 'inet6') - if startCmd6 and startCmd6 != startCmd: - res &= self.executeCmd(startCmd6, self.timeout) - if not res: - raise RuntimeError("Error %s action %s/%s" % (operation, self._jail, self._name,)) - except ValueError as e: - raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, e)) - - COND_FAMILIES = ('inet4', 'inet6') + err = 'Script error' + if not family: # all started: + family = [famoper for (famoper,v) in self.__started.iteritems() if v] + for famoper in family: + try: + cmd = self._getOperation(tag, famoper) + ret = True + # avoid double execution of same command for both families: + if cmd and cmd not in self._operationExecuted(tag, lambda f: f != famoper): + realCmd = cmd + if self._jail: + # simulate action info with "empty" ticket: + aInfo = getattr(self._jail.actions, 'actionInfo', None) + if not aInfo: + aInfo = self._jail.actions._getActionInfo(None) + setattr(self._jail.actions, 'actionInfo', aInfo) + aInfo['time'] = MyTime.time() + aInfo['family'] = famoper + # replace dynamical tags, important - don't cache, no recursion and auto-escape here + realCmd = self.replaceDynamicTags(cmd, aInfo) + ret = self.executeCmd(realCmd, self.timeout) + res &= ret + if afterExec: afterExec(famoper, ret) + self._operationExecuted(tag, famoper, cmd if ret else None) + except ValueError as e: + res = False + err = e + if not res: + raise RuntimeError("Error %s action %s/%s: %r" % (operation, self._jail, self._name, err)) + return res + + @property + def _hasCondSection(self): + v = self._properties.get('__hasCondSection') + if v is not None: + return v + v = False + for n in self._properties: + if CONDITIONAL_FAM_RE.match(n): + v = True + break + self._properties['__hasCondSection'] = v + return v + + @property + def _families(self): + v = self._properties.get('__families') + if v: return v + v = self._properties.get('families') + if v and not isinstance(v, (list,set)): # pragma: no cover - still unused + v = splitwords(v) + elif self._hasCondSection: # all conditional families: + # todo: check it is needed at all # common (resp. ipv4) + ipv6 if allowed: + v = ['inet4', 'inet6'] if allowed_ipv6() else ['inet4'] + else: # all action tags seems to be the same + v = [''] + self._properties['__families'] = v + return v @property def _startOnDemand(self): """Checks the action depends on family (conditional)""" v = self._properties.get('actionstart_on_demand') - if v is None: - v = False - for n in self._properties: - if CONDITIONAL_FAM_RE.match(n): - v = True - break + if v is not None: + return v + # not set - auto-recognize (depending on conditional): + v = self._hasCondSection self._properties['actionstart_on_demand'] = v return v - def start(self, family=[]): + def start(self): + """Executes the "actionstart" command. + + Replace the tags in the action command with actions properties + and executes the resulting command. + """ + return self._start() + + def _start(self, family=None, forceStart=False): """Executes the "actionstart" command. Replace the tags in the action command with actions properties and executes the resulting command. """ - if not family: - # check the action depends on family (conditional): - if self._startOnDemand: + # check the action depends on family (conditional): + if self._startOnDemand: + if not forceStart: return True - elif self.__started.get(family): # pragma: no cover - normally unreachable + elif not forceStart and self.__started.get(family): # pragma: no cover - normally unreachable return True - return self._executeOperation('<actionstart>', 'starting', family=family) + family = [family] if family is not None else self._families + def _started(family, ret): + if ret: + self._operationExecuted('<actionstop>', family, None) + self.__started[family] = 1 + ret = self._executeOperation('<actionstart>', 'starting', family=family, afterExec=_started) + return ret - def ban(self, aInfo): - """Executes the "actionban" command. + def ban(self, aInfo, cmd='<actionban>'): + """Executes the given command ("actionban" or "actionreban"). Replaces the tags in the action command with actions properties and ban information, and executes the resulting command. @@ -454,21 +560,14 @@ class CommandAction(ActionBase): the ban. """ # if we should start the action on demand (conditional by family): + family = aInfo.get('family', '') if self._startOnDemand: - family = aInfo.get('family') if not self.__started.get(family): - self.start(family) - self.__started[family] = 1 - # mark also another families as "started" (-1), if they are equal - # (on demand, but the same for ipv4 and ipv6): - cmd = self._getOperation('<actionstart>', family) - for f in CommandAction.COND_FAMILIES: - if f != family and not self.__started.get(f): - if cmd == self._getOperation('<actionstart>', f): - self.__started[f] = -1 + self._start(family, forceStart=True) # ban: - if not self._processCmd('<actionban>', aInfo): + if not self._processCmd(cmd, aInfo): raise RuntimeError("Error banning %(ip)s" % aInfo) + self.__started[family] = self.__started.get(family, 0) | 3; # started and contains items @property def _prolongable(self): @@ -502,8 +601,25 @@ class CommandAction(ActionBase): Dictionary which includes information in relation to the ban. """ - if not self._processCmd('<actionunban>', aInfo): - raise RuntimeError("Error unbanning %(ip)s" % aInfo) + family = aInfo.get('family', '') + if self.__started.get(family, 0) & 2: # contains items + if not self._processCmd('<actionunban>', aInfo): + raise RuntimeError("Error unbanning %(ip)s" % aInfo) + + def reban(self, aInfo): + """Executes the "actionreban" command if available, otherwise simply repeat "actionban". + + Replaces the tags in the action command with actions properties + and ban information, and executes the resulting command. + + Parameters + ---------- + aInfo : dict + Dictionary which includes information in relation to + the ban. + """ + # re-ban: + return self.ban(aInfo, '<actionreban>' if self.actionreban else '<actionban>') def flush(self): """Executes the "actionflush" command. @@ -514,15 +630,15 @@ class CommandAction(ActionBase): Replaces the tags in the action command with actions properties and executes the resulting command. """ - family = [] - # collect started families, if started on demand (conditional): - if self._startOnDemand: - for f in CommandAction.COND_FAMILIES: - if self.__started.get(f) == 1: # only real started: - family.append(f) - # if no started (on demand) actions: - if not family: return True - return self._executeOperation('<actionflush>', 'flushing', family=family) + # collect started families, may be started on demand (conditional): + family = [f for (f,v) in self.__started.iteritems() if v & 3 == 3]; # started and contains items + # if nothing contains items: + if not family: return True + # flush: + def _afterFlush(family, ret): + if ret and self.__started.get(family): + self.__started[family] &= ~2; # no items anymore + return self._executeOperation('<actionflush>', 'flushing', family=family, afterExec=_afterFlush) def stop(self): """Executes the "actionstop" command. @@ -530,16 +646,30 @@ class CommandAction(ActionBase): Replaces the tags in the action command with actions properties and executes the resulting command. """ - family = [] + return self._stop() + + def _stop(self, family=None): + """Executes the "actionstop" command. + + Replaces the tags in the action command with actions properties + and executes the resulting command. + """ # collect started families, if started on demand (conditional): - if self._startOnDemand: - for f in CommandAction.COND_FAMILIES: - if self.__started.get(f) == 1: # only real started: - family.append(f) - self.__started[f] = 0 + if family is None: + family = [f for (f,v) in self.__started.iteritems() if v] # if no started (on demand) actions: if not family: return True - return self._executeOperation('<actionstop>', 'stopping', family=family) + self.__started = {} + else: + try: + self.__started[family] &= 0 + family = [family] + except KeyError: # pragma: no cover + return True + def _stopped(family, ret): + if ret: + self._operationExecuted('<actionstart>', family, None) + return self._executeOperation('<actionstop>', 'stopping', family=family, afterExec=_stopped) def reload(self, **kwargs): """Executes the "actionreload" command. @@ -554,6 +684,20 @@ class CommandAction(ActionBase): """ return self._executeOperation('<actionreload>', 'reloading') + def consistencyCheck(self, beforeRepair=None): + """Executes the invariant check with repair if expected (conditional). + """ + ret = True + # for each started family: + if self.actioncheck: + for (family, started) in self.__started.items(): + if started and not self._invariantCheck(family, beforeRepair): + # reset started flag and command of executed operation: + self.__started[family] = 0 + self._operationExecuted('<actionstart>', family, None) + ret &= False + return ret + ESCAPE_CRE = re.compile(r"""[\\#&;`|*?~<>^()\[\]{}$'"\n\r]""") @classmethod @@ -586,7 +730,7 @@ class CommandAction(ActionBase): return value @classmethod - def replaceTag(cls, query, aInfo, conditional='', cache=None): + def replaceTag(cls, query, aInfo, conditional='', addrepl=None, cache=None): """Replaces tags in `query` with property values. Parameters @@ -627,7 +771,8 @@ class CommandAction(ActionBase): pass # interpolation of dictionary: if subInfo is None: - subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags) + subInfo = substituteRecursiveTags(aInfo, conditional, ignore=cls._escapedTags, + addrepl=addrepl) # cache if possible: if csubkey is not None: cache[csubkey] = subInfo @@ -678,7 +823,7 @@ class CommandAction(ActionBase): ESCAPE_VN_CRE = re.compile(r"\W") @classmethod - def replaceDynamicTags(cls, realCmd, aInfo): + def replaceDynamicTags(cls, realCmd, aInfo, escapeVal=None): """Replaces dynamical tags in `query` with property values. **Important** @@ -703,16 +848,17 @@ class CommandAction(ActionBase): # array for escaped vars: varsDict = dict() - def escapeVal(tag, value): - # if the value should be escaped: - if cls.ESCAPE_CRE.search(value): - # That one needs to be escaped since its content is - # out of our control - tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) - varsDict[tag] = value # add variable - value = '$'+tag # replacement as variable - # replacement for tag: - return value + if not escapeVal: + def escapeVal(tag, value): + # if the value should be escaped: + if cls.ESCAPE_CRE.search(value): + # That one needs to be escaped since its content is + # out of our control + tag = 'f2bV_%s' % cls.ESCAPE_VN_CRE.sub('_', tag) + varsDict[tag] = value # add variable + value = '$'+tag # replacement as variable + # replacement for tag: + return value # additional replacement as calling map: ADD_REPL_TAGS_CM = CallingMap(ADD_REPL_TAGS) @@ -736,7 +882,7 @@ class CommandAction(ActionBase): tickData = aInfo.get("F-*") if not tickData: tickData = {} def substTag(m): - tag = mapTag2Opt(m.groups()[0]) + tag = mapTag2Opt(m.group(1)) try: value = uni_string(tickData[tag]) except KeyError: @@ -750,7 +896,58 @@ class CommandAction(ActionBase): realCmd = Utils.buildShellCmd(realCmd, varsDict) return realCmd - def _processCmd(self, cmd, aInfo=None, conditional=''): + @property + def banEpoch(self): + return getattr(self, '_banEpoch', 0) + def invalidateBanEpoch(self): + """Increments ban epoch of jail and this action, so already banned tickets would cause + a re-ban for all tickets with previous epoch.""" + if self._jail is not None: + self._banEpoch = self._jail.actions.banEpoch = self._jail.actions.banEpoch + 1 + else: + self._banEpoch = self.banEpoch + 1 + + def _invariantCheck(self, family=None, beforeRepair=None, forceStart=True): + """Executes a substituted `actioncheck` command. + """ + # for started action/family only (avoid check not started inet4 if inet6 gets broken): + if not forceStart and family is not None and family not in self.__started: + return 1 + checkCmd = self._getOperation('<actioncheck>', family) + if not checkCmd or self.executeCmd(checkCmd, self.timeout): + return 1 + # if don't need repair/restore - just return: + if beforeRepair and not beforeRepair(): + return -1 + self._logSys.error( + "Invariant check failed. Trying to restore a sane environment") + # increment ban epoch of jail and this action (allows re-ban on already banned): + self.invalidateBanEpoch() + # try to find repair command, if exists - exec it: + repairCmd = self._getOperation('<actionrepair>', family) + if repairCmd: + if not self.executeCmd(repairCmd, self.timeout): + self.__started[family] = 0 + self._logSys.critical("Unable to restore environment") + return 0 + self.__started[family] = 1 + else: + # no repair command, try to restart action... + # [WARNING] TODO: be sure all banactions get a repair command, because + # otherwise stop/start will theoretically remove all the bans, + # but the tickets are still in BanManager, so in case of new failures + # it will not be banned, because "already banned" will happen. + try: + self._stop(family) + except RuntimeError: # bypass error in stop (if start/check succeeded hereafter). + pass + self._start(family, forceStart=forceStart or not self._startOnDemand) + if self.__started.get(family) and not self.executeCmd(checkCmd, self.timeout): + self._logSys.critical("Unable to restore environment") + return 0 + return 1 + + def _processCmd(self, cmd, aInfo=None): """Executes a command with preliminary checks and substitutions. Before executing any commands, executes the "check" command first @@ -775,55 +972,43 @@ class CommandAction(ActionBase): return True # conditional corresponding family of the given ip: - if conditional == '': - conditional = 'family=inet4' - if allowed_ipv6: - try: - ip = aInfo["ip"] - if ip and asip(ip).isIPv6: - conditional = 'family=inet6' - except KeyError: - pass + try: + family = aInfo["family"] + except (KeyError, TypeError): + family = '' + + repcnt = 0 + while True: - checkCmd = self.replaceTag('<actioncheck>', self._properties, - conditional=conditional, cache=self.__substCache) - if checkCmd: - if not self.executeCmd(checkCmd, self.timeout): - self._logSys.error( - "Invariant check failed. Trying to restore a sane environment") - # try to find repair command, if exists - exec it: - repairCmd = self.replaceTag('<actionrepair>', self._properties, - conditional=conditional, cache=self.__substCache) - if repairCmd: - if not self.executeCmd(repairCmd, self.timeout): - self._logSys.critical("Unable to restore environment") + # got some error, do invariant check: + if repcnt and self.actioncheck: + # don't repair/restore if unban (no matter): + def _beforeRepair(): + if cmd == '<actionunban>' and not self._properties.get('actionrepair_on_unban'): + self._logSys.error("Invariant check failed. Unban is impossible.") return False - else: - # no repair command, try to restart action... - # [WARNING] TODO: be sure all banactions get a repair command, because - # otherwise stop/start will theoretically remove all the bans, - # but the tickets are still in BanManager, so in case of new failures - # it will not be banned, because "already banned" will happen. - try: - self.stop() - except RuntimeError: # bypass error in stop (if start/check succeeded hereafter). - pass - self.start() - if not self.executeCmd(checkCmd, self.timeout): - self._logSys.critical("Unable to restore environment") + return True + # check and repair if broken: + ret = self._invariantCheck(family, _beforeRepair, forceStart=(cmd != '<actionunban>')) + # if not sane (and not restored) return: + if ret != 1: return False - # Replace static fields - realCmd = self.replaceTag(cmd, self._properties, - conditional=conditional, cache=self.__substCache) + # Replace static fields + realCmd = self.replaceTag(cmd, self._properties, + conditional=('family='+family if family else ''), cache=self.__substCache) - # Replace dynamical tags, important - don't cache, no recursion and auto-escape here - if aInfo is not None: - realCmd = self.replaceDynamicTags(realCmd, aInfo) - else: - realCmd = cmd + # Replace dynamical tags, important - don't cache, no recursion and auto-escape here + if aInfo is not None: + realCmd = self.replaceDynamicTags(realCmd, aInfo) + else: + realCmd = cmd - return self.executeCmd(realCmd, self.timeout) + # try execute command: + ret = self.executeCmd(realCmd, self.timeout) + repcnt += 1 + if ret or repcnt > 1: + return ret @staticmethod def executeCmd(realCmd, timeout=60, **kwargs): @@ -848,7 +1033,7 @@ class CommandAction(ActionBase): RuntimeError If command execution times out. """ - if logSys.getEffectiveLevel() < logging.DEBUG: # pragma: no cover + if logSys.getEffectiveLevel() < logging.DEBUG: logSys.log(9, realCmd) if not realCmd: logSys.debug("Nothing to do") diff --git a/fail2ban/server/actions.py b/fail2ban/server/actions.py index 3d862275..fa045ab5 100644 --- a/fail2ban/server/actions.py +++ b/fail2ban/server/actions.py @@ -28,13 +28,14 @@ import logging import os import sys import time -from collections import Mapping try: - from collections import OrderedDict + from collections.abc import Mapping except ImportError: - OrderedDict = dict + from collections import Mapping +from collections import OrderedDict from .banmanager import BanManager, BanTicket +from .ipdns import IPAddr from .jailthread import JailThread from .action import ActionBase, CommandAction, CallingMap from .mytime import MyTime @@ -75,12 +76,18 @@ class Actions(JailThread, Mapping): """ def __init__(self, jail): - JailThread.__init__(self) + JailThread.__init__(self, name="f2b/a."+jail.name) ## The jail which contains this action. self._jail = jail self._actions = OrderedDict() ## The ban manager. - self.__banManager = BanManager() + self.banManager = BanManager() + self.banEpoch = 0 + self.__lastConsistencyCheckTM = 0 + ## Precedence of ban (over unban), so max number of tickets banned (to call an unban check): + self.banPrecedence = 10 + ## Max count of outdated tickets to unban per each __checkUnBan operation: + self.unbanMaxCount = self.banPrecedence * 2 @staticmethod def _load_python_module(pythonModule): @@ -156,8 +163,8 @@ class Actions(JailThread, Mapping): delacts = OrderedDict((name, action) for name, action in self._actions.iteritems() if name not in self._reload_actions) if len(delacts): - # unban all tickets using remove action only: - self.__flushBan(db=False, actions=delacts) + # unban all tickets using removed actions only: + self.__flushBan(db=False, actions=delacts, stop=True) # stop and remove it: self.stopActions(actions=delacts) delattr(self, '_reload_actions') @@ -193,7 +200,7 @@ class Actions(JailThread, Mapping): def setBanTime(self, value): value = MyTime.str2seconds(value) - self.__banManager.setBanTime(value) + self.banManager.setBanTime(value) logSys.info(" banTime: %s" % value) ## @@ -202,7 +209,38 @@ class Actions(JailThread, Mapping): # @return the time def getBanTime(self): - return self.__banManager.getBanTime() + return self.banManager.getBanTime() + + def getBanned(self, ids): + lst = self.banManager.getBanList() + if not ids: + return lst + if len(ids) == 1: + return 1 if ids[0] in lst else 0 + return map(lambda ip: 1 if ip in lst else 0, ids) + + def getBanList(self, withTime=False): + """Returns the list of banned IP addresses. + + Returns + ------- + list + The list of banned IP addresses. + """ + return self.banManager.getBanList(ordered=True, withTime=withTime) + + def addBannedIP(self, ip): + """Ban an IP or list of IPs.""" + unixTime = MyTime.time() + + if isinstance(ip, list): + # Multiple IPs: + tickets = (BanTicket(ip, unixTime) for ip in ip) + else: + # Single IP: + tickets = (BanTicket(ip, unixTime),) + + return self.__checkBan(tickets) def removeBannedIP(self, ip=None, db=True, ifexists=False): """Removes banned IP calling actions' unban method @@ -212,8 +250,8 @@ class Actions(JailThread, Mapping): Parameters ---------- - ip : str or IPAddr or None - The IP address to unban or all IPs if None + ip : list, str, IPAddr or None + The IP address (or multiple IPs as list) to unban or all IPs if None Raises ------ @@ -223,19 +261,42 @@ class Actions(JailThread, Mapping): # Unban all? if ip is None: return self.__flushBan(db) + # Multiple IPs: + if isinstance(ip, (list, tuple)): + missed = [] + cnt = 0 + for i in ip: + try: + cnt += self.removeBannedIP(i, db, ifexists) + except ValueError: + if not ifexists: + missed.append(i) + if missed: + raise ValueError("not banned: %r" % missed) + return cnt # Single IP: # Always delete ip from database (also if currently not banned) if db and self._jail.database is not None: self._jail.database.delBan(self._jail, ip) # Find the ticket with the IP. - ticket = self.__banManager.getTicketByID(ip) + ticket = self.banManager.getTicketByID(ip) if ticket is not None: # Unban the IP. self.__unBan(ticket) else: + # Multiple IPs by subnet or dns: + if not isinstance(ip, IPAddr): + ipa = IPAddr(ip) + if not ipa.isSingle: # subnet (mask/cidr) or raw (may be dns/hostname): + ips = filter(ipa.contains, self.banManager.getBanList()) + if ips: + return self.removeBannedIP(ips, db, ifexists) + # not found: + msg = "%s is not banned" % ip + logSys.log(logging.MSG, msg) if ifexists: return 0 - raise ValueError("%s is not banned" % ip) + raise ValueError(msg) return 1 @@ -244,9 +305,7 @@ class Actions(JailThread, Mapping): """ if actions is None: actions = self._actions - revactions = actions.items() - revactions.reverse() - for name, action in revactions: + for name, action in reversed(actions.items()): try: action.stop() except Exception as e: @@ -268,6 +327,7 @@ class Actions(JailThread, Mapping): bool True when the thread exits nicely. """ + cnt = 0 for name, action in self._actions.iteritems(): try: action.start() @@ -276,16 +336,34 @@ class Actions(JailThread, Mapping): self._jail.name, name, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) while self.active: - if self.idle: - logSys.debug("Actions: enter idle mode") - Utils.wait_for(lambda: not self.active or not self.idle, - lambda: False, self.sleeptime) - logSys.debug("Actions: leave idle mode") - continue - if not Utils.wait_for(lambda: not self.active or self.__checkBan(), self.sleeptime): - self.__checkUnBan() - - self.__flushBan() + try: + if self.idle: + logSys.debug("Actions: enter idle mode") + Utils.wait_for(lambda: not self.active or not self.idle, + lambda: False, self.sleeptime) + logSys.debug("Actions: leave idle mode") + continue + # wait for ban (stop if gets inactive, pending ban or unban): + bancnt = 0 + wt = min(self.sleeptime, self.banManager._nextUnbanTime - MyTime.time()) + logSys.log(5, "Actions: wait for pending tickets %s (default %s)", wt, self.sleeptime) + if Utils.wait_for(lambda: not self.active or self._jail.hasFailTickets, wt): + bancnt = self.__checkBan() + cnt += bancnt + # unban if nothing is banned not later than banned tickets >= banPrecedence + if not bancnt or cnt >= self.banPrecedence: + if self.active: + # let shrink the ban list faster + bancnt *= 2 + logSys.log(5, "Actions: check-unban %s, bancnt %s, max: %s", bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount, bancnt, self.unbanMaxCount) + self.__checkUnBan(bancnt if bancnt and bancnt < self.unbanMaxCount else self.unbanMaxCount) + cnt = 0 + except Exception as e: # pragma: no cover + logSys.error("[%s] unhandled error in actions thread: %s", + self._jail.name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + + self.__flushBan(stop=True) self.stopActions() return True @@ -314,7 +392,12 @@ class Actions(JailThread, Mapping): "ipfailures": lambda self: self._mi4ip(True).getAttempt(), "ipjailfailures": lambda self: self._mi4ip().getAttempt(), # raw ticket info: - "raw-ticket": lambda self: repr(self.__ticket) + "raw-ticket": lambda self: repr(self.__ticket), + # jail info: + "jail.banned": lambda self: self.__jail.actions.banManager.size(), + "jail.banned_total": lambda self: self.__jail.actions.banManager.getBanTotal(), + "jail.found": lambda self: self.__jail.filter.failManager.size(), + "jail.found_total": lambda self: self.__jail.filter.failManager.getFailTotal() } __slots__ = CallingMap.__slots__ + ('__ticket', '__jail', '__mi4ip') @@ -332,7 +415,7 @@ class Actions(JailThread, Mapping): def _getBanTime(self): btime = self.__ticket.getBanTime() if btime is None: btime = self.__jail.actions.getBanTime() - return btime + return int(btime) def _mi4ip(self, overalljails=False): """Gets bans merged once, a helper for lambda(s), prevents stop of executing action by any exception inside. @@ -377,15 +460,26 @@ class Actions(JailThread, Mapping): return mi[idx] if mi[idx] is not None else self.__ticket - def __getActionInfo(self, ticket): + def _getActionInfo(self, ticket): + if not ticket: + ticket = BanTicket("", MyTime.time()) aInfo = Actions.ActionInfo(ticket, self._jail) return aInfo + def __getFailTickets(self, count=100): + """Generator to get maximal count failure tickets from fail-manager.""" + cnt = 0 + while cnt < count: + ticket = self._jail.getFailTicket() + if not ticket: + break + yield ticket + cnt += 1 - def __checkBan(self): + def __checkBan(self, tickets=None): """Check for IP address to ban. - Look in the jail queue for FailTicket. If a ticket is available, + If tickets are not specified look in the jail queue for FailTicket. If a ticket is available, it executes the "ban" command and adds a ticket to the BanManager. Returns @@ -394,17 +488,17 @@ class Actions(JailThread, Mapping): True if an IP address get banned. """ cnt = 0 - while cnt < 100: - ticket = self._jail.getFailTicket() - if not ticket: - break + if not tickets: + tickets = self.__getFailTickets(self.banPrecedence) + rebanacts = None + for ticket in tickets: bTicket = BanTicket.wrap(ticket) - btime = ticket.getBanTime(self.__banManager.getBanTime()) - ip = bTicket.getIP() - aInfo = self.__getActionInfo(bTicket) + btime = ticket.getBanTime(self.banManager.getBanTime()) + ip = bTicket.getID() + aInfo = self._getActionInfo(bTicket) reason = {} - if self.__banManager.addBanTicket(bTicket, reason=reason): + if self.banManager.addBanTicket(bTicket, reason=reason): cnt += 1 # report ticket to observer, to check time should be increased and hereafter observer writes ban to database (asynchronous) if Observers.Main is not None and not bTicket.restored: @@ -413,7 +507,7 @@ class Actions(JailThread, Mapping): # do actions : for name, action in self._actions.iteritems(): try: - if ticket.restored and getattr(action, 'norestored', False): + if bTicket.restored and getattr(action, 'norestored', False): continue if not aInfo.immutable: aInfo.reset() action.ban(aInfo) @@ -425,6 +519,8 @@ class Actions(JailThread, Mapping): exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # after all actions are processed set banned flag: bTicket.banned = True + if self.banEpoch: # be sure tickets always have the same ban epoch (default 0): + bTicket.banEpoch = self.banEpoch else: if reason.get('expired', 0): logSys.info('[%s] Ignore %s, expired bantime', self._jail.name, ip) @@ -442,15 +538,67 @@ class Actions(JailThread, Mapping): else logging.NOTICE if diftm < 60 \ else logging.WARNING logSys.log(ll, "[%s] %s already banned", self._jail.name, ip) + # if long time after ban - do consistency check (something is wrong here): + if bTicket.banEpoch == self.banEpoch and diftm > 3: + # avoid too often checks: + if not rebanacts and MyTime.time() > self.__lastConsistencyCheckTM + 3: + self.__lastConsistencyCheckTM = MyTime.time() + for action in self._actions.itervalues(): + if hasattr(action, 'consistencyCheck'): + action.consistencyCheck() + # check epoch in order to reban it: + if bTicket.banEpoch < self.banEpoch: + if not rebanacts: rebanacts = dict( + (name, action) for name, action in self._actions.iteritems() + if action.banEpoch > bTicket.banEpoch) + cnt += self.__reBan(bTicket, actions=rebanacts) + else: # pragma: no cover - unexpected: ticket is not banned for some reasons - reban using all actions: + cnt += self.__reBan(bTicket) + # add ban to database moved to observer (should previously check not already banned + # and increase ticket time if "bantime.increment" set) if cnt: logSys.debug("Banned %s / %s, %s ticket(s) in %r", cnt, - self.__banManager.getBanTotal(), self.__banManager.size(), self._jail.name) + self.banManager.getBanTotal(), self.banManager.size(), self._jail.name) return cnt + def __reBan(self, ticket, actions=None, log=True): + """Repeat bans for the ticket. + + Executes the actions in order to reban the host given in the + ticket. + + Parameters + ---------- + ticket : Ticket + Ticket to reban + """ + actions = actions or self._actions + ip = ticket.getID() + aInfo = self._getActionInfo(ticket) + if log: + logSys.notice("[%s] Reban %s%s", self._jail.name, ip, (', action %r' % actions.keys()[0] if len(actions) == 1 else '')) + for name, action in actions.iteritems(): + try: + logSys.debug("[%s] action %r: reban %s", self._jail.name, name, ip) + if not aInfo.immutable: aInfo.reset() + action.reban(aInfo) + except Exception as e: + logSys.error( + "Failed to execute reban jail '%s' action '%s' " + "info '%r': %s", + self._jail.name, name, aInfo, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + return 0 + # after all actions are processed set banned flag: + ticket.banned = True + if self.banEpoch: # be sure tickets always have the same ban epoch (default 0): + ticket.banEpoch = self.banEpoch + return 1 + def _prolongBan(self, ticket): # prevent to prolong ticket that was removed in-between, # if it in ban list - ban time already prolonged (and it stays there): - if not self.__banManager._inBanList(ticket): return + if not self.banManager._inBanList(ticket): return # do actions : aInfo = None for name, action in self._actions.iteritems(): @@ -460,7 +608,7 @@ class Actions(JailThread, Mapping): if not action._prolongable: continue if aInfo is None: - aInfo = self.__getActionInfo(ticket) + aInfo = self._getActionInfo(ticket) if not aInfo.immutable: aInfo.reset() action.prolong(aInfo) except Exception as e: @@ -470,21 +618,21 @@ class Actions(JailThread, Mapping): self._jail.name, name, aInfo, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - def __checkUnBan(self): + def __checkUnBan(self, maxCount=None): """Check for IP address to unban. Unban IP addresses which are outdated. """ - lst = self.__banManager.unBanList(MyTime.time()) + lst = self.banManager.unBanList(MyTime.time(), maxCount) for ticket in lst: self.__unBan(ticket) cnt = len(lst) if cnt: logSys.debug("Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt - def __flushBan(self, db=False, actions=None): + def __flushBan(self, db=False, actions=None, stop=False): """Flush the ban list. Unban all IP address which are still in the banning list. @@ -495,31 +643,47 @@ class Actions(JailThread, Mapping): log = True if actions is None: logSys.debug(" Flush ban list") - lst = self.__banManager.flushBanList() + lst = self.banManager.flushBanList() else: log = False # don't log "[jail] Unban ..." if removing actions only. - lst = iter(self.__banManager) + lst = iter(self.banManager) cnt = 0 # first we'll execute flush for actions supporting this operation: unbactions = {} for name, action in (actions if actions is not None else self._actions).iteritems(): - if hasattr(action, 'flush') and action.actionflush: - logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) - action.flush() - else: - unbactions[name] = action + try: + if hasattr(action, 'flush') and (not isinstance(action, CommandAction) or action.actionflush): + logSys.notice("[%s] Flush ticket(s) with %s", self._jail.name, name) + if action.flush(): + continue + except Exception as e: + logSys.error("Failed to flush bans in jail '%s' action '%s': %s", + self._jail.name, name, e, + exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) + logSys.info("No flush occurred, do consistency check") + if hasattr(action, 'consistencyCheck'): + def _beforeRepair(): + if stop and not getattr(action, 'actionrepair_on_unban', None): # don't need repair on stop + logSys.error("Invariant check failed. Flush is impossible.") + return False + return True + action.consistencyCheck(_beforeRepair) + continue + # fallback to single unbans: + logSys.debug(" Unban tickets each individualy") + unbactions[name] = action actions = unbactions # flush the database also: if db and self._jail.database is not None: logSys.debug(" Flush jail in database") self._jail.database.delBan(self._jail) - # unban each ticket with non-flasheable actions: + # unban each ticket with non-flusheable actions: for ticket in lst: # unban ip: self.__unBan(ticket, actions=actions, log=log) cnt += 1 logSys.debug(" Unbanned %s, %s ticket(s) in %r", - cnt, self.__banManager.size(), self._jail.name) + cnt, self.banManager.size(), self._jail.name) return cnt def __unBan(self, ticket, actions=None, log=True): @@ -537,14 +701,12 @@ class Actions(JailThread, Mapping): unbactions = self._actions else: unbactions = actions - ip = ticket.getIP() - aInfo = self.__getActionInfo(ticket) + ip = ticket.getID() + aInfo = self._getActionInfo(ticket) if log: - logSys.notice("[%s] Unban %s", self._jail.name, aInfo["ip"]) + logSys.notice("[%s] Unban %s", self._jail.name, ip) for name, action in unbactions.iteritems(): try: - if ticket.restored and getattr(action, 'norestored', False): - continue logSys.debug("[%s] action %r: unban %s", self._jail.name, name, ip) if not aInfo.immutable: aInfo.reset() action.unban(aInfo) @@ -559,17 +721,23 @@ class Actions(JailThread, Mapping): """Status of current and total ban counts and current banned IP list. """ # TODO: Allow this list to be printed as 'status' output - supported_flavors = ["basic", "cymru"] + supported_flavors = ["short", "basic", "cymru"] if flavor is None or flavor not in supported_flavors: logSys.warning("Unsupported extended jail status flavor %r. Supported: %s" % (flavor, supported_flavors)) # Always print this information (basic) - ret = [("Currently banned", self.__banManager.size()), - ("Total banned", self.__banManager.getBanTotal()), - ("Banned IP list", self.__banManager.getBanList())] + if flavor != "short": + banned = self.banManager.getBanList() + cnt = len(banned) + else: + cnt = self.banManager.size() + ret = [("Currently banned", cnt), + ("Total banned", self.banManager.getBanTotal())] + if flavor != "short": + ret += [("Banned IP list", banned)] if flavor == "cymru": - cymru_info = self.__banManager.getBanListExtendedCymruInfo() + cymru_info = self.banManager.getBanListExtendedCymruInfo() ret += \ - [("Banned ASN list", self.__banManager.geBanListExtendedASN(cymru_info)), - ("Banned Country list", self.__banManager.geBanListExtendedCountry(cymru_info)), - ("Banned RIR list", self.__banManager.geBanListExtendedRIR(cymru_info))] + [("Banned ASN list", self.banManager.geBanListExtendedASN(cymru_info)), + ("Banned Country list", self.banManager.geBanListExtendedCountry(cymru_info)), + ("Banned RIR list", self.banManager.geBanListExtendedRIR(cymru_info))] return ret diff --git a/fail2ban/server/banmanager.py b/fail2ban/server/banmanager.py index 1340fb52..9168d5b8 100644 --- a/fail2ban/server/banmanager.py +++ b/fail2ban/server/banmanager.py @@ -57,7 +57,7 @@ class BanManager: ## Total number of banned IP address self.__banTotal = 0 ## The time for next unban process (for performance and load reasons): - self.__nextUnbanTime = BanTicket.MAX_TIME + self._nextUnbanTime = BanTicket.MAX_TIME ## # Set the ban time. @@ -66,7 +66,6 @@ class BanManager: # @param value the time def setBanTime(self, value): - with self.__lock: self.__banTime = int(value) ## @@ -76,7 +75,6 @@ class BanManager: # @return the time def getBanTime(self): - with self.__lock: return self.__banTime ## @@ -85,7 +83,6 @@ class BanManager: # @param value total number def setBanTotal(self, value): - with self.__lock: self.__banTotal = value ## @@ -94,7 +91,6 @@ class BanManager: # @return the total number def getBanTotal(self): - with self.__lock: return self.__banTotal ## @@ -102,9 +98,22 @@ class BanManager: # # @return IP list - def getBanList(self): + def getBanList(self, ordered=False, withTime=False): + if not ordered: + return list(self.__banList.keys()) with self.__lock: - return self.__banList.keys() + lst = [] + for ticket in self.__banList.itervalues(): + eob = ticket.getEndOfBanTime(self.__banTime) + lst.append((ticket,eob)) + lst.sort(key=lambda t: t[1]) + t2s = MyTime.time2str + if withTime: + return ['%s \t%s + %d = %s' % ( + t[0].getID(), + t2s(t[0].getTime()), t[0].getBanTime(self.__banTime), t2s(t[1]) + ) for t in lst] + return [t[0].getID() for t in lst] ## # Returns a iterator to ban list (used in reload, so idle). @@ -112,8 +121,8 @@ class BanManager: # @return ban list iterator def __iter__(self): - with self.__lock: - return self.__banList.itervalues() + # ensure iterator is safe - traverse over the list in snapshot created within lock (GIL): + return iter(list(self.__banList.values())) ## # Returns normalized value @@ -284,8 +293,8 @@ class BanManager: self.__banTotal += 1 ticket.incrBanCount() # correct next unban time: - if self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if self._nextUnbanTime > eob: + self._nextUnbanTime = eob return True ## @@ -314,27 +323,28 @@ class BanManager: # @param time the time # @return the list of ticket to unban - def unBanList(self, time): + def unBanList(self, time, maxCount=0x7fffffff): with self.__lock: - # Permanent banning - if self.__banTime < 0: - return list() - # Check next unban time: - if self.__nextUnbanTime > time: + nextUnbanTime = self._nextUnbanTime + if nextUnbanTime > time: return list() # Gets the list of ticket to remove (thereby correct next unban time). unBanList = {} - self.__nextUnbanTime = BanTicket.MAX_TIME + nextUnbanTime = BanTicket.MAX_TIME for fid,ticket in self.__banList.iteritems(): # current time greater as end of ban - timed out: eob = ticket.getEndOfBanTime(self.__banTime) if time > eob: unBanList[fid] = ticket - elif self.__nextUnbanTime > eob: - self.__nextUnbanTime = eob + if len(unBanList) >= maxCount: # stop search cycle, so reset back the next check time + nextUnbanTime = self._nextUnbanTime + break + elif nextUnbanTime > eob: + nextUnbanTime = eob + self._nextUnbanTime = nextUnbanTime # Removes tickets. if len(unBanList): if len(unBanList) / 2.0 <= len(self.__banList) / 3.0: diff --git a/fail2ban/server/database.py b/fail2ban/server/database.py index ac01382e..877cbb93 100644 --- a/fail2ban/server/database.py +++ b/fail2ban/server/database.py @@ -104,7 +104,11 @@ def commitandrollback(f): def wrapper(self, *args, **kwargs): with self._lock: # Threading lock with self._db: # Auto commit and rollback on exception - return f(self, self._db.cursor(), *args, **kwargs) + cur = self._db.cursor() + try: + return f(self, cur, *args, **kwargs) + finally: + cur.close() return wrapper @@ -191,7 +195,7 @@ class Fail2BanDb(object): def __init__(self, filename, purgeAge=24*60*60, outDatedFactor=3): - self.maxEntries = 50 + self.maxMatches = 10 self._lock = RLock() self._dbFilename = filename self._purgeAge = purgeAge @@ -253,7 +257,7 @@ class Fail2BanDb(object): self.repairDB() else: version = cur.fetchone()[0] - if version < Fail2BanDb.__version__: + if version != Fail2BanDb.__version__: newversion = self.updateDb(version) if newversion == Fail2BanDb.__version__: logSys.warning( "Database updated from '%r' to '%r'", @@ -301,9 +305,11 @@ class Fail2BanDb(object): try: # backup logSys.info("Trying to repair database %s", self._dbFilename) - shutil.move(self._dbFilename, self._dbBackupFilename) - logSys.info(" Database backup created: %s", self._dbBackupFilename) - + if not os.path.isfile(self._dbBackupFilename): + shutil.move(self._dbFilename, self._dbBackupFilename) + logSys.info(" Database backup created: %s", self._dbBackupFilename) + elif os.path.isfile(self._dbFilename): + os.remove(self._dbFilename) # first try to repair using dump/restore in order Utils.executeCmd((r"""f2b_db=$0; f2b_dbbk=$1; sqlite3 "$f2b_dbbk" ".dump" | sqlite3 "$f2b_db" """, self._dbFilename, self._dbBackupFilename)) @@ -415,7 +421,7 @@ class Fail2BanDb(object): logSys.error("Failed to upgrade database '%s': %s", self._dbFilename, e.args[0], exc_info=logSys.getEffectiveLevel() <= 10) - raise + self.repairDB() @commitandrollback def addJail(self, cur, jail): @@ -489,22 +495,24 @@ class Fail2BanDb(object): If log was already present in database, value of last position in the log file; else `None` """ + return self._addLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) + + def _addLog(self, cur, jail, name, pos=0, md5=None): lastLinePos = None cur.execute( "SELECT firstlinemd5, lastfilepos FROM logs " "WHERE jail=? AND path=?", - (jail.name, container.getFileName())) + (jail.name, name)) try: firstLineMD5, lastLinePos = cur.fetchone() except TypeError: - firstLineMD5 = False + firstLineMD5 = None - cur.execute( - "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " - "VALUES(?, ?, ?, ?)", - (jail.name, container.getFileName(), - container.getHash(), container.getPos())) - if container.getHash() != firstLineMD5: + if firstLineMD5 is None and (pos or md5 is not None): + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + if md5 is not None and md5 != firstLineMD5: lastLinePos = None return lastLinePos @@ -533,7 +541,7 @@ class Fail2BanDb(object): return set(row[0] for row in cur.fetchmany()) @commitandrollback - def updateLog(self, cur, *args, **kwargs): + def updateLog(self, cur, jail, container): """Updates hash and last position in log file. Parameters @@ -543,14 +551,48 @@ class Fail2BanDb(object): container : FileContainer File container of the log file being updated. """ - self._updateLog(cur, *args, **kwargs) + self._updateLog(cur, jail, container.getFileName(), container.getPos(), container.getHash()) - def _updateLog(self, cur, jail, container): + def _updateLog(self, cur, jail, name, pos, md5): cur.execute( "UPDATE logs SET firstlinemd5=?, lastfilepos=? " - "WHERE jail=? AND path=?", - (container.getHash(), container.getPos(), - jail.name, container.getFileName())) + "WHERE jail=? AND path=?", (md5, pos, jail.name, name)) + # be sure it is set (if not available): + if not cur.rowcount: + cur.execute( + "INSERT OR REPLACE INTO logs(jail, path, firstlinemd5, lastfilepos) " + "VALUES(?, ?, ?, ?)", (jail.name, name, md5, pos)) + + @commitandrollback + def getJournalPos(self, cur, jail, name, time=0, iso=None): + """Get journal position from database. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + + Returns + ------- + int (or float) + Last position (as time) if it was already present in database; else `None` + """ + return self._addLog(cur, jail, name, time, iso); # no hash, just time as iso + + @commitandrollback + def updateJournal(self, cur, jail, name, time, iso): + """Updates last position (as time) of journal. + + Parameters + ---------- + jail : Jail + Jail of which the journal belongs to. + name, time, iso : + Journal name (typically systemd-journal) and last known time. + """ + self._updateLog(cur, jail, name, time, iso); # no hash, just time as iso @commitandrollback def addBan(self, cur, jail, ticket): @@ -563,7 +605,7 @@ class Fail2BanDb(object): ticket : BanTicket Ticket of the ban to be added. """ - ip = str(ticket.getIP()) + ip = str(ticket.getID()) try: del self._bansMergedCache[(ip, jail)] except KeyError: @@ -575,8 +617,13 @@ class Fail2BanDb(object): #TODO: Implement data parts once arbitrary match keys completed data = ticket.getData() matches = data.get('matches') - if matches and len(matches) > self.maxEntries: - data['matches'] = matches[-self.maxEntries:] + if self.maxMatches: + if matches and len(matches) > self.maxMatches: + data = data.copy() + data['matches'] = matches[-self.maxMatches:] + elif matches: + data = data.copy() + del data['matches'] cur.execute( "INSERT INTO bans(jail, ip, timeofban, bantime, bancount, data) VALUES(?, ?, ?, ?, ?, ?)", (jail.name, ip, int(round(ticket.getTime())), ticket.getBanTime(jail.actions.getBanTime()), ticket.getBanCount(), @@ -710,7 +757,7 @@ class Fail2BanDb(object): tickdata = {} m = data.get('matches', []) # pre-insert "maxadd" enries (because tickets are ordered desc by time) - maxadd = self.maxEntries - len(matches) + maxadd = self.maxMatches - len(matches) if maxadd > 0: if len(m) <= maxadd: matches = m + matches @@ -748,8 +795,8 @@ class Fail2BanDb(object): queryArgs.append(fromtime) if overalljails or jail is None: query += " GROUP BY ip ORDER BY timeofban DESC LIMIT 1" - cur = self._db.cursor() - return cur.execute(query, queryArgs) + # repack iterator as long as in lock: + return list(cur.execute(query, queryArgs)) def _getCurrentBans(self, cur, jail = None, ip = None, forbantime=None, fromtime=None): queryArgs = [] @@ -768,12 +815,12 @@ class Fail2BanDb(object): queryArgs.append(fromtime - forbantime) if ip is None: query += " GROUP BY ip ORDER BY ip, timeofban DESC" - cur = self._db.cursor() + else: + query += " ORDER BY timeofban DESC LIMIT 1" return cur.execute(query, queryArgs) - @commitandrollback - def getCurrentBans(self, cur, jail=None, ip=None, forbantime=None, fromtime=None, - correctBanTime=True + def getCurrentBans(self, jail=None, ip=None, forbantime=None, fromtime=None, + correctBanTime=True, maxmatches=None ): """Reads tickets (with merged info) currently affected from ban from the database. @@ -784,49 +831,65 @@ class Fail2BanDb(object): (and therefore endOfBan) of the ticket (normally it is ban-time of jail as maximum) for all tickets with ban-time greater (or persistent). """ - if fromtime is None: - fromtime = MyTime.time() - tickets = [] - ticket = None - if correctBanTime is True: - correctBanTime = jail.getMaxBanTime() if jail is not None else None - # don't change if persistent allowed: - if correctBanTime == -1: correctBanTime = None - - for ticket in self._getCurrentBans(cur, jail=jail, ip=ip, - forbantime=forbantime, fromtime=fromtime - ): - # can produce unpack error (database may return sporadical wrong-empty row): - try: - banip, timeofban, bantime, bancount, data = ticket - # additionally check for empty values: - if banip is None or banip == "": # pragma: no cover - raise ValueError('unexpected value %r' % (banip,)) - # if bantime unknown (after upgrade-db from earlier version), just use min known ban-time: - if bantime == -2: # todo: remove it in future version - bantime = jail.actions.getBanTime() if jail is not None else ( - correctBanTime if correctBanTime else 600) - elif correctBanTime and correctBanTime >= 0: - # if persistent ban (or greater as max), use current max-bantime of the jail: - if bantime == -1 or bantime > correctBanTime: - bantime = correctBanTime - # after correction check the end of ban again: - if bantime != -1 and timeofban + bantime <= fromtime: - # not persistent and too old - ignore it: - logSys.debug("ignore ticket (with new max ban-time %r): too old %r <= %r, ticket: %r", - bantime, timeofban + bantime, fromtime, ticket) + cur = self._db.cursor() + try: + if fromtime is None: + fromtime = MyTime.time() + tickets = [] + ticket = None + if correctBanTime is True: + correctBanTime = jail.getMaxBanTime() if jail is not None else None + # don't change if persistent allowed: + if correctBanTime == -1: correctBanTime = None + + with self._lock: + bans = self._getCurrentBans(cur, jail=jail, ip=ip, + forbantime=forbantime, fromtime=fromtime + ) + for ticket in bans: + # can produce unpack error (database may return sporadical wrong-empty row): + try: + banip, timeofban, bantime, bancount, data = ticket + # additionally check for empty values: + if banip is None or banip == "": # pragma: no cover + raise ValueError('unexpected value %r' % (banip,)) + # if bantime unknown (after upgrade-db from earlier version), just use min known ban-time: + if bantime == -2: # todo: remove it in future version + bantime = jail.actions.getBanTime() if jail is not None else ( + correctBanTime if correctBanTime else 600) + elif correctBanTime and correctBanTime >= 0: + # if persistent ban (or greater as max), use current max-bantime of the jail: + if bantime == -1 or bantime > correctBanTime: + bantime = correctBanTime + # after correction check the end of ban again: + if bantime != -1 and timeofban + bantime <= fromtime: + # not persistent and too old - ignore it: + logSys.debug("ignore ticket (with new max ban-time %r): too old %r <= %r, ticket: %r", + bantime, timeofban + bantime, fromtime, ticket) + continue + except ValueError as e: # pragma: no cover + logSys.debug("get current bans: ignore row %r - %s", ticket, e) continue - except ValueError as e: # pragma: no cover - logSys.debug("get current bans: ignore row %r - %s", ticket, e) - continue - # logSys.debug('restore ticket %r, %r, %r', banip, timeofban, data) - ticket = FailTicket(banip, timeofban, data=data) - # logSys.debug('restored ticket: %r', ticket) - ticket.setBanTime(bantime) - ticket.setBanCount(bancount) - tickets.append(ticket) - - return tickets if ip is None else ticket + # logSys.debug('restore ticket %r, %r, %r', banip, timeofban, data) + ticket = FailTicket(banip, timeofban, data=data) + # filter matches if expected (current count > as maxmatches specified): + if maxmatches is None: + maxmatches = self.maxMatches + if maxmatches: + matches = ticket.getMatches() + if matches and len(matches) > maxmatches: + ticket.setMatches(matches[-maxmatches:]) + else: + ticket.setMatches(None) + # logSys.debug('restored ticket: %r', ticket) + ticket.setBanTime(bantime) + ticket.setBanCount(bancount) + if ip is not None: return ticket + tickets.append(ticket) + finally: + cur.close() + + return tickets def _cleanjails(self, cur): """Remove empty jails jails and log files from database. diff --git a/fail2ban/server/datedetector.py b/fail2ban/server/datedetector.py index 2e85b940..b90e1b26 100644 --- a/fail2ban/server/datedetector.py +++ b/fail2ban/server/datedetector.py @@ -35,7 +35,7 @@ from ..helpers import getLogger # Gets the instance of the logger. logSys = getLogger(__name__) -logLevel = 6 +logLevel = 5 RE_DATE_PREMATCH = re.compile(r"(?<!\\)\{DATE\}", re.IGNORECASE) DD_patternCache = Utils.Cache(maxCount=1000, maxTime=60*60) @@ -128,52 +128,52 @@ class DateDetectorCache(object): # 2005-01-23T21:59:59.981746, 2005-01-23 21:59:59, 2005-01-23 8:59:59 # simple date: 2005/01/23 21:59:59 # custom for syslog-ng 2006.12.21 06:43:20 - "%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T| ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?", + r"%ExY(?P<_sep>[-/.])%m(?P=_sep)%d(?:T| ?)%H:%M:%S(?:[.,]%f)?(?:\s*%z)?", # asctime with optional day, subsecond and/or year: # Sun Jan 23 21:59:59.011 2005 - "(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # asctime with optional day, subsecond and/or year coming after day # http://bugs.debian.org/798923 # Sun Jan 23 2005 21:59:59.011 - "(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?", + r"(?:%a )?%b %d %ExY %k:%M:%S(?:\.%f)?", # simple date too (from x11vnc): 23/01/2005 21:59:59 # and with optional year given by 2 digits: 23/01/05 21:59:59 # (See http://bugs.debian.org/537610) # 17-07-2008 17:23:25 - "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S", + r"%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S", # Apache format optional time zone: # [31/Oct/2006:09:22:55 -0000] # 26-Jul-2007 15:20:52 # named 26-Jul-2007 15:20:52.252 # roundcube 26-Jul-2007 15:20:52 +0200 - "%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?", + r"%d(?P<_sep>[-/])%b(?P=_sep)%ExY[ :]?%H:%M:%S(?:\.%f)?(?: %z)?", # CPanel 05/20/2008:01:57:39 - "%m/%d/%ExY:%H:%M:%S", + r"%m/%d/%ExY:%H:%M:%S", # 01-27-2012 16:22:44.252 # subseconds explicit to avoid possible %m<->%d confusion # with previous ("%d-%m-%ExY %k:%M:%S" by "%d(?P<_sep>[-/])%m(?P=_sep)(?:%ExY|%Exy) %k:%M:%S") - "%m-%d-%ExY %k:%M:%S(?:\.%f)?", + r"%m-%d-%ExY %k:%M:%S(?:\.%f)?", # Epoch - "EPOCH", + r"EPOCH", # Only time information in the log - "{^LN-BEG}%H:%M:%S", + r"{^LN-BEG}%H:%M:%S", # <09/16/08@05:03:30> - "^<%m/%d/%Exy@%H:%M:%S>", + r"^<%m/%d/%Exy@%H:%M:%S>", # MySQL: 130322 11:46:11 - "%Exy%Exm%Exd ?%H:%M:%S", + r"%Exy%Exm%Exd ?%H:%M:%S", # Apache Tomcat - "%b %d, %ExY %I:%M:%S %p", + r"%b %d, %ExY %I:%M:%S %p", # ASSP: Apr-27-13 02:33:06 - "^%b-%d-%Exy %k:%M:%S", + r"^%b-%d-%Exy %k:%M:%S", # 20050123T215959, 20050123 215959, 20050123 85959 - "%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?", + r"%ExY%Exm%Exd(?:T| ?)%ExH%ExM%ExS(?:[.,]%f)?(?:\s*%z)?", # prefixed with optional named time zone (monit): # PDT Apr 16 21:05:29 - "(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%Z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # +00:00 Jan 23 21:59:59.011 2005 - "(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", + r"(?:%z )?(?:%a )?%b %d %k:%M:%S(?:\.%f)?(?: %ExY)?", # TAI64N - "TAI64N", + r"TAI64N", ] @property @@ -282,6 +282,8 @@ class DateDetector(object): elif "{DATE}" in key: self.addDefaultTemplate(preMatch=pattern, allDefaults=False) return + elif key == "{NONE}": + template = _getPatternTemplate('{UNB}^', key) else: template = _getPatternTemplate(pattern, key) @@ -337,65 +339,76 @@ class DateDetector(object): # if no templates specified - default templates should be used: if not len(self.__templates): self.addDefaultTemplate() - logSys.log(logLevel-1, "try to match time for line: %.120s", line) - match = None + log = logSys.log if logSys.getEffectiveLevel() <= logLevel else lambda *args: None + log(logLevel-1, "try to match time for line: %.120s", line) + # first try to use last template with same start/end position: + match = None + found = None, 0x7fffffff, 0x7fffffff, -1 ignoreBySearch = 0x7fffffff i = self.__lastTemplIdx if i < len(self.__templates): ddtempl = self.__templates[i] template = ddtempl.template if template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END): - if logSys.getEffectiveLevel() <= logLevel-1: # pragma: no cover - very-heavy debug - logSys.log(logLevel-1, " try to match last anchored template #%02i ...", i) + log(logLevel-1, " try to match last anchored template #%02i ...", i) match = template.matchDate(line) ignoreBySearch = i else: distance, endpos = self.__lastPos[0], self.__lastEndPos[0] - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", - i, distance, endpos, - line[distance-1:distance], self.__lastPos[1], - line[distance:endpos], - line[endpos:endpos+1], self.__lastEndPos[1]) - # check same boundaries left/right, otherwise possible collision/pattern switch: - if (line[distance-1:distance] == self.__lastPos[1] and - line[endpos:endpos+1] == self.__lastEndPos[1] - ): + log(logLevel-1, " try to match last template #%02i (from %r to %r): ...%r==%r %s %r==%r...", + i, distance, endpos, + line[distance-1:distance], self.__lastPos[1], + line[distance:endpos], + line[endpos:endpos+1], self.__lastEndPos[2]) + # check same boundaries left/right, outside fully equal, inside only if not alnum (e. g. bound RE + # with space or some special char), otherwise possible collision/pattern switch: + if (( + line[distance-1:distance] == self.__lastPos[1] or + (line[distance:distance+1] == self.__lastPos[2] and not self.__lastPos[2].isalnum()) + ) and ( + line[endpos:endpos+1] == self.__lastEndPos[2] or + (line[endpos-1:endpos] == self.__lastEndPos[1] and not self.__lastEndPos[1].isalnum()) + )): + # search in line part only: + log(logLevel-1, " boundaries are correct, search in part %r", line[distance:endpos]) match = template.matchDate(line, distance, endpos) + else: + log(logLevel-1, " boundaries show conflict, try whole search") + match = template.matchDate(line) + ignoreBySearch = i if match: distance = match.start() endpos = match.end() # if different position, possible collision/pattern switch: if ( + len(self.__templates) == 1 or # single template: template.flags & (DateTemplate.LINE_BEGIN|DateTemplate.LINE_END) or (distance == self.__lastPos[0] and endpos == self.__lastEndPos[0]) ): - logSys.log(logLevel, " matched last time template #%02i", i) + log(logLevel, " matched last time template #%02i", i) else: - logSys.log(logLevel, " ** last pattern collision - pattern change, search ...") + log(logLevel, " ** last pattern collision - pattern change, reserve & search ...") + found = match, distance, endpos, i; # save current best alternative match = None else: - logSys.log(logLevel, " ** last pattern not found - pattern change, search ...") + log(logLevel, " ** last pattern not found - pattern change, search ...") # search template and better match: if not match: - logSys.log(logLevel, " search template (%i) ...", len(self.__templates)) - found = None, 0x7fffffff, 0x7fffffff, -1 + log(logLevel, " search template (%i) ...", len(self.__templates)) i = 0 for ddtempl in self.__templates: - if logSys.getEffectiveLevel() <= logLevel-1: - logSys.log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) if i == ignoreBySearch: i += 1 continue + log(logLevel-1, " try template #%02i: %s", i, ddtempl.name) template = ddtempl.template match = template.matchDate(line) if match: distance = match.start() endpos = match.end() - if logSys.getEffectiveLevel() <= logLevel: - logSys.log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", - i, distance, ddtempl.distance, self.__lastPos[0], template.name) + log(logLevel, " matched time template #%02i (at %r <= %r, %r) %s", + i, distance, ddtempl.distance, self.__lastPos[0], template.name) ## last (or single) template - fast stop: if i+1 >= len(self.__templates): break @@ -408,7 +421,7 @@ class DateDetector(object): ## [grave] if distance changed, possible date-match was found somewhere ## in body of message, so save this template, and search further: if distance > ddtempl.distance or distance > self.__lastPos[0]: - logSys.log(logLevel, " ** distance collision - pattern change, reserve") + log(logLevel, " ** distance collision - pattern change, reserve") ## shortest of both: if distance < found[1]: found = match, distance, endpos, i @@ -422,7 +435,7 @@ class DateDetector(object): # check other template was found (use this one with shortest distance): if not match and found[0]: match, distance, endpos, i = found - logSys.log(logLevel, " use best time template #%02i", i) + log(logLevel, " use best time template #%02i", i) ddtempl = self.__templates[i] template = ddtempl.template # we've winner, incr hits, set distance, usage, reorder, etc: @@ -432,8 +445,8 @@ class DateDetector(object): ddtempl.distance = distance if self.__firstUnused == i: self.__firstUnused += 1 - self.__lastPos = distance, line[distance-1:distance] - self.__lastEndPos = endpos, line[endpos:endpos+1] + self.__lastPos = distance, line[distance-1:distance], line[distance] + self.__lastEndPos = endpos, line[endpos-1], line[endpos:endpos+1] # if not first - try to reorder current template (bubble up), they will be not sorted anymore: if i and i != self.__lastTemplIdx: i = self._reorderTemplate(i) @@ -442,7 +455,7 @@ class DateDetector(object): return (match, template) # not found: - logSys.log(logLevel, " no template.") + log(logLevel, " no template.") return (None, None) @property diff --git a/fail2ban/server/datetemplate.py b/fail2ban/server/datetemplate.py index e032c2b0..518805bb 100644 --- a/fail2ban/server/datetemplate.py +++ b/fail2ban/server/datetemplate.py @@ -35,16 +35,18 @@ logSys = getLogger(__name__) # check already grouped contains "(", but ignores char "\(" and conditional "(?(id)...)": RE_GROUPED = re.compile(r'(?<!(?:\(\?))(?<!\\)\((?!\?)') RE_GROUP = ( re.compile(r'^((?:\(\?\w+\))?\^?(?:\(\?\w+\))?)(.*?)(\$?)$'), r"\1(\2)\3" ) +RE_GLOBALFLAGS = re.compile(r'((?:^|(?!<\\))\(\?[a-z]+\))') +RE_EXLINE_NO_BOUNDS = re.compile(r'^\{UNB\}') RE_EXLINE_BOUND_BEG = re.compile(r'^\{\^LN-BEG\}') -RE_EXSANC_BOUND_BEG = re.compile(r'^\(\?:\^\|\\b\|\\W\)') +RE_EXSANC_BOUND_BEG = re.compile(r'^\((?:\?:)?\^\|\\b\|\\W\)') RE_EXEANC_BOUND_BEG = re.compile(r'\(\?=\\b\|\\W\|\$\)$') -RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\(\?:\^)') +RE_NO_WRD_BOUND_BEG = re.compile(r'^\(*(?:\(\?\w+\))?(?:\^|\(*\*\*|\((?:\?:)?\^)') RE_NO_WRD_BOUND_END = re.compile(r'(?<!\\)(?:\$\)?|\\b|\\s|\*\*\)*)$') RE_DEL_WRD_BOUNDS = ( re.compile(r'^\(*(?:\(\?\w+\))?\(*\*\*|(?<!\\)\*\*\)*$'), lambda m: m.group().replace('**', '') ) -RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\(\?:\^(?!\|))') +RE_LINE_BOUND_BEG = re.compile(r'^(?:\(\?\w+\))?(?:\^|\((?:\?:)?\^(?!\|))') RE_LINE_BOUND_END = re.compile(r'(?<![\\\|])(?:\$\)?)$') RE_ALPHA_PATTERN = re.compile(r'(?<!\%)\%[aAbBpc]') @@ -82,7 +84,7 @@ class DateTemplate(object): return self._regex def setRegex(self, regex, wordBegin=True, wordEnd=True): - """Sets regex to use for searching for date in log line. + r"""Sets regex to use for searching for date in log line. Parameters ---------- @@ -109,6 +111,11 @@ class DateTemplate(object): # because it may be very slow in negative case (by long log-lines not matching pattern) regex = regex.strip() + # cut global flags like (?iu) from RE in order to pre-set it after processing: + gf = RE_GLOBALFLAGS.search(regex) + if gf: + regex = RE_GLOBALFLAGS.sub('', regex, count=1) + # check word boundaries needed: boundBegin = wordBegin and not RE_NO_WRD_BOUND_BEG.search(regex) boundEnd = wordEnd and not RE_NO_WRD_BOUND_END.search(regex) # if no group add it now, should always have a group(1): @@ -119,7 +126,7 @@ class DateTemplate(object): if boundBegin: self.flags |= DateTemplate.WORD_BEGIN if wordBegin != 'start' else DateTemplate.LINE_BEGIN if wordBegin != 'start': - regex = r'(?:^|\b|\W)' + regex + regex = r'(?=^|\b|\W)' + regex else: regex = r"^(?:\W{0,2})?" + regex if not self.name.startswith('{^LN-BEG}'): @@ -128,12 +135,16 @@ class DateTemplate(object): if boundEnd: self.flags |= DateTemplate.WORD_END regex += r'(?=\b|\W|$)' - if RE_LINE_BOUND_BEG.search(regex): self.flags |= DateTemplate.LINE_BEGIN - if RE_LINE_BOUND_END.search(regex): self.flags |= DateTemplate.LINE_END + if not (self.flags & DateTemplate.LINE_BEGIN) and RE_LINE_BOUND_BEG.search(regex): + self.flags |= DateTemplate.LINE_BEGIN + if not (self.flags & DateTemplate.LINE_END) and RE_LINE_BOUND_END.search(regex): + self.flags |= DateTemplate.LINE_END # remove possible special pattern "**" in front and end of regex: regex = RE_DEL_WRD_BOUNDS[0].sub(RE_DEL_WRD_BOUNDS[1], regex) + if gf: # restore global flags: + regex = gf.group(1) + regex self._regex = regex - logSys.log(7, ' constructed regex %s', regex) + logSys.log(4, ' constructed regex %s', regex) self._cRegex = None regex = property(getRegex, setRegex, doc= @@ -156,6 +167,7 @@ class DateTemplate(object): """ if not self._cRegex: self._compileRegex() + logSys.log(4, " search %s", self.regex) dateMatch = self._cRegex.search(line, *args); # pos, endpos if dateMatch: self.hits += 1 @@ -188,7 +200,7 @@ class DateTemplate(object): def unboundPattern(pattern): return RE_EXEANC_BOUND_BEG.sub('', RE_EXSANC_BOUND_BEG.sub('', - RE_EXLINE_BOUND_BEG.sub('', pattern) + RE_EXLINE_BOUND_BEG.sub('', RE_EXLINE_NO_BOUNDS.sub('', pattern)) ) ) @@ -215,8 +227,10 @@ class DateEpoch(DateTemplate): self.name = "LongEpoch" if not pattern else pattern epochRE = r"\d{10,11}(?:\d{3}(?:\.\d{1,6}|\d{3})?)?" if pattern: - # pattern should capture/cut out the whole match: - regex = "(" + RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) + ")" + # pattern should find the whole pattern, but cut out grouped match (or whole match if no groups specified): + regex = RE_EPOCH_PATTERN.sub(lambda v: "(%s)" % epochRE, pattern) + if not RE_GROUPED.search(pattern): + regex = "(" + regex + ")" self._grpIdx = 2 self.setRegex(regex) elif not lineBeginOnly: @@ -297,18 +311,25 @@ class DatePatternRegex(DateTemplate): def setRegex(self, pattern, wordBegin=True, wordEnd=True): # original pattern: self._pattern = pattern + # if unbound signalled - reset boundaries left and right: + if RE_EXLINE_NO_BOUNDS.search(pattern): + pattern = RE_EXLINE_NO_BOUNDS.sub('', pattern) + wordBegin = wordEnd = False # if explicit given {^LN-BEG} - remove it from pattern and set 'start' in wordBegin: if wordBegin and RE_EXLINE_BOUND_BEG.search(pattern): pattern = RE_EXLINE_BOUND_BEG.sub('', pattern) wordBegin = 'start' - # wrap to regex: - fmt = self._patternRE.sub(r'%(\1)s', pattern) - self.name = fmt % self._patternName - regex = fmt % timeRE - # if expected add (?iu) for "ignore case" and "unicode": - if RE_ALPHA_PATTERN.search(pattern): - regex = r'(?iu)' + regex - super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) + try: + # wrap to regex: + fmt = self._patternRE.sub(r'%(\1)s', pattern) + self.name = fmt % self._patternName + regex = fmt % timeRE + # if expected add (?iu) for "ignore case" and "unicode": + if RE_ALPHA_PATTERN.search(pattern): + regex = r'(?iu)' + regex + super(DatePatternRegex, self).setRegex(regex, wordBegin, wordEnd) + except Exception as e: + raise TypeError("Failed to set datepattern '%s' (may be an invalid format or unescaped percent char): %s" % (pattern, e)) def getDate(self, line, dateMatch=None, default_tz=None): """Method to return the date for a log line. diff --git a/fail2ban/server/failmanager.py b/fail2ban/server/failmanager.py index 6ce9b74e..3c71d51a 100644 --- a/fail2ban/server/failmanager.py +++ b/fail2ban/server/failmanager.py @@ -43,26 +43,20 @@ class FailManager: self.__maxRetry = 3 self.__maxTime = 600 self.__failTotal = 0 - self.maxEntries = 50 + self.maxMatches = 5 self.__bgSvc = BgService() def setFailTotal(self, value): - with self.__lock: - self.__failTotal = value + self.__failTotal = value def getFailTotal(self): - with self.__lock: - return self.__failTotal + return self.__failTotal def getFailCount(self): # may be slow on large list of failures, should be used for test purposes only... with self.__lock: return len(self.__failList), sum([f.getRetry() for f in self.__failList.values()]) - def getFailTotal(self): - with self.__lock: - return self.__failTotal - def setMaxRetry(self, value): self.__maxRetry = value @@ -87,24 +81,24 @@ class FailManager: attempt = 1 else: # will be incremented / extended (be sure we have at least +1 attempt): - matches = ticket.getMatches() + matches = ticket.getMatches() if self.maxMatches else None attempt = ticket.getAttempt() if attempt <= 0: attempt += 1 unixTime = ticket.getTime() - fData.setLastTime(unixTime) - if fData.getLastReset() < unixTime - self.__maxTime: - fData.setLastReset(unixTime) - fData.setRetry(0) + fData.adjustTime(unixTime, self.__maxTime) fData.inc(matches, attempt, count) - # truncate to maxEntries: - matches = fData.getMatches() - if len(matches) > self.maxEntries: - fData.setMatches(matches[-self.maxEntries:]) + # truncate to maxMatches: + if self.maxMatches: + matches = fData.getMatches() + if len(matches) > self.maxMatches: + fData.setMatches(matches[-self.maxMatches:]) + else: + fData.setMatches(None) except KeyError: # not found - already banned - prevent to add failure if comes from observer: if observed or isinstance(ticket, BanTicket): - return + return ticket.getRetry() # if already FailTicket - add it direct, otherwise create (using copy all ticket data): if isinstance(ticket, FailTicket): fData = ticket; @@ -130,13 +124,13 @@ class FailManager: return attempts def size(self): - with self.__lock: - return len(self.__failList) + return len(self.__failList) def cleanup(self, time): + time -= self.__maxTime with self.__lock: todelete = [fid for fid,item in self.__failList.iteritems() \ - if item.getLastTime() + self.__maxTime <= time] + if item.getTime() <= time] if len(todelete) == len(self.__failList): # remove all: self.__failList = dict() @@ -150,7 +144,7 @@ class FailManager: else: # create new dictionary without items to be deleted: self.__failList = dict((fid,item) for fid,item in self.__failList.iteritems() \ - if item.getLastTime() + self.__maxTime > time) + if item.getTime() > time) self.__bgSvc.service() def delFailure(self, fid): @@ -162,7 +156,7 @@ class FailManager: def toBan(self, fid=None): with self.__lock: - for fid in ([fid] if fid != None and fid in self.__failList else self.__failList): + for fid in ([fid] if fid is not None and fid in self.__failList else self.__failList): data = self.__failList[fid] if data.getRetry() >= self.__maxRetry: del self.__failList[fid] diff --git a/fail2ban/server/failregex.py b/fail2ban/server/failregex.py index 672bc32a..a9b144af 100644 --- a/fail2ban/server/failregex.py +++ b/fail2ban/server/failregex.py @@ -37,25 +37,28 @@ R_HOST = [ r"""(?:::f{4,6}:)?(?P<ip4>%s)""" % (IPAddr.IP_4_RE,), # separated ipv6: r"""(?P<ip6>%s)""" % (IPAddr.IP_6_RE,), - # place-holder for ipv6 enclosed in optional [] (used in addr-, host-regex) - "", # separated dns: r"""(?P<dns>[\w\-.^_]*\w)""", # place-holder for ADDR tag-replacement (joined): "", # place-holder for HOST tag replacement (joined): - "" + "", + # CIDR in simplest integer form: + r"(?P<cidr>\d+)", + # place-holder for SUBNET tag-replacement + "", ] RI_IPV4 = 0 RI_IPV6 = 1 -RI_IPV6BR = 2 -RI_DNS = 3 -RI_ADDR = 4 -RI_HOST = 5 +RI_DNS = 2 +RI_ADDR = 3 +RI_HOST = 4 +RI_CIDR = 5 +RI_SUBNET = 6 -R_HOST[RI_IPV6BR] = r"""\[?%s\]?""" % (R_HOST[RI_IPV6],) -R_HOST[RI_ADDR] = "(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR])),) -R_HOST[RI_HOST] = "(?:%s)" % ("|".join((R_HOST[RI_IPV4], R_HOST[RI_IPV6BR], R_HOST[RI_DNS])),) +R_HOST[RI_ADDR] = r"\[?(?:%s|%s)\]?" % (R_HOST[RI_IPV4], R_HOST[RI_IPV6],) +R_HOST[RI_HOST] = r"(?:%s|%s)" % (R_HOST[RI_ADDR], R_HOST[RI_DNS],) +R_HOST[RI_SUBNET] = r"\[?(?:%s|%s)(?:/%s)?\]?" % (R_HOST[RI_IPV4], R_HOST[RI_IPV6], R_HOST[RI_CIDR],) RH4TAG = { # separated ipv4 (self closed, closed): @@ -68,6 +71,11 @@ RH4TAG = { # for separate usage of 2 address groups only (regardless of `usedns`), `ip4` and `ip6` together "ADDR": R_HOST[RI_ADDR], "F-ADDR/": R_HOST[RI_ADDR], + # subnet tags for usage as `<ADDR>/<CIDR>` or `<SUBNET>`: + "CIDR": R_HOST[RI_CIDR], + "F-CIDR/": R_HOST[RI_CIDR], + "SUBNET": R_HOST[RI_SUBNET], + "F-SUBNET/":R_HOST[RI_SUBNET], # separated dns (self closed, closed): "DNS": R_HOST[RI_DNS], "F-DNS/": R_HOST[RI_DNS], @@ -79,20 +87,31 @@ RH4TAG = { # default failure groups map for customizable expressions (with different group-id): R_MAP = { - "ID": "fid", - "PORT": "fport", + "id": "fid", + "port": "fport", } +# map global flags like ((?i)xxx) or (?:(?i)xxx) to local flags (?i:xxx) if supported by RE-engine in this python version: +try: + re.search("^re(?i:val)$", "reVAL") + R_GLOB2LOCFLAGS = ( re.compile(r"(?<!\\)\((?:\?:)?(\(\?[a-z]+)\)"), r"\1:" ) +except: + R_GLOB2LOCFLAGS = () + def mapTag2Opt(tag): - try: # if should be mapped: - return R_MAP[tag] - except KeyError: - return tag.lower() + tag = tag.lower() + return R_MAP.get(tag, tag) -# alternate names to be merged, e. g. alt_user_1 -> user ... +# complex names: +# ALT_ - alternate names to be merged, e. g. alt_user_1 -> user ... ALTNAME_PRE = 'alt_' -ALTNAME_CRE = re.compile(r'^' + ALTNAME_PRE + r'(.*)(?:_\d+)?$') +# TUPLE_ - names of parts to be combined to single value as tuple +TUPNAME_PRE = 'tuple_' + +COMPLNAME_PRE = (ALTNAME_PRE, TUPNAME_PRE) +COMPLNAME_CRE = re.compile(r'^(' + '|'.join(COMPLNAME_PRE) + r')(.*?)(?:_\d+)?$') + ## # Regular expression class. @@ -116,20 +135,33 @@ class Regex: # if regex.lstrip() == '': raise RegexException("Cannot add empty regex") + # special handling wrapping global flags to local flags: + if R_GLOB2LOCFLAGS: + regex = R_GLOB2LOCFLAGS[0].sub(R_GLOB2LOCFLAGS[1], regex) try: self._regexObj = re.compile(regex, re.MULTILINE if multiline else 0) self._regex = regex - self._altValues = {} + self._altValues = [] + self._tupleValues = [] for k in filter( - lambda k: len(k) > len(ALTNAME_PRE) and k.startswith(ALTNAME_PRE), - self._regexObj.groupindex + lambda k: len(k) > len(COMPLNAME_PRE[0]), self._regexObj.groupindex ): - n = ALTNAME_CRE.match(k).group(1) - self._altValues[k] = n - self._altValues = list(self._altValues.items()) if len(self._altValues) else None - except sre_constants.error: - raise RegexException("Unable to compile regular expression '%s'" % - regex) + n = COMPLNAME_CRE.match(k) + if n: + g, n = n.group(1), mapTag2Opt(n.group(2)) + if g == ALTNAME_PRE: + self._altValues.append((k,n)) + else: + self._tupleValues.append((k,n)) + self._altValues.sort() + self._tupleValues.sort() + self._altValues = self._altValues if len(self._altValues) else None + self._tupleValues = self._tupleValues if len(self._tupleValues) else None + except sre_constants.error as e: + raise RegexException("Unable to compile regular expression '%s':\n%s" % + (regex, e)) + # set fetch handler depending on presence of alternate (or tuple) tags: + self.getGroups = self._getGroupsWithAlt if (self._altValues or self._tupleValues) else self._getGroups def __str__(self): return "%s(%r)" % (self.__class__.__name__, self._regex) @@ -269,18 +301,33 @@ class Regex: # Returns all matched groups. # - def getGroups(self): - if not self._altValues: - return self._matchCache.groupdict() - # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + def _getGroups(self): + return self._matchCache.groupdict() + + def _getGroupsWithAlt(self): fail = self._matchCache.groupdict() #fail = fail.copy() - for k,n in self._altValues: - v = fail.get(k) - if v and not fail.get(n): - fail[n] = v + # merge alternate values (e. g. 'alt_user_1' -> 'user' or 'alt_host' -> 'host'): + if self._altValues: + for k,n in self._altValues: + v = fail.get(k) + if v and not fail.get(n): + fail[n] = v + # combine tuple values (e. g. 'id', 'tuple_id' ... 'tuple_id_N' -> 'id'): + if self._tupleValues: + for k,n in self._tupleValues: + v = fail.get(k) + t = fail.get(n) + if isinstance(t, tuple): + t += (v,) + else: + t = (t,v,) + fail[n] = t return fail + def getGroups(self): # pragma: no cover - abstract function (replaced in __init__) + pass + ## # Returns skipped lines. # @@ -416,3 +463,7 @@ class FailRegex(Regex): def getHost(self): return self.getFailID(("ip4", "ip6", "dns")) + + def getIP(self): + fail = self.getGroups() + return IPAddr(self.getFailID(("ip4", "ip6")), int(fail.get("cidr") or IPAddr.CIDR_UNSPEC)) diff --git a/fail2ban/server/filter.py b/fail2ban/server/filter.py index 945ee9dd..68968284 100644 --- a/fail2ban/server/filter.py +++ b/fail2ban/server/filter.py @@ -81,6 +81,7 @@ class Filter(JailThread): ## Ignore own IPs flag: self.__ignoreSelf = True ## The ignore IP list. + self.__ignoreIpSet = set() self.__ignoreIpList = [] ## External command self.__ignoreCommand = False @@ -93,6 +94,8 @@ class Filter(JailThread): ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## Next service (cleanup) time + self.__nextSvcTime = -(1<<63) ## if set, treat log lines without explicit time zone to be in this time zone self.__logtimezone = None ## Default or preferred encoding (to decode bytes from file or journal): @@ -102,14 +105,28 @@ class Filter(JailThread): ## Error counter (protected, so can be used in filter implementations) ## if it reached 100 (at once), run-cycle will go idle self._errors = 0 + ## Next time to update log or journal position in database: + self._nextUpdateTM = 0 + ## Pending updates (must be executed at next update time or during stop): + self._pendDBUpdates = {} ## return raw host (host is not dns): self.returnRawHost = False ## check each regex (used for test purposes): self.checkAllRegex = False + ## avoid finding of pending failures (without ID/IP, used in fail2ban-regex): + self.ignorePending = True + ## callback called on ignoreregex match : + self.onIgnoreRegex = None ## if true ignores obsolete failures (failure time < now - findTime): self.checkFindTime = True + ## shows that filter is in operation mode (processing new messages): + self.inOperation = True ## Ticks counter self.ticks = 0 + ## Processed lines counter + self.procLines = 0 + ## Thread name: + self.name="f2b/f."+self.jailName self.dateDetector = DateDetector() logSys.debug("Created %s", self) @@ -167,7 +184,7 @@ class Filter(JailThread): # @param value the regular expression def addFailRegex(self, value): - multiLine = self.getMaxLines() > 1 + multiLine = self.__lineBufferSize > 1 try: regex = FailRegex(value, prefRegex=self.__prefRegex, multiline=multiLine, useDns=self.__useDns) @@ -428,31 +445,45 @@ class Filter(JailThread): ) else: self.__ignoreCache = None - ## - # Ban an IP - http://blogs.buanzo.com.ar/2009/04/fail2ban-patch-ban-ip-address-manually.html - # Arturo 'Buanzo' Busleiman <buanzo@buanzo.com.ar> - # - # to enable banip fail2ban-client BAN command - def addBannedIP(self, ip): + def performBan(self, ip=None): + """Performs a ban for IPs (or given ip) that are reached maxretry of the jail.""" + while True: + try: + ticket = self.failManager.toBan(ip) + except FailManagerEmpty: + break + self.jail.putFailTicket(ticket) + if ip: break + self.performSvc() + + def performSvc(self, force=False): + """Performs a service tasks (clean failure list).""" + tm = MyTime.time() + # avoid too early clean up: + if force or tm >= self.__nextSvcTime: + self.__nextSvcTime = tm + 5 + # clean up failure list: + self.failManager.cleanup(tm) + + def addAttempt(self, ip, *matches): + """Generate a failed attempt for ip""" if not isinstance(ip, IPAddr): ip = IPAddr(ip) + matches = list(matches) # tuple to list + # Generate the failure attempt for the IP: unixTime = MyTime.time() - ticket = FailTicket(ip, unixTime) - if self._inIgnoreIPList(ip, ticket, log_ignore=False): - logSys.warning('Requested to manually ban an ignored IP %s. User knows best. Proceeding to ban it.', ip) - self.failManager.addFailure(ticket, self.failManager.getMaxRetry()) + ticket = FailTicket(ip, unixTime, matches=matches) + logSys.info( + "[%s] Attempt %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") + ) + attempts = self.failManager.addFailure(ticket, len(matches) or 1) + # Perform the ban if this attempt is resulted to: + if attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) - # Perform the banning of the IP now. - try: # pragma: no branch - exception is the only way out - while True: - ticket = self.failManager.toBan(ip) - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - - return ip + return 1 ## # Ignore own IP/DNS. @@ -479,28 +510,36 @@ class Filter(JailThread): # Create IP address object ip = IPAddr(ipstr) # Avoid exact duplicates - if ip in self.__ignoreIpList: - logSys.warn(" Ignore duplicate %r (%r), already in ignore list", ip, ipstr) + if ip in self.__ignoreIpSet or ip in self.__ignoreIpList: + logSys.log(logging.MSG, " Ignore duplicate %r (%r), already in ignore list", ip, ipstr) return # log and append to ignore list logSys.debug(" Add %r to ignore list (%r)", ip, ipstr) - self.__ignoreIpList.append(ip) + # if single IP (not DNS or a subnet) add to set, otherwise to list: + if ip.isSingle: + self.__ignoreIpSet.add(ip) + else: + self.__ignoreIpList.append(ip) def delIgnoreIP(self, ip=None): # clear all: if ip is None: + self.__ignoreIpSet.clear() del self.__ignoreIpList[:] return # delete by ip: logSys.debug(" Remove %r from ignore list", ip) - self.__ignoreIpList.remove(ip) + if ip in self.__ignoreIpSet: + self.__ignoreIpSet.remove(ip) + else: + self.__ignoreIpList.remove(ip) def logIgnoreIp(self, ip, log_ignore, ignore_source="unknown source"): if log_ignore: logSys.info("[%s] Ignore %s by %s", self.jailName, ip, ignore_source) def getIgnoreIP(self): - return self.__ignoreIpList + return self.__ignoreIpList + list(self.__ignoreIpSet) ## # Check if IP address/DNS is in the ignore list. @@ -514,7 +553,7 @@ class Filter(JailThread): ticket = None if isinstance(ip, FailTicket): ticket = ip - ip = ticket.getIP() + ip = ticket.getID() elif not isinstance(ip, IPAddr): ip = IPAddr(ip) return self._inIgnoreIPList(ip, ticket, log_ignore) @@ -540,8 +579,11 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, True) return True + # check if the IP is covered by ignore IP (in set or in subnet/dns): + if ip in self.__ignoreIpSet: + self.logIgnoreIp(ip, log_ignore, ignore_source="ip") + return True for net in self.__ignoreIpList: - # check if the IP is covered by ignore IP if ip.isInNet(net): self.logIgnoreIp(ip, log_ignore, ignore_source=("ip" if net.isValid else "dns")) if self.__ignoreCache: c.set(key, True) @@ -564,50 +606,126 @@ class Filter(JailThread): if self.__ignoreCache: c.set(key, False) return False + def _logWarnOnce(self, nextLTM, *args): + """Log some issue as warning once per day, otherwise level 7""" + if MyTime.time() < getattr(self, nextLTM, 0): + if logSys.getEffectiveLevel() <= 7: logSys.log(7, *(args[0])) + else: + setattr(self, nextLTM, MyTime.time() + 24*60*60) + for args in args: + logSys.warning('[%s] ' + args[0], self.jailName, *args[1:]) + def processLine(self, line, date=None): """Split the time portion from log msg and return findFailures on them """ + logSys.log(7, "Working on line %r", line) + + noDate = False if date: tupleLine = line + line = "".join(line) + self.__lastTimeText = tupleLine[1] + self.__lastDate = date else: - l = line.rstrip('\r\n') - logSys.log(7, "Working on line %r", line) - - (timeMatch, template) = self.dateDetector.matchTime(l) - if timeMatch: - tupleLine = ( - l[:timeMatch.start(1)], - l[timeMatch.start(1):timeMatch.end(1)], - l[timeMatch.end(1):], - (timeMatch, template) - ) + # try to parse date: + timeMatch = self.dateDetector.matchTime(line) + m = timeMatch[0] + if m: + s = m.start(1) + e = m.end(1) + m = line[s:e] + tupleLine = (line[:s], m, line[e:]) + if m: # found and not empty - retrive date: + date = self.dateDetector.getTime(m, timeMatch) + if date is not None: + # Lets get the time part + date = date[0] + self.__lastTimeText = m + self.__lastDate = date + else: + logSys.error("findFailure failed to parse timeText: %s", m) + # matched empty value - date is optional or not available - set it to last known or now: + elif self.__lastDate and self.__lastDate > MyTime.time() - 60: + # set it to last known: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate + else: + # set it to now: + date = MyTime.time() + else: + tupleLine = ("", "", line) + # still no date - try to use last known: + if date is None: + noDate = True + if self.__lastDate and self.__lastDate > MyTime.time() - 60: + tupleLine = ("", self.__lastTimeText, line) + date = self.__lastDate + elif self.checkFindTime and self.inOperation: + date = MyTime.time() + + if self.checkFindTime and date is not None: + # if in operation (modifications have been really found): + if self.inOperation: + # if weird date - we'd simulate now for timeing issue (too large deviation from now): + delta = int(date - MyTime.time()) + if abs(delta) > 60: + # log timing issue as warning once per day: + self._logWarnOnce("_next_simByTimeWarn", + ("Detected a log entry %s %s the current time in operation mode. " + "This looks like a %s problem. Treating such entries as if they just happened.", + MyTime.seconds2str(abs(delta)), "before" if delta < 0 else "after", + "latency" if -3300 <= delta < 0 else "timezone" + ), + ("Please check a jail for a timing issue. Line with odd timestamp: %s", + line)) + # simulate now as date: + date = MyTime.time() + self.__lastDate = date else: - tupleLine = (l, "", "", None) + # in initialization (restore) phase, if too old - ignore: + if date < MyTime.time() - self.getFindTime(): + # log time zone issue as warning once per day: + self._logWarnOnce("_next_ignByTimeWarn", + ("Ignoring all log entries older than %ss; these are probably" + + " messages generated while fail2ban was not running.", + self.getFindTime()), + ("Please check a jail for a timing issue. Line with odd timestamp: %s", + line)) + # ignore - too old (obsolete) entry: + return [] # save last line (lazy convert of process line tuple to string on demand): self.processedLine = lambda: "".join(tupleLine[::2]) - return self.findFailure(tupleLine, date) + return self.findFailure(tupleLine, date, noDate=noDate) def processLineAndAdd(self, line, date=None): """Processes the line for failures and populates failManager """ try: - for element in self.processLine(line, date): - ip = element[1] - unixTime = element[2] - fail = element[3] + for (_, ip, unixTime, fail) in self.processLine(line, date): logSys.debug("Processing line with time:%s and ip:%s", unixTime, ip) + # ensure the time is not in the future, e. g. by some estimated (assumed) time: + if self.checkFindTime and unixTime > MyTime.time(): + unixTime = MyTime.time() tick = FailTicket(ip, unixTime, data=fail) if self._inIgnoreIPList(ip, tick): continue logSys.info( - "[%s] Found %s - %s", self.jailName, ip, datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S") + "[%s] Found %s - %s", self.jailName, ip, MyTime.time2str(unixTime) ) - self.failManager.addFailure(tick) + attempts = self.failManager.addFailure(tick) + # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, + # we can speedup ban, so do it as soon as possible: + if attempts >= self.failManager.getMaxRetry(): + self.performBan(ip) # report to observer - failure was found, for possibly increasing of it retry counter (asynchronous) if Observers.Main is not None: - Observers.Main.add('failureFound', self.failManager, self.jail, tick) + Observers.Main.add('failureFound', self.jail, tick) + self.procLines += 1 + # every 100 lines check need to perform service tasks: + if self.procLines % 100 == 0: + self.performSvc() # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 @@ -617,7 +735,7 @@ class Filter(JailThread): # incr common error counter: self.commonError() - def commonError(self): + def commonError(self, reason="common", exc=None): # incr error counter, stop processing (going idle) after 100th error : self._errors += 1 # sleep a little bit (to get around time-related errors): @@ -627,20 +745,26 @@ class Filter(JailThread): self._errors //= 2 self.idle = True - ## - # Returns true if the line should be ignored. - # - # Uses ignoreregex. - # @param line: the line - # @return: a boolean - - def ignoreLine(self, tupleLines): - buf = Regex._tupleLinesBuf(tupleLines) + def _ignoreLine(self, buf, orgBuffer, failRegex=None): + # if multi-line buffer - use matched only, otherwise (single line) - original buf: + if failRegex and self.__lineBufferSize > 1: + orgBuffer = failRegex.getMatchedTupleLines() + buf = Regex._tupleLinesBuf(orgBuffer) + # search ignored: + fnd = None for ignoreRegexIndex, ignoreRegex in enumerate(self.__ignoreRegex): - ignoreRegex.search(buf, tupleLines) + ignoreRegex.search(buf, orgBuffer) if ignoreRegex.hasMatched(): - return ignoreRegexIndex - return None + fnd = ignoreRegexIndex + logSys.log(7, " Matched ignoreregex %d and was ignored", fnd) + if self.onIgnoreRegex: self.onIgnoreRegex(fnd, ignoreRegex) + # remove ignored match: + if not self.checkAllRegex or self.__lineBufferSize > 1: + # todo: check ignoreRegex.getUnmatchedTupleLines() would be better (fix testGetFailuresMultiLineIgnoreRegex): + if failRegex: + self.__lineBuffer = failRegex.getUnmatchedTupleLines() + if not self.checkAllRegex: break + return fnd def _updateUsers(self, fail, user=()): users = fail.get('users') @@ -650,49 +774,33 @@ class Filter(JailThread): fail['users'] = users = set() users.add(user) return users - return None - - # # ATM incremental (non-empty only) merge deactivated ... - # @staticmethod - # def _updateFailure(self, mlfidGroups, fail): - # # reset old failure-ids when new types of id available in this failure: - # fids = set() - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if fail.get(k): - # fids.add(k) - # if fids: - # for k in ('fid', 'ip4', 'ip6', 'dns'): - # if k not in fids: - # try: - # del mlfidGroups[k] - # except: - # pass - # # update not empty values: - # mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v)) + return users def _mergeFailure(self, mlfid, fail, failRegex): mlfidFail = self.mlfidCache.get(mlfid) if self.__mlfidCache else None users = None nfflgs = 0 - if fail.get('nofail'): nfflgs |= 1 - if fail.get('mlfforget'): nfflgs |= 2 + if fail.get("mlfgained"): + nfflgs |= (8|1) + if not fail.get('nofail'): + fail['nofail'] = fail["mlfgained"] + elif fail.get('nofail'): nfflgs |= 1 + if fail.pop('mlfforget', None): nfflgs |= 2 # if multi-line failure id (connection id) known: if mlfidFail: mlfidGroups = mlfidFail[1] # update users set (hold all users of connect): users = self._updateUsers(mlfidGroups, fail.get('user')) - # be sure we've correct current state ('nofail' only from last failure) - try: - del mlfidGroups['nofail'] - except KeyError: - pass - # # ATM incremental (non-empty only) merge deactivated (for future version only), - # # it can be simulated using alternate value tags, like <F-ALT_VAL>...</F-ALT_VAL>, - # # so previous value 'val' will be overwritten only if 'alt_val' is not empty... - # _updateFailure(mlfidGroups, fail) - # + # be sure we've correct current state ('nofail' and 'mlfgained' only from last failure) + if mlfidGroups.pop('nofail', None): nfflgs |= 4 + if mlfidGroups.pop('mlfgained', None): nfflgs |= 4 + # gained resets all pending failures (retaining users to check it later) + if nfflgs & 8: mlfidGroups.pop('mlfpending', None) + # if we had no pending failures then clear the matches (they are already provided): + if (nfflgs & 4) == 0 and not mlfidGroups.get('mlfpending', 0): + mlfidGroups.pop("matches", None) # overwrite multi-line failure with all values, available in fail: - mlfidGroups.update(fail) + mlfidGroups.update(((k,v) for k,v in fail.iteritems() if v is not None)) # new merged failure data: fail = mlfidGroups # if forget (disconnect/reset) - remove cached entry: @@ -703,23 +811,19 @@ class Filter(JailThread): mlfidFail = [self.__lastDate, fail] self.mlfidCache.set(mlfid, mlfidFail) # check users in order to avoid reset failure by multiple logon-attempts: - if users and len(users) > 1: - # we've new user, reset 'nofail' because of multiple users attempts: - try: - del fail['nofail'] - except KeyError: - pass + if fail.pop('mlfpending', 0) or users and len(users) > 1: + # we've pending failures or new user, reset 'nofail' because of failures or multiple users attempts: + fail.pop('nofail', None) + fail.pop('mlfgained', None) + nfflgs &= ~(8|1) # reset nofail and gained # merge matches: - if not fail.get('nofail'): # current state (corresponding users) - try: - m = fail.pop("nofail-matches") - m += fail.get("matches", []) - except KeyError: - m = fail.get("matches", []) - if not (nfflgs & 2): # not mlfforget: + if (nfflgs & 1) == 0: # current nofail state (corresponding users) + m = fail.pop("nofail-matches", []) + m += fail.get("matches", []) + if (nfflgs & 8) == 0: # no gain signaled m += failRegex.getMatchedTupleLines() fail["matches"] = m - elif not (nfflgs & 2) and (nfflgs & 1): # not mlfforget and nofail: + elif (nfflgs & 3) == 1: # not mlfforget and nofail: fail["nofail-matches"] = fail.get("nofail-matches", []) + failRegex.getMatchedTupleLines() # return merged: return fail @@ -732,72 +836,41 @@ class Filter(JailThread): # to find the logging time. # @return a dict with IP and timestamp. - def findFailure(self, tupleLine, date=None): + def findFailure(self, tupleLine, date, noDate=False): failList = list() ll = logSys.getEffectiveLevel() - returnRawHost = self.returnRawHost - cidr = IPAddr.CIDR_UNSPEC - if self.__useDns == "raw": - returnRawHost = True - cidr = IPAddr.CIDR_RAW - - # Checks if we mut ignore this line. - if self.ignoreLine([tupleLine[::2]]) is not None: - # The ignoreregex matched. Return. - if ll <= 7: logSys.log(7, "Matched ignoreregex and was \"%s\" ignored", - "".join(tupleLine[::2])) - return failList - - timeText = tupleLine[1] - if date: - self.__lastTimeText = timeText - self.__lastDate = date - elif timeText: - - dateTimeMatch = self.dateDetector.getTime(timeText, tupleLine[3]) - - if dateTimeMatch is None: - logSys.error("findFailure failed to parse timeText: %s", timeText) - date = self.__lastDate - - else: - # Lets get the time part - date = dateTimeMatch[0] - - self.__lastTimeText = timeText - self.__lastDate = date - else: - timeText = self.__lastTimeText or "".join(tupleLine[::2]) - date = self.__lastDate - - if self.checkFindTime and date is not None and date < MyTime.time() - self.getFindTime(): - if ll <= 5: logSys.log(5, "Ignore line since time %s < %s - %s", - date, MyTime.time(), self.getFindTime()) - return failList + defcidr = IPAddr.CIDR_UNSPEC + if self.__useDns == "raw" or self.returnRawHost: + defcidr = IPAddr.CIDR_RAW if self.__lineBufferSize > 1: - orgBuffer = self.__lineBuffer = ( - self.__lineBuffer + [tupleLine[:3]])[-self.__lineBufferSize:] + self.__lineBuffer.append(tupleLine) + orgBuffer = self.__lineBuffer = self.__lineBuffer[-self.__lineBufferSize:] else: - orgBuffer = self.__lineBuffer = [tupleLine[:3]] - if ll <= 5: logSys.log(5, "Looking for match of %r", self.__lineBuffer) - buf = Regex._tupleLinesBuf(self.__lineBuffer) + orgBuffer = self.__lineBuffer = [tupleLine] + if ll <= 5: logSys.log(5, "Looking for match of %r", orgBuffer) + buf = Regex._tupleLinesBuf(orgBuffer) + + # Checks if we must ignore this line (only if fewer ignoreregex than failregex). + if self.__ignoreRegex and len(self.__ignoreRegex) < len(self.__failRegex) - 2: + if self._ignoreLine(buf, orgBuffer) is not None: + # The ignoreregex matched. Return. + return failList # Pre-filter fail regex (if available): preGroups = {} if self.__prefRegex: if ll <= 5: logSys.log(5, " Looking for prefregex %r", self.__prefRegex.getRegex()) - self.__prefRegex.search(buf, self.__lineBuffer) + self.__prefRegex.search(buf, orgBuffer) if not self.__prefRegex.hasMatched(): if ll <= 5: logSys.log(5, " Prefregex not matched") return failList preGroups = self.__prefRegex.getGroups() if ll <= 7: logSys.log(7, " Pre-filter matched %s", preGroups) - repl = preGroups.get('content') + repl = preGroups.pop('content', None) # Content replacement: if repl: - del preGroups['content'] self.__lineBuffer, buf = [('', '', repl)], None # Iterates over all the regular expressions. @@ -815,31 +888,25 @@ class Filter(JailThread): # The failregex matched. if ll <= 7: logSys.log(7, " Matched failregex %d: %s", failRegexIndex, fail) # Checks if we must ignore this match. - if self.ignoreLine(failRegex.getMatchedTupleLines()) \ - is not None: + if self.__ignoreRegex and self._ignoreLine(buf, orgBuffer, failRegex) is not None: # The ignoreregex matched. Remove ignored match. - self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None - if ll <= 7: logSys.log(7, " Matched ignoreregex and was ignored") + buf = None if not self.checkAllRegex: break - else: - continue - if date is None: - logSys.warning( - "Found a match for %r but no valid date/time " - "found for %r. Please try setting a custom " - "date pattern (see man page jail.conf(5)). " - "If format is complex, please " - "file a detailed issue on" - " https://github.com/fail2ban/fail2ban/issues " - "in order to get support for this format.", - "\n".join(failRegex.getMatchedLines()), timeText) continue + if noDate: + self._logWarnOnce("_next_noTimeWarn", + ("Found a match but no valid date/time found for %r.", tupleLine[1]), + ("Match without a timestamp: %s", "\n".join(failRegex.getMatchedLines())), + ("Please try setting a custom date pattern (see man page jail.conf(5)).",) + ) + if date is None and self.checkFindTime: continue # we should check all regex (bypass on multi-line, otherwise too complex): - if not self.checkAllRegex or self.getMaxLines() > 1: + if not self.checkAllRegex or self.__lineBufferSize > 1: self.__lineBuffer, buf = failRegex.getUnmatchedTupleLines(), None # merge data if multi-line failure: - raw = returnRawHost + cidr = defcidr + raw = (defcidr == IPAddr.CIDR_RAW) if preGroups: currFail, fail = fail, preGroups.copy() fail.update(currFail) @@ -858,44 +925,50 @@ class Filter(JailThread): # failure-id: fid = fail.get('fid') # ip-address or host: - host = fail.get('ip4') - if host is not None: - cidr = IPAddr.FAM_IPv4 + ip = fail.get('ip4') + if ip is not None: + cidr = int(fail.get('cidr') or IPAddr.FAM_IPv4) raw = True else: - host = fail.get('ip6') - if host is not None: - cidr = IPAddr.FAM_IPv6 + ip = fail.get('ip6') + if ip is not None: + cidr = int(fail.get('cidr') or IPAddr.FAM_IPv6) raw = True - if host is None: - host = fail.get('dns') - if host is None: - # first try to check we have mlfid case (cache connection id): - if fid is None and mlfid is None: - # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: - fid = failRegex.getFailID() - host = fid - cidr = IPAddr.CIDR_RAW + else: + ip = fail.get('dns') + if ip is None: + # first try to check we have mlfid case (cache connection id): + if fid is None and mlfid is None: + # if no failure-id also (obscure case, wrong regex), throw error inside getFailID: + fid = failRegex.getFailID() + ip = fid + raw = True # if mlfid case (not failure): - if host is None: + if ip is None: if ll <= 7: logSys.log(7, "No failure-id by mlfid %r in regex %s: %s", mlfid, failRegexIndex, fail.get('mlfforget', "waiting for identifier")) - if not self.checkAllRegex: return failList - ips = [None] + fail['mlfpending'] = 1; # mark failure is pending + if not self.checkAllRegex and self.ignorePending: return failList + fids = [None] # if raw - add single ip or failure-id, # otherwise expand host to multiple ips using dns (or ignore it if not valid): elif raw: - ip = IPAddr(host, cidr) - # check host equal failure-id, if not - failure with complex id: - if fid is not None and fid != host: - ip = IPAddr(fid, IPAddr.CIDR_RAW) - ips = [ip] + # check ip/host equal failure-id, if not - failure with complex id: + if fid is None or fid == ip: + fid = IPAddr(ip, cidr) + else: + fail['ip'] = IPAddr(ip, cidr) + fid = IPAddr(fid, defcidr) + fids = [fid] # otherwise, try to use dns conversion: else: - ips = DNSUtils.textToIp(host, self.__useDns) + fids = DNSUtils.textToIp(ip, self.__useDns) + # if checkAllRegex we must make a copy (to be sure next RE doesn't change merged/cached failure): + if self.checkAllRegex and mlfid is not None: + fail = fail.copy() # append failure with match to the list: - for ip in ips: - failList.append([failRegexIndex, ip, date, fail]) + for fid in fids: + failList.append([failRegexIndex, fid, date, fail]) if not self.checkAllRegex: break except RegexException as e: # pragma: no cover - unsure if reachable @@ -938,7 +1011,7 @@ class FileFilter(Filter): log.setPos(lastpos) self.__logs[path] = log logSys.info("Added logfile: %r (pos = %s, hash = %s)" , path, log.getPos(), log.getHash()) - if autoSeek: + if autoSeek and not tail: self.__autoSeek[path] = autoSeek self._addLogPath(path) # backend specific @@ -957,9 +1030,6 @@ class FileFilter(Filter): log = self.__logs.pop(path) except KeyError: return - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) logSys.info("Removed logfile: %r", path) self._delLogPath(path) return @@ -1022,7 +1092,8 @@ class FileFilter(Filter): # MyTime.time()-self.findTime. When a failure is detected, a FailTicket # is created and is added to the FailManager. - def getFailures(self, filename): + def getFailures(self, filename, inOperation=None): + if self.idle: return False log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) @@ -1067,15 +1138,26 @@ class FileFilter(Filter): if has_content: while not self.idle: line = log.readline() - if not line or not self.active: - # The jail reached the bottom or has been stopped + if not self.active: break; # jail has been stopped + if line is None: + # The jail reached the bottom, simply set in operation for this log + # (since we are first time at end of file, growing is only possible after modifications): + log.inOperation = True break + # acquire in operation from log and process: + self.inOperation = inOperation if inOperation is not None else log.inOperation self.processLineAndAdd(line) finally: log.close() - db = self.jail.database - if db is not None: - db.updateLog(self.jail, log) + if self.jail.database is not None: + self._pendDBUpdates[log] = 1 + if ( + self.ticks % 100 == 0 + or MyTime.time() >= self._nextUpdateTM + or not self.active + ): + self._updateDBPending() + self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 return True ## @@ -1086,7 +1168,9 @@ class FileFilter(Filter): fs = container.getFileSize() if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Seek to find time %s (%s), file size %s", date, - datetime.datetime.fromtimestamp(date).strftime("%Y-%m-%d %H:%M:%S"), fs) + MyTime.time2str(date), fs) + if not fs: + return minp = container.getPos() maxp = fs tryPos = minp @@ -1110,8 +1194,8 @@ class FileFilter(Filter): dateTimeMatch = None nextp = None while True: - line = container.readline() - if not line: + line = container.readline(False) + if line is None: break (timeMatch, template) = self.dateDetector.matchTime(line) if timeMatch: @@ -1165,7 +1249,7 @@ class FileFilter(Filter): container.setPos(foundPos) if logSys.getEffectiveLevel() <= logging.DEBUG: logSys.debug("Position %s from %s, found time %s (%s) within %s seeks", lastPos, fs, foundTime, - (datetime.datetime.fromtimestamp(foundTime).strftime("%Y-%m-%d %H:%M:%S") if foundTime is not None else ''), cntr) + (MyTime.time2str(foundTime) if foundTime is not None else ''), cntr) def status(self, flavor="basic"): """Status of Filter plus files being monitored. @@ -1175,12 +1259,33 @@ class FileFilter(Filter): ret.append(("File list", path)) return ret - def stop(self): - """Stop monitoring of log-file(s) + def _updateDBPending(self): + """Apply pending updates (log position) to database. + """ + db = self.jail.database + while True: + try: + log, args = self._pendDBUpdates.popitem() + except KeyError: + break + db.updateLog(self.jail, log) + + def onStop(self): + """Stop monitoring of log-file(s). Invoked after run method. """ + # ensure positions of pending logs are up-to-date: + if self._pendDBUpdates and self.jail.database: + self._updateDBPending() # stop files monitoring: for path in self.__logs.keys(): self.delLogPath(path) + + def stop(self): + """Stop filter + """ + # normally onStop will be called automatically in thread after its run ends, + # but for backwards compatibilities we'll invoke it in caller of stop method. + self.onStop() # stop thread: super(Filter, self).stop() @@ -1208,32 +1313,56 @@ except ImportError: # pragma: no cover class FileContainer: - def __init__(self, filename, encoding, tail = False): + def __init__(self, filename, encoding, tail=False, doOpen=False): self.__filename = filename + self.waitForLineEnd = True self.setEncoding(encoding) self.__tail = tail self.__handler = None + self.__pos = 0 + self.__pos4hash = 0 + self.__hash = '' + self.__hashNextTime = time.time() + 30 # Try to open the file. Raises an exception if an error occurred. handler = open(filename, 'rb') - stats = os.fstat(handler.fileno()) - self.__ino = stats.st_ino + if doOpen: # fail2ban-regex only (don't need to reopen it and check for rotation) + self.__handler = handler + return try: - firstLine = handler.readline() - # Computes the MD5 of the first line. - self.__hash = md5sum(firstLine).hexdigest() - # Start at the beginning of file if tail mode is off. - if tail: - handler.seek(0, 2) - self.__pos = handler.tell() - else: - self.__pos = 0 + stats = os.fstat(handler.fileno()) + self.__ino = stats.st_ino + if stats.st_size: + firstLine = handler.readline() + # first line available and contains new-line: + if firstLine != firstLine.rstrip(b'\r\n'): + # Computes the MD5 of the first line. + self.__hash = md5sum(firstLine).hexdigest() + # if tail mode scroll to the end of file + if tail: + handler.seek(0, 2) + self.__pos = handler.tell() finally: handler.close() + ## shows that log is in operation mode (expecting new messages only from here): + self.inOperation = tail + + def __hash__(self): + return hash(self.__filename) + def __eq__(self, other): + return (id(self) == id(other) or + self.__filename == (other.__filename if isinstance(other, FileContainer) else other) + ) + def __repr__(self): + return 'file-log:'+self.__filename def getFileName(self): return self.__filename def getFileSize(self): + h = self.__handler + if h is not None: + stats = os.fstat(h.fileno()) + return stats.st_size return os.path.getsize(self.__filename); def setEncoding(self, encoding): @@ -1252,38 +1381,54 @@ class FileContainer: def setPos(self, value): self.__pos = value - def open(self): - self.__handler = open(self.__filename, 'rb') - # Set the file descriptor to be FD_CLOEXEC - fd = self.__handler.fileno() - flags = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) - # Stat the file before even attempting to read it - stats = os.fstat(self.__handler.fileno()) - if not stats.st_size: - # yoh: so it is still an empty file -- nothing should be - # read from it yet - # print "D: no content -- return" - return False - firstLine = self.__handler.readline() - # Computes the MD5 of the first line. - myHash = md5sum(firstLine).hexdigest() - ## print "D: fn=%s hashes=%s/%s inos=%s/%s pos=%s rotate=%s" % ( - ## self.__filename, self.__hash, myHash, stats.st_ino, self.__ino, self.__pos, - ## self.__hash != myHash or self.__ino != stats.st_ino) - ## sys.stdout.flush() - # Compare hash and inode - if self.__hash != myHash or self.__ino != stats.st_ino: - logSys.log(logging.MSG, "Log rotation detected for %s", self.__filename) - self.__hash = myHash - self.__ino = stats.st_ino - self.__pos = 0 - # Sets the file pointer to the last position. - self.__handler.seek(self.__pos) + def open(self, forcePos=None): + h = open(self.__filename, 'rb') + try: + # Set the file descriptor to be FD_CLOEXEC + fd = h.fileno() + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + myHash = self.__hash + # Stat the file before even attempting to read it + stats = os.fstat(h.fileno()) + rotflg = stats.st_size < self.__pos or stats.st_ino != self.__ino + if rotflg or not len(myHash) or time.time() > self.__hashNextTime: + myHash = '' + firstLine = h.readline() + # Computes the MD5 of the first line (if it is complete) + if firstLine != firstLine.rstrip(b'\r\n'): + myHash = md5sum(firstLine).hexdigest() + self.__hashNextTime = time.time() + 30 + elif stats.st_size == self.__pos: + myHash = self.__hash + # Compare size, hash and inode + if rotflg or myHash != self.__hash: + if self.__hash != '': + logSys.log(logging.MSG, "Log rotation detected for %s, reason: %r", self.__filename, + (stats.st_size, self.__pos, stats.st_ino, self.__ino, myHash, self.__hash)) + self.__ino = stats.st_ino + self.__pos = 0 + self.__hash = myHash + # if nothing to read from file yet (empty or no new data): + if forcePos is not None: + self.__pos = forcePos + elif stats.st_size <= self.__pos: + return False + # Sets the file pointer to the last position. + h.seek(self.__pos) + # leave file open (to read content): + self.__handler = h; h = None + finally: + # close (no content or error only) + if h: + h.close(); h = None return True def seek(self, offs, endLine=True): h = self.__handler + if h is None: + self.open(offs) + h = self.__handler # seek to given position h.seek(offs, 0) # goto end of next line @@ -1301,38 +1446,98 @@ class FileContainer: try: return line.decode(enc, 'strict') except (UnicodeDecodeError, UnicodeEncodeError) as e: + # avoid warning if got incomplete end of line (e. g. '\n' in "...[0A" followed by "00]..." for utf-16le: + if (e.end == len(line) and line[e.start] in b'\r\n'): + return line[0:e.start].decode(enc, 'replace') global _decode_line_warn - lev = logging.DEBUG - if _decode_line_warn.get(filename, 0) <= MyTime.time(): + lev = 7 + if not _decode_line_warn.get(filename, 0): lev = logging.WARNING - _decode_line_warn[filename] = MyTime.time() + 24*60*60 + _decode_line_warn.set(filename, 1) logSys.log(lev, - "Error decoding line from '%s' with '%s'." - " Consider setting logencoding=utf-8 (or another appropriate" - " encoding) for this jail. Continuing" - " to process line ignoring invalid characters: %r", - filename, enc, line) + "Error decoding line from '%s' with '%s'.", filename, enc) + if logSys.getEffectiveLevel() <= lev: + logSys.log(lev, + "Consider setting logencoding to appropriate encoding for this jail. " + "Continuing to process line ignoring invalid characters: %r", + line) # decode with replacing error chars: line = line.decode(enc, 'replace') return line - def readline(self): + def readline(self, complete=True): + """Read line from file + + In opposite to pythons readline it doesn't return new-line, + so returns either the line if line is complete (and complete=True) or None + if line is not complete (and complete=True) or there is no content to read. + If line is complete (and complete is True), it also shift current known + position to begin of next line. + + Also it is safe against interim new-line bytes (e. g. part of multi-byte char) + in given encoding. + """ if self.__handler is None: return "" - return FileContainer.decode_line( - self.getFileName(), self.getEncoding(), self.__handler.readline()) + # read raw bytes up to \n char: + b = self.__handler.readline() + if not b: + return None + bl = len(b) + # convert to log-encoding (new-line char could disappear if it is part of multi-byte sequence): + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if complete: + if l == r: + # try to fill buffer in order to find line-end in log encoding: + fnd = 0 + while 1: + r = self.__handler.readline() + if not r: + break + b += r + bl += len(r) + # convert to log-encoding: + r = FileContainer.decode_line( + self.getFileName(), self.getEncoding(), b) + # ensure new-line is not in the middle (buffered 2 strings, e. g. in utf-16le it is "...[0A"+"00]..."): + e = r.find('\n') + if e >= 0 and e != len(r)-1: + l, r = r[0:e], r[0:e+1] + # back to bytes and get offset to seek after NL: + r = r.encode(self.getEncoding(), 'replace') + self.__handler.seek(-bl+len(r), 1) + return l + # trim new-line at end and check the line was written complete (contains a new-line): + l = r.rstrip('\r\n') + if l != r: + return l + if self.waitForLineEnd: + # not fulfilled - seek back and return: + self.__handler.seek(-bl, 1) + return None + return l def close(self): - if not self.__handler is None: - # Saves the last position. + if self.__handler is not None: + # Saves the last real position. self.__pos = self.__handler.tell() # Closes the file. self.__handler.close() self.__handler = None - ## print "D: Closed %s with pos %d" % (handler, self.__pos) - ## sys.stdout.flush() -_decode_line_warn = {} + def __iter__(self): + return self + def next(self): + line = self.readline() + if line is None: + self.close() + raise StopIteration + return line + +_decode_line_warn = Utils.Cache(maxCount=1000, maxTime=24*60*60); ## diff --git a/fail2ban/server/filtergamin.py b/fail2ban/server/filtergamin.py index 3baf8c54..c5373445 100644 --- a/fail2ban/server/filtergamin.py +++ b/fail2ban/server/filtergamin.py @@ -55,7 +55,6 @@ class FilterGamin(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Gamin monitor self.monitor = gamin.WatchMonitor() fd = self.monitor.get_fd() @@ -64,28 +63,12 @@ class FilterGamin(FileFilter): logSys.debug("Created FilterGamin") def callback(self, path, event): - logSys.debug("Got event: " + repr(event) + " for " + path) + logSys.log(4, "Got event: " + repr(event) + " for " + path) if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists): logSys.debug("File changed: " + path) - self.__modified = True self.ticks += 1 - self._process_file(path) - - def _process_file(self, path): - """Process a given file - - TODO -- RF: - this is a common logic and must be shared/provided by FileFilter - """ self.getFailures(path) - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False ## # Add a log file path @@ -132,6 +115,9 @@ class FilterGamin(FileFilter): Utils.wait_for(lambda: not self.active or self._handleEvents(), self.sleeptime) self.ticks += 1 + if self.ticks % 10 == 0: + self.performSvc() + logSys.debug("[%s] filter terminated", self.jailName) return True diff --git a/fail2ban/server/filterpoll.py b/fail2ban/server/filterpoll.py index 5905c5b5..196955e5 100644 --- a/fail2ban/server/filterpoll.py +++ b/fail2ban/server/filterpoll.py @@ -27,9 +27,7 @@ __license__ = "GPL" import os import time -from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime from .utils import Utils from ..helpers import getLogger, logging @@ -55,7 +53,6 @@ class FilterPoll(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() @@ -98,8 +95,8 @@ class FilterPoll(FileFilter): def run(self): while self.active: try: - if logSys.getEffectiveLevel() <= 6: - logSys.log(6, "Woke up idle=%s with %d files monitored", + if logSys.getEffectiveLevel() <= 4: + logSys.log(4, "Woke up idle=%s with %d files monitored", self.idle, self.getLogCount()) if self.idle: if not Utils.wait_for(lambda: not self.active or not self.idle, @@ -111,26 +108,21 @@ class FilterPoll(FileFilter): modlst = [] Utils.wait_for(lambda: not self.active or self.getModified(modlst), self.sleeptime) + if not self.active: # pragma: no cover - timing + break for filename in modlst: self.getFailures(filename) - self.__modified = True self.ticks += 1 - if self.__modified: - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) return True @@ -144,11 +136,11 @@ class FilterPoll(FileFilter): try: logStats = os.stat(filename) stats = logStats.st_mtime, logStats.st_ino, logStats.st_size - pstats = self.__prevStats.get(filename, (0)) - if logSys.getEffectiveLevel() <= 5: + pstats = self.__prevStats.get(filename, (0,)) + if logSys.getEffectiveLevel() <= 4: # we do not want to waste time on strftime etc if not necessary dt = logStats.st_mtime - pstats[0] - logSys.log(5, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s", + logSys.log(4, "Checking %s for being modified. Previous/current stats: %s / %s. dt: %s", filename, pstats, stats, dt) # os.system("stat %s | grep Modify" % filename) self.__file404Cnt[filename] = 0 diff --git a/fail2ban/server/filterpyinotify.py b/fail2ban/server/filterpyinotify.py index 4f3262b6..16b6cfd5 100644 --- a/fail2ban/server/filterpyinotify.py +++ b/fail2ban/server/filterpyinotify.py @@ -75,7 +75,6 @@ class FilterPyinotify(FileFilter): def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__notifier = None @@ -87,7 +86,7 @@ class FilterPyinotify(FileFilter): logSys.debug("Created FilterPyinotify") def callback(self, event, origin=''): - logSys.log(7, "[%s] %sCallback for Event: %s", self.jailName, origin, event) + logSys.log(4, "[%s] %sCallback for Event: %s", self.jailName, origin, event) path = event.pathname # check watching of this path: isWF = False @@ -140,13 +139,6 @@ class FilterPyinotify(FileFilter): """ if not self.idle: self.getFailures(path) - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) - self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -173,7 +165,7 @@ class FilterPyinotify(FileFilter): return found = {} minTime = 60 - for path, (retardTM, isDir) in self.__pending.iteritems(): + for path, (retardTM, isDir) in list(self.__pending.items()): if ntm - self.__pendingChkTime < retardTM: if minTime > retardTM: minTime = retardTM continue @@ -192,10 +184,11 @@ class FilterPyinotify(FileFilter): for path, isDir in found.iteritems(): self._delPending(path) # refresh monitoring of this: - self._refreshWatcher(path, isDir=isDir) + if isDir is not None: + self._refreshWatcher(path, isDir=isDir) if isDir: # check all files belong to this dir: - for logpath in self.__watchFiles: + for logpath in list(self.__watchFiles): if logpath.startswith(path + pathsep): # if still no file - add to pending, otherwise refresh and process: if not os.path.isfile(logpath): @@ -275,28 +268,32 @@ class FilterPyinotify(FileFilter): def _addLogPath(self, path): self._addFileWatcher(path) - self._process_file(path) + # notify (wake up if in waiting): + if self.active: + self.__pendingMinTime = 0 + # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): + self._addPending(path, ('INITIAL', path), isDir=None) - ## + ## # Delete a log path # # @param path the log file to delete def _delLogPath(self, path): + self._delPending(path) if not self._delFileWatcher(path): # pragma: no cover logSys.error("Failed to remove watch on path: %s", path) - self._delPending(path) path_dir = dirname(path) - for k in self.__watchFiles: + for k in list(self.__watchFiles): if k.startswith(path_dir + pathsep): path_dir = None break if path_dir: # Remove watches for the directory # since there is no other monitored file under this directory - self._delDirWatcher(path_dir) self._delPending(path_dir) + self._delDirWatcher(path_dir) # pyinotify.ProcessEvent default handler: def __process_default(self, event): @@ -342,16 +339,26 @@ class FilterPyinotify(FileFilter): self.__notifier.process_events() # wait for events / timeout: - notify_maxtout = self.__notify_maxtout def __check_events(): - return not self.active or self.__notifier.check_events(timeout=notify_maxtout) - if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)): + return ( + not self.active + or bool(self.__notifier.check_events(timeout=self.__notify_maxtout)) + or (self.__pendingMinTime and self.__pending) + ) + wres = Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)) + if wres: if not self.active: break - self.__notifier.read_events() + if not isinstance(wres, dict): + self.__notifier.read_events() + + self.ticks += 1 # check pending files/dirs (logrotate ready): - if not self.idle: - self._checkPending() + if self.idle: + continue + self._checkPending() + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -359,10 +366,8 @@ class FilterPyinotify(FileFilter): logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) - self.ticks += 1 - logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff --git a/fail2ban/server/filtersystemd.py b/fail2ban/server/filtersystemd.py index f87fdb4e..a83b7a13 100644 --- a/fail2ban/server/filtersystemd.py +++ b/fail2ban/server/filtersystemd.py @@ -22,7 +22,7 @@ __author__ = "Steven Hiscocks" __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" -import datetime +import os import time from distutils.version import LooseVersion @@ -86,9 +86,18 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover files.extend(glob.glob(p)) args['files'] = list(set(files)) + # Default flags is SYSTEM_ONLY(4). This would lead to ignore user session files, + # so can prevent "Too many open files" errors on a lot of user sessions (see gh-2392): try: args['flags'] = int(kwargs.pop('journalflags')) except KeyError: + # be sure all journal types will be opened if files/path specified (don't set flags): + if ('files' not in args or not len(args['files'])) and ('path' not in args or not args['path']): + args['flags'] = int(os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", 4)) + + try: + args['namespace'] = kwargs.pop('namespace') + except KeyError: pass return args @@ -186,6 +195,13 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover def getJournalReader(self): return self.__journal + def getJrnEntTime(self, logentry): + """ Returns time of entry as tuple (ISO-str, Posix).""" + date = logentry.get('_SOURCE_REALTIME_TIMESTAMP') + if date is None: + date = logentry.get('__REALTIME_TIMESTAMP') + return (date.isoformat(), time.mktime(date.timetuple()) + date.microsecond/1.0E6) + ## # Format journal log entry into syslog style # @@ -208,12 +224,18 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if not v: v = logentry.get('_PID') if v: - logelements[-1] += ("[%i]" % v) + try: # [integer] (if already numeric): + v = "[%i]" % v + except TypeError: + try: # as [integer] (try to convert to int): + v = "[%i]" % int(v, 0) + except (TypeError, ValueError): # fallback - [string] as it is + v = "[%s]" % v + logelements[-1] += v logelements[-1] += ":" if logelements[-1] == "kernel:": - if '_SOURCE_MONOTONIC_TIMESTAMP' in logentry: - monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') - else: + monotonic = logentry.get('_SOURCE_MONOTONIC_TIMESTAMP') + if monotonic is None: monotonic = logentry.get('__MONOTONIC_TIMESTAMP')[0] logelements.append("[%12.6f]" % monotonic.total_seconds()) msg = logentry.get('MESSAGE','') @@ -224,19 +246,21 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover logline = " ".join(logelements) - date = logentry.get('_SOURCE_REALTIME_TIMESTAMP', - logentry.get('__REALTIME_TIMESTAMP')) + date = self.getJrnEntTime(logentry) logSys.log(5, "[%s] Read systemd journal entry: %s %s", self.jailName, - date.isoformat(), logline) + date[0], logline) ## use the same type for 1st argument: - return ((logline[:0], date.isoformat(), logline), - time.mktime(date.timetuple()) + date.microsecond/1.0E6) + return ((logline[:0], date[0] + ' ', logline.replace('\n', '\\n')), date[1]) def seekToTime(self, date): - if not isinstance(date, datetime.datetime): - date = datetime.datetime.fromtimestamp(date) + if isinstance(date, (int, long)): + date = float(date) self.__journal.seek_realtime(date) + def inOperationMode(self): + self.inOperation = True + logSys.info("[%s] Jail is in operation now (process new journal entries)", self.jailName) + ## # Main loop. # @@ -247,14 +271,40 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover if not self.getJournalMatch(): logSys.notice( - "Jail started without 'journalmatch' set. " + "[%s] Jail started without 'journalmatch' set. " "Jail regexs will be checked against all journal entries, " - "which is not advised for performance reasons.") + "which is not advised for performance reasons.", self.jailName) + + # Save current cursor position (to recognize in operation mode): + logentry = None + try: + self.__journal.seek_tail() + logentry = self.__journal.get_previous() + if logentry: + self.__journal.get_next() + except OSError: + logentry = None # Reading failure, so safe to ignore + if logentry: + # Try to obtain the last known time (position of journal) + startTime = 0 + if self.jail.database is not None: + startTime = self.jail.database.getJournalPos(self.jail, 'systemd-journal') or 0 + # Seek to max(last_known_time, now - findtime) in journal + startTime = max( startTime, MyTime.time() - int(self.getFindTime()) ) + self.seekToTime(startTime) + # Not in operation while we'll read old messages ... + self.inOperation = False + # Save current time in order to check time to switch "in operation" mode + startTime = (1, MyTime.time(), logentry.get('__CURSOR')) + else: + # empty journal or no entries for current filter: + self.inOperationMode() + # seek_tail() seems to have a bug by no entries (could bypass some entries hereafter), so seek to now instead: + startTime = MyTime.time() + self.seekToTime(startTime) + # for possible future switches of in-operation mode: + startTime = (0, startTime) - # Seek to now - findtime in journal - start_time = datetime.datetime.now() - \ - datetime.timedelta(seconds=int(self.getFindTime())) - self.seekToTime(start_time) # Move back one entry to ensure do not end up in dead space # if start time beyond end of journal try: @@ -262,18 +312,37 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover except OSError: pass # Reading failure, so safe to ignore + wcode = journal.NOP + line = None while self.active: # wait for records (or for timeout in sleeptime seconds): try: - ## todo: find better method as wait_for to break (e.g. notify) journal.wait(self.sleeptime), - ## don't use `journal.close()` for it, because in some python/systemd implementation it may - ## cause abnormal program termination - #self.__journal.wait(self.sleeptime) != journal.NOP - ## - ## wait for entries without sleep in intervals, because "sleeping" in journal.wait: - Utils.wait_for(lambda: not self.active or \ - self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.NOP, - self.sleeptime, 0.00001) + ## wait for entries using journal.wait: + if wcode == journal.NOP and self.inOperation: + ## todo: find better method as wait_for to break (e.g. notify) journal.wait(self.sleeptime), + ## don't use `journal.close()` for it, because in some python/systemd implementation it may + ## cause abnormal program termination (e. g. segfault) + ## + ## wait for entries without sleep in intervals, because "sleeping" in journal.wait, + ## journal.NOP is 0, so we can wait for non zero (APPEND or INVALIDATE): + wcode = Utils.wait_for(lambda: not self.active and journal.APPEND or \ + self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL), + self.sleeptime, 0.00001) + ## if invalidate (due to rotation, vacuuming or journal files added/removed etc): + if self.active and wcode == journal.INVALIDATE: + if self.ticks: + logSys.log(logging.DEBUG, "[%s] Invalidate signaled, take a little break (rotation ends)", self.jailName) + time.sleep(self.sleeptime * 0.25) + Utils.wait_for(lambda: not self.active or \ + self.__journal.wait(Utils.DEFAULT_SLEEP_INTERVAL) != journal.INVALIDATE, + self.sleeptime * 3, 0.00001) + if self.ticks: + # move back and forth to ensure do not end up in dead space by rotation or vacuuming, + # if position beyond end of journal (gh-3396) + try: + if self.__journal.get_previous(): self.__journal.get_next() + except OSError: + pass if self.idle: # because journal.wait will returns immediatelly if we have records in journal, # just wait a little bit here for not idle, to prevent hi-load: @@ -292,43 +361,95 @@ class FilterSystemd(JournalFilter): # pragma: systemd no cover e, exc_info=logSys.getEffectiveLevel() <= logging.DEBUG) self.ticks += 1 if logentry: - self.processLineAndAdd( - *self.formatJournalEntry(logentry)) + line, tm = self.formatJournalEntry(logentry) + # switch "in operation" mode if we'll find start entry (+ some delta): + if not self.inOperation: + if tm >= MyTime.time() - 1: # reached now (approximated): + self.inOperationMode() + elif startTime[0] == 1: + # if it reached start entry (or get read time larger than start time) + if logentry.get('__CURSOR') == startTime[2] or tm > startTime[1]: + # give the filter same time it needed to reach the start entry: + startTime = (0, MyTime.time()*2 - startTime[1]) + elif tm > startTime[1]: # reached start time (approximated): + self.inOperationMode() + # process line + self.processLineAndAdd(line, tm) self.__modified += 1 if self.__modified >= 100: # todo: should be configurable + wcode = journal.APPEND; # don't need wait - there are still unprocessed entries break else: + # "in operation" mode since we don't have messages anymore (reached end of journal): + if not self.inOperation: + self.inOperationMode() + wcode = journal.NOP; # enter wait - no more entries to process break - if self.__modified: - try: - while True: - ticket = self.failManager.toBan() - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) + self.__modified = 0 + if self.ticks % 10 == 0: + self.performSvc() + # update position in log (time and iso string): + if self.jail.database: + if line: + self._pendDBUpdates['systemd-journal'] = (tm, line[1]) + line = None + if self._pendDBUpdates and ( + self.ticks % 100 == 0 + or MyTime.time() >= self._nextUpdateTM + or not self.active + ): + self._updateDBPending() + self._nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break + wcode = journal.NOP logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) # close journal: + self.closeJournal() + + logSys.debug("[%s] filter exited (systemd)", self.jailName) + return True + + def closeJournal(self): try: - if self.__journal: - self.__journal.close() + jnl, self.__journal = self.__journal, None + if jnl: + jnl.close() except Exception as e: # pragma: no cover logSys.error("Close journal failed: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) - logSys.debug("[%s] filter exited (systemd)", self.jailName) - return True def status(self, flavor="basic"): ret = super(FilterSystemd, self).status(flavor=flavor) ret.append(("Journal matches", [" + ".join(" ".join(match) for match in self.__matches)])) return ret + + def _updateDBPending(self): + """Apply pending updates (jornal position) to database. + """ + db = self.jail.database + while True: + try: + log, args = self._pendDBUpdates.popitem() + except KeyError: + break + db.updateJournal(self.jail, log, *args) + + def onStop(self): + """Stop monitoring of journal. Invoked after run method. + """ + # close journal: + self.closeJournal() + # ensure positions of pending logs are up-to-date: + if self._pendDBUpdates and self.jail.database: + self._updateDBPending() + diff --git a/fail2ban/server/ipdns.py b/fail2ban/server/ipdns.py index 2841eac1..b435c6df 100644 --- a/fail2ban/server/ipdns.py +++ b/fail2ban/server/ipdns.py @@ -42,6 +42,32 @@ def asip(ip): return ip return IPAddr(ip) +def getfqdn(name=''): + """Get fully-qualified hostname of given host, thereby resolve of an external + IPs and name will be preferred before the local domain (or a loopback), see gh-2438 + """ + try: + name = name or socket.gethostname() + names = ( + ai[3] for ai in socket.getaddrinfo( + name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME + ) if ai[3] + ) + if names: + # first try to find a fqdn starting with the host name like www.domain.tld for www: + pref = name+'.' + first = None + for ai in names: + if ai.startswith(pref): + return ai + if not first: first = ai + # not found - simply use first known fqdn: + return first + except socket.error: + pass + # fallback to python's own getfqdn routine: + return socket.getfqdn(name) + ## # Utils class for DNS handling. @@ -64,18 +90,18 @@ class DNSUtils: if ips is not None: return ips # retrieve ips - ips = list() + ips = set() saveerr = None - for fam, ipfam in ((socket.AF_INET, IPAddr.FAM_IPv4), (socket.AF_INET6, IPAddr.FAM_IPv6)): + for fam in ((socket.AF_INET,socket.AF_INET6) if DNSUtils.IPv6IsAllowed() else (socket.AF_INET,)): try: for result in socket.getaddrinfo(dns, None, fam, 0, socket.IPPROTO_TCP): # if getaddrinfo returns something unexpected: if len(result) < 4 or not len(result[4]): continue # get ip from `(2, 1, 6, '', ('127.0.0.1', 0))`,be sure we've an ip-string # (some python-versions resp. host configurations causes returning of integer there): - ip = IPAddr(str(result[4][0]), ipfam) + ip = IPAddr(str(result[4][0]), IPAddr._AF2FAM(fam)) if ip.isValid: - ips.append(ip) + ips.add(ip) except Exception as e: saveerr = e if not ips and saveerr: @@ -103,19 +129,19 @@ class DNSUtils: def textToIp(text, useDns): """ Return the IP of DNS found in a given text. """ - ipList = list() + ipList = set() # Search for plain IP plainIP = IPAddr.searchIP(text) if plainIP is not None: ip = IPAddr(plainIP) if ip.isValid: - ipList.append(ip) + ipList.add(ip) # If we are allowed to resolve -- give it a try if nothing was found if useDns in ("yes", "warn") and not ipList: # Try to get IP from possible DNS ip = DNSUtils.dnsToIp(text) - ipList.extend(ip) + ipList.update(ip) if ip and useDns == "warn": logSys.warning("Determined IP using DNS Lookup: %s = %s", text, ipList) @@ -128,54 +154,145 @@ class DNSUtils: # try find cached own hostnames (this tuple-key cannot be used elsewhere): key = ('self','hostname', fqdn) name = DNSUtils.CACHE_ipToName.get(key) + if name is not None: + return name # get it using different ways (hostname, fully-qualified or vice versa): - if name is None: - name = '' - for hostname in ( - (socket.getfqdn, socket.gethostname) if fqdn else (socket.gethostname, socket.getfqdn) - ): - try: - name = hostname() - break - except Exception as e: # pragma: no cover - logSys.warning("Retrieving own hostnames failed: %s", e) + name = '' + for hostname in ( + (getfqdn, socket.gethostname) if fqdn else (socket.gethostname, getfqdn) + ): + try: + name = hostname() + break + except Exception as e: # pragma: no cover + logSys.warning("Retrieving own hostnames failed: %s", e) # cache and return : DNSUtils.CACHE_ipToName.set(key, name) return name + # key find cached own hostnames (this tuple-key cannot be used elsewhere): + _getSelfNames_key = ('self','dns') + @staticmethod def getSelfNames(): """Get own host names of self""" - # try find cached own hostnames (this tuple-key cannot be used elsewhere): - key = ('self','dns') - names = DNSUtils.CACHE_ipToName.get(key) + # try find cached own hostnames: + names = DNSUtils.CACHE_ipToName.get(DNSUtils._getSelfNames_key) + if names is not None: + return names # get it using different ways (a set with names of localhost, hostname, fully qualified): - if names is None: - names = set([ - 'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True) - ]) - set(['']) # getHostname can return '' + names = set([ + 'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True) + ]) - set(['']) # getHostname can return '' # cache and return : - DNSUtils.CACHE_ipToName.set(key, names) + DNSUtils.CACHE_ipToName.set(DNSUtils._getSelfNames_key, names) return names + # key to find cached network interfaces IPs (this tuple-key cannot be used elsewhere): + _getNetIntrfIPs_key = ('netintrf','ips') + + @staticmethod + def getNetIntrfIPs(): + """Get own IP addresses of self""" + # to find cached own IPs: + ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getNetIntrfIPs_key) + if ips is not None: + return ips + # try to obtain from network interfaces if possible (implemented for this platform): + try: + ips = IPAddrSet([a for ni, a in DNSUtils._NetworkInterfacesAddrs()]) + except: + ips = IPAddrSet() + # cache and return : + DNSUtils.CACHE_nameToIp.set(DNSUtils._getNetIntrfIPs_key, ips) + return ips + + # key to find cached own IPs (this tuple-key cannot be used elsewhere): + _getSelfIPs_key = ('self','ips') + @staticmethod def getSelfIPs(): """Get own IP addresses of self""" - # try find cached own IPs (this tuple-key cannot be used elsewhere): - key = ('self','ips') - ips = DNSUtils.CACHE_nameToIp.get(key) - # get it using different ways (a set with IPs of localhost, hostname, fully qualified): - if ips is None: - ips = set() - for hostname in DNSUtils.getSelfNames(): - try: - ips |= set(DNSUtils.textToIp(hostname, 'yes')) - except Exception as e: # pragma: no cover - logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) + # to find cached own IPs: + ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getSelfIPs_key) + if ips is not None: + return ips + # firstly try to obtain from network interfaces if possible (implemented for this platform): + ips = IPAddrSet(DNSUtils.getNetIntrfIPs()) + # extend it using different ways (a set with IPs of localhost, hostname, fully qualified): + for hostname in DNSUtils.getSelfNames(): + try: + ips |= IPAddrSet(DNSUtils.dnsToIp(hostname)) + except Exception as e: # pragma: no cover + logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) # cache and return : - DNSUtils.CACHE_nameToIp.set(key, ips) + DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips) return ips + _IPv6IsAllowed = None + + @staticmethod + def _IPv6IsSupportedBySystem(): + if not socket.has_ipv6: + return False + # try to check sysctl net.ipv6.conf.all.disable_ipv6: + try: + with open('/proc/sys/net/ipv6/conf/all/disable_ipv6', 'rb') as f: + # if 1 - disabled, 0 - enabled + return not int(f.read()) + except: + pass + s = None + try: + # try to create INET6 socket: + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + # bind it to free port for any interface supporting IPv6: + s.bind(("", 0)); + return True + except Exception as e: # pragma: no cover + if hasattr(e, 'errno'): + import errno + # negative (-9 'Address family not supported', etc) or not available/supported: + if e.errno < 0 or e.errno in (errno.EADDRNOTAVAIL, errno.EAFNOSUPPORT): + return False + # in use: + if e.errno in (errno.EADDRINUSE, errno.EACCES): # normally unreachable (free port and root) + return True + finally: + if s: s.close() + # unable to detect: + return None + + @staticmethod + def setIPv6IsAllowed(value): + DNSUtils._IPv6IsAllowed = value + logSys.debug("IPv6 is %s", ('on' if value else 'off') if value is not None else 'auto') + return value + + # key to find cached value of IPv6 allowance (this tuple-key cannot be used elsewhere): + _IPv6IsAllowed_key = ('self','ipv6-allowed') + + @staticmethod + def IPv6IsAllowed(): + if DNSUtils._IPv6IsAllowed is not None: + return DNSUtils._IPv6IsAllowed + v = DNSUtils.CACHE_nameToIp.get(DNSUtils._IPv6IsAllowed_key) + if v is not None: + return v + v = DNSUtils._IPv6IsSupportedBySystem() + if v is None: + # detect by IPs of host: + ips = DNSUtils.getNetIntrfIPs() + if not ips: + DNSUtils._IPv6IsAllowed = True; # avoid self recursion from getSelfIPs -> dnsToIp -> IPv6IsAllowed + try: + ips = DNSUtils.getSelfIPs() + finally: + DNSUtils._IPv6IsAllowed = None + v = any((':' in ip.ntoa) for ip in ips) + DNSUtils.CACHE_nameToIp.set(DNSUtils._IPv6IsAllowed_key, v) + return v + ## # Class for IP address handling. @@ -197,14 +314,23 @@ class IPAddr(object): __slots__ = '_family','_addr','_plen','_maskplen','_raw' # todo: make configurable the expired time and max count of cache entries: - CACHE_OBJ = Utils.Cache(maxCount=1000, maxTime=5*60) + CACHE_OBJ = Utils.Cache(maxCount=10000, maxTime=5*60) CIDR_RAW = -2 CIDR_UNSPEC = -1 FAM_IPv4 = CIDR_RAW - socket.AF_INET FAM_IPv6 = CIDR_RAW - socket.AF_INET6 + @staticmethod + def _AF2FAM(v): + return IPAddr.CIDR_RAW - v def __new__(cls, ipstr, cidr=CIDR_UNSPEC): + if cidr == IPAddr.CIDR_UNSPEC and isinstance(ipstr, (tuple, list)): + cidr = IPAddr.CIDR_RAW + if cidr == IPAddr.CIDR_RAW: # don't cache raw + ip = super(IPAddr, cls).__new__(cls) + ip.__init(ipstr, cidr) + return ip # check already cached as IPAddr args = (ipstr, cidr) ip = IPAddr.CACHE_OBJ.get(args) @@ -221,7 +347,8 @@ class IPAddr(object): return ip ip = super(IPAddr, cls).__new__(cls) ip.__init(ipstr, cidr) - IPAddr.CACHE_OBJ.set(args, ip) + if ip._family != IPAddr.CIDR_RAW: + IPAddr.CACHE_OBJ.set(args, ip) return ip @staticmethod @@ -301,7 +428,7 @@ class IPAddr(object): return repr(self.ntoa) def __str__(self): - return self.ntoa + return self.ntoa if isinstance(self.ntoa, basestring) else str(self.ntoa) def __reduce__(self): """IPAddr pickle-handler, that simply wraps IPAddr to the str @@ -343,6 +470,12 @@ class IPAddr(object): """ return self._family != socket.AF_UNSPEC + @property + def isSingle(self): + """Returns whether the object is a single IP address (not DNS and subnet) + """ + return self._plen == {socket.AF_INET: 32, socket.AF_INET6: 128}.get(self._family, -1000) + def __eq__(self, other): if self._family == IPAddr.CIDR_RAW and not isinstance(other, IPAddr): return self._raw == other @@ -475,6 +608,14 @@ class IPAddr(object): return (self.addr & mask) == net.addr + def contains(self, ip): + """Return whether the object (as network) contains given IP + """ + return isinstance(ip, IPAddr) and (ip == self or ip.isInNet(self)) + + def __contains__(self, ip): + return self.contains(ip) + # Pre-calculated map: addr to maskplen def __getMaskMap(): m6 = (1 << 128)-1 @@ -524,3 +665,135 @@ class IPAddr(object): # An IPv4 compatible IPv6 to be reused IPAddr.IP6_4COMPAT = IPAddr("::ffff:0:0", 96) + + +class IPAddrSet(set): + + hasSubNet = False + + def __init__(self, ips=[]): + ips2 = set() + for ip in ips: + if not isinstance(ip, IPAddr): ip = IPAddr(ip) + ips2.add(ip) + self.hasSubNet |= not ip.isSingle + set.__init__(self, ips2) + + def add(self, ip): + if not isinstance(ip, IPAddr): ip = IPAddr(ip) + self.hasSubNet |= not ip.isSingle + set.add(self, ip) + + def __contains__(self, ip): + if not isinstance(ip, IPAddr): ip = IPAddr(ip) + # IP can be found directly or IP is in each subnet: + return set.__contains__(self, ip) or (self.hasSubNet and any(n.contains(ip) for n in self)) + + +def _NetworkInterfacesAddrs(withMask=False): + + # Closure implementing lazy load modules and libc and define _NetworkInterfacesAddrs on demand: + # Currently tested on Linux only (TODO: implement for MacOS, Solaris, etc) + try: + from ctypes import ( + Structure, Union, POINTER, + pointer, get_errno, cast, + c_ushort, c_byte, c_void_p, c_char_p, c_uint, c_int, c_uint16, c_uint32 + ) + import ctypes.util + import ctypes + + class struct_sockaddr(Structure): + _fields_ = [ + ('sa_family', c_ushort), + ('sa_data', c_byte * 14),] + + class struct_sockaddr_in(Structure): + _fields_ = [ + ('sin_family', c_ushort), + ('sin_port', c_uint16), + ('sin_addr', c_byte * 4)] + + class struct_sockaddr_in6(Structure): + _fields_ = [ + ('sin6_family', c_ushort), + ('sin6_port', c_uint16), + ('sin6_flowinfo', c_uint32), + ('sin6_addr', c_byte * 16), + ('sin6_scope_id', c_uint32)] + + class union_ifa_ifu(Union): + _fields_ = [ + ('ifu_broadaddr', POINTER(struct_sockaddr)), + ('ifu_dstaddr', POINTER(struct_sockaddr)),] + + class struct_ifaddrs(Structure): + pass + struct_ifaddrs._fields_ = [ + ('ifa_next', POINTER(struct_ifaddrs)), + ('ifa_name', c_char_p), + ('ifa_flags', c_uint), + ('ifa_addr', POINTER(struct_sockaddr)), + ('ifa_netmask', POINTER(struct_sockaddr)), + ('ifa_ifu', union_ifa_ifu), + ('ifa_data', c_void_p),] + + libc = ctypes.CDLL(ctypes.util.find_library('c') or "") + if not libc.getifaddrs: # pragma: no cover + raise NotImplementedError('libc.getifaddrs is not available') + + def ifap_iter(ifap): + ifa = ifap.contents + while True: + yield ifa + if not ifa.ifa_next: + break + ifa = ifa.ifa_next.contents + + def getfamaddr(ifa, withMask=False): + sa = ifa.ifa_addr.contents + fam = sa.sa_family + if fam == socket.AF_INET: + sa = cast(pointer(sa), POINTER(struct_sockaddr_in)).contents + addr = socket.inet_ntop(fam, sa.sin_addr) + if withMask: + nm = ifa.ifa_netmask.contents + if nm is not None and nm.sa_family == socket.AF_INET: + nm = cast(pointer(nm), POINTER(struct_sockaddr_in)).contents + addr += '/'+socket.inet_ntop(fam, nm.sin_addr) + return IPAddr(addr) + elif fam == socket.AF_INET6: + sa = cast(pointer(sa), POINTER(struct_sockaddr_in6)).contents + addr = socket.inet_ntop(fam, sa.sin6_addr) + if withMask: + nm = ifa.ifa_netmask.contents + if nm is not None and nm.sa_family == socket.AF_INET6: + nm = cast(pointer(nm), POINTER(struct_sockaddr_in6)).contents + addr += '/'+socket.inet_ntop(fam, nm.sin6_addr) + return IPAddr(addr) + return None + + def _NetworkInterfacesAddrs(withMask=False): + ifap = POINTER(struct_ifaddrs)() + result = libc.getifaddrs(pointer(ifap)) + if result != 0: + raise OSError(get_errno()) + del result + try: + for ifa in ifap_iter(ifap): + name = ifa.ifa_name.decode("UTF-8") + addr = getfamaddr(ifa, withMask) + if addr: + yield name, addr + finally: + libc.freeifaddrs(ifap) + + except Exception as e: # pragma: no cover + _init_error = NotImplementedError(e) + def _NetworkInterfacesAddrs(): + raise _init_error + + DNSUtils._NetworkInterfacesAddrs = staticmethod(_NetworkInterfacesAddrs); + return _NetworkInterfacesAddrs(withMask) + +DNSUtils._NetworkInterfacesAddrs = staticmethod(_NetworkInterfacesAddrs); diff --git a/fail2ban/server/jail.py b/fail2ban/server/jail.py index c6d61856..2c84e475 100644 --- a/fail2ban/server/jail.py +++ b/fail2ban/server/jail.py @@ -161,6 +161,10 @@ class Jail(object): """ return self.__db + @database.setter + def database(self, value): + self.__db = value; + @property def filter(self): """The filter which the jail is using to monitor log files. @@ -192,6 +196,12 @@ class Jail(object): ("Actions", self.actions.status(flavor=flavor)), ] + @property + def hasFailTickets(self): + """Retrieve whether queue has tickets to ban. + """ + return not self.__queue.empty() + def putFailTicket(self, ticket): """Add a fail ticket to the jail. @@ -281,11 +291,11 @@ class Jail(object): # use ban time as search time if we have not enabled a increasing: forbantime = self.actions.getBanTime() for ticket in self.database.getCurrentBans(jail=self, forbantime=forbantime, - correctBanTime=correctBanTime + correctBanTime=correctBanTime, maxmatches=self.filter.failManager.maxMatches ): try: #logSys.debug('restored ticket: %s', ticket) - if self.filter.inIgnoreIPList(ticket.getIP(), log_ignore=True): continue + if self.filter.inIgnoreIPList(ticket.getID(), log_ignore=True): continue # mark ticked was restored from database - does not put it again into db: ticket.restored = True # correct start time / ban time (by the same end of ban): diff --git a/fail2ban/server/jails.py b/fail2ban/server/jails.py index 972a8c4b..eaaa9518 100644 --- a/fail2ban/server/jails.py +++ b/fail2ban/server/jails.py @@ -22,7 +22,10 @@ __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2013- Yaroslav Halchenko" __license__ = "GPL" from threading import Lock -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from ..exceptions import DuplicateJailException, UnknownJailException from .jail import Jail @@ -64,8 +67,7 @@ class Jails(Mapping): """ with self.__lock: if name in self._jails: - if noduplicates: - raise DuplicateJailException(name) + raise DuplicateJailException(name) else: self._jails[name] = Jail(name, backend, db) diff --git a/fail2ban/server/jailthread.py b/fail2ban/server/jailthread.py index 5c7afd38..67955a06 100644 --- a/fail2ban/server/jailthread.py +++ b/fail2ban/server/jailthread.py @@ -29,7 +29,7 @@ from threading import Thread from abc import abstractmethod from .utils import Utils -from ..helpers import excepthook +from ..helpers import excepthook, prctl_set_th_name class JailThread(Thread): @@ -67,6 +67,8 @@ class JailThread(Thread): def run_with_except_hook(*args, **kwargs): try: run(*args, **kwargs) + # call on stop callback to do some finalizations: + self.onStop() except Exception as e: # avoid very sporadic error "'NoneType' object has no attribute 'exc_info'" (https://bugs.python.org/issue7336) # only extremely fast systems are affected ATM (2.7 / 3.x), if thread ends nothing is available here. @@ -76,6 +78,15 @@ class JailThread(Thread): print(e) self.run = run_with_except_hook + if sys.version_info >= (3,): # pragma: 2.x no cover + def _bootstrap(self): + prctl_set_th_name(self.name) + return super(JailThread, self)._bootstrap(); + else: # pragma: 3.x no cover + def __bootstrap(self): + prctl_set_th_name(self.name) + return Thread._Thread__bootstrap(self) + @abstractmethod def status(self, flavor="basic"): # pragma: no cover - abstract """Abstract - Should provide status information. @@ -88,6 +99,12 @@ class JailThread(Thread): self.active = True super(JailThread, self).start() + @abstractmethod + def onStop(self): # pragma: no cover - absract + """Abstract - Called when thread ends (after run). + """ + pass + def stop(self): """Sets `active` property to False, to flag run method to return. """ @@ -108,4 +125,9 @@ class JailThread(Thread): if self.active is not None: super(JailThread, self).join() - +## python 2.x replace binding of private __bootstrap method: +if sys.version_info < (3,): # pragma: 3.x no cover + JailThread._Thread__bootstrap = JailThread._JailThread__bootstrap +## python 3.9, restore isAlive method: +elif not hasattr(JailThread, 'isAlive'): # pragma: 2.x no cover + JailThread.isAlive = JailThread.is_alive diff --git a/fail2ban/server/mytime.py b/fail2ban/server/mytime.py index 49199887..315d8a30 100644 --- a/fail2ban/server/mytime.py +++ b/fail2ban/server/mytime.py @@ -113,6 +113,19 @@ class MyTime: return time.localtime(x) else: return time.localtime(MyTime.myTime) + + @staticmethod + def time2str(unixTime, format="%Y-%m-%d %H:%M:%S"): + """Convert time to a string representing as date and time using given format. + Default format is ISO 8601, YYYY-MM-DD HH:MM:SS without microseconds. + + @return ISO-capable string representation of given unixTime + """ + # consider end of 9999th year (in GMT+23 to avoid year overflow in other TZ) + dt = datetime.datetime.fromtimestamp( + unixTime).replace(microsecond=0 + ) if unixTime < 253402214400 else datetime.datetime(9999, 12, 31, 23, 59, 59) + return dt.strftime(format) ## precreate/precompile primitives used in str2seconds: @@ -161,3 +174,62 @@ class MyTime: val = rexp.sub(rpl, val) val = MyTime._str2sec_fini.sub(r"\1+\2", val) return eval(val) + + class seconds2str(): + """Converts seconds to string on demand (if string representation needed). + Ex: seconds2str(86400*390) = 1y 3w 4d + seconds2str(86400*368) = 1y 3d + seconds2str(86400*365.5) = 1y + seconds2str(86400*2+3600*7+60*15) = 2d 7h 15m + seconds2str(86400*2+3599) = 2d 1h + seconds2str(3600-5) = 1h + seconds2str(3600-10) = 59m 50s + seconds2str(59) = 59s + """ + def __init__(self, sec): + self.sec = sec + def __str__(self): + # s = str(datetime.timedelta(seconds=int(self.sec))) + # return s if s[-3:] != ":00" else s[:-3] + s = self.sec; c = 3 + # automatic accuracy: round by large values (and maximally 3 groups) + if s >= 31536000: # a year as 365*24*60*60 (don't need to consider leap year by this accuracy) + s = int(round(float(s)/86400)) # round by a day + r = str(s//365) + 'y '; s %= 365 + if s >= 7: + r += str(s//7) + 'w '; s %= 7 + if s: + r += str(s) + 'd ' + return r[:-1] + if s >= 604800: # a week as 24*60*60*7 + s = int(round(float(s)/3600)) # round by a hour + r = str(s//168) + 'w '; s %= 168 + if s >= 24: + r += str(s//24) + 'd '; s %= 24 + if s: + r += str(s) + 'h ' + return r[:-1] + if s >= 86400: # a day as 24*60*60 + s = int(round(float(s)/60)) # round by a minute + r = str(s//1440) + 'd '; s %= 1440 + if s >= 60: + r += str(s//60) + 'h '; s %= 60 + if s: + r += str(s) + 'm ' + return r[:-1] + if s >= 3595: # a hour as 60*60 (- 5 seconds) + s = int(round(float(s)/10)) # round by 10 seconds + r = str(s//360) + 'h '; s %= 360 + if s >= 6: # a minute + r += str(s//6) + 'm '; s %= 6 + return r[:-1] + r = '' + if s >= 60: # a minute + r += str(s//60) + 'm '; s %= 60 + if s: # remaining seconds + r += str(s) + 's ' + elif not self.sec: # 0s + r = '0 ' + return r[:-1] + def __repr__(self): + return self.__str__() diff --git a/fail2ban/server/observer.py b/fail2ban/server/observer.py index c3fa7d54..b1c9b37d 100644 --- a/fail2ban/server/observer.py +++ b/fail2ban/server/observer.py @@ -62,7 +62,7 @@ class ObserverThread(JailThread): def __init__(self): # init thread - super(ObserverThread, self).__init__(name='Observer') + super(ObserverThread, self).__init__(name='f2b/observer') # before started - idle: self.idle = True ## Event queue @@ -87,7 +87,7 @@ class ObserverThread(JailThread): except KeyError: raise KeyError("Invalid event index : %s" % i) - def __delitem__(self, name): + def __delitem__(self, i): try: del self._queue[i] except KeyError: @@ -146,9 +146,11 @@ class ObserverThread(JailThread): def pulse_notify(self): """Notify wakeup (sets /and resets/ notify event) """ - if not self._paused and self._notify: - self._notify.set() - #self._notify.clear() + if not self._paused: + n = self._notify + if n: + n.set() + #n.clear() def add(self, *event): """Add a event to queue and notify thread to wake up. @@ -230,13 +232,14 @@ class ObserverThread(JailThread): if self._paused: continue else: - ## notify event deleted (shutdown) - just sleep a litle bit (waiting for shutdown events, prevent high cpu usage) + ## notify event deleted (shutdown) - just sleep a little bit (waiting for shutdown events, prevent high cpu usage) time.sleep(ObserverThread.DEFAULT_SLEEP_INTERVAL) ## stop by shutdown and empty queue : if not self.is_full: break ## end of main loop - exit logSys.info("Observer stopped, %s events remaining.", len(self._queue)) + self._notify = None #print("Observer stopped, %s events remaining." % len(self._queue)) except Exception as e: logSys.error('Observer stopped after error: %s', e, exc_info=True) @@ -262,9 +265,8 @@ class ObserverThread(JailThread): if not self.active: super(ObserverThread, self).start() - def stop(self): + def stop(self, wtime=5, forceQuit=True): if self.active and self._notify: - wtime = 5 logSys.info("Observer stop ... try to end queue %s seconds", wtime) #print("Observer stop ....") # just add shutdown job to make possible wait later until full (events remaining) @@ -276,10 +278,15 @@ class ObserverThread(JailThread): #self.pulse_notify() self._notify = None # wait max wtime seconds until full (events remaining) - self.wait_empty(wtime) - n.clear() - self.active = False - self.wait_idle(0.5) + if self.wait_empty(wtime) or forceQuit: + n.clear() + self.active = False; # leave outer (active) loop + self._paused = True; # leave inner (queue) loop + self.__db = None + else: + self._notify = n + return self.wait_idle(min(wtime, 0.5)) and not self.is_full + return True @property def is_full(self): @@ -357,15 +364,15 @@ class ObserverThread(JailThread): ## [Async] ban time increment functionality ... ## ----------------------------------------- - def failureFound(self, failManager, jail, ticket): + def failureFound(self, jail, ticket): """ Notify observer a failure for ip was found Observer will check ip was known (bad) and possibly increase an retry count """ # check jail active : - if not jail.isAlive(): + if not jail.isAlive() or not jail.getBanTimeExtra("increment"): return - ip = ticket.getIP() + ip = ticket.getID() unixTime = ticket.getTime() logSys.debug("[%s] Observer: failure found %s", jail.name, ip) # increase retry count for known (bad) ip, corresponding banCount of it (one try will count than 2, 3, 5, 9 ...) : @@ -373,7 +380,7 @@ class ObserverThread(JailThread): retryCount = 1 timeOfBan = None try: - maxRetry = failManager.getMaxRetry() + maxRetry = jail.filter.failManager.getMaxRetry() db = jail.database if db is not None: for banCount, timeOfBan, lastBanTime in db.getBan(ip, jail): @@ -393,21 +400,15 @@ class ObserverThread(JailThread): return # retry counter was increased - add it again: logSys.info("[%s] Found %s, bad - %s, %s # -> %s%s", jail.name, ip, - datetime.datetime.fromtimestamp(unixTime).strftime("%Y-%m-%d %H:%M:%S"), banCount, retryCount, + MyTime.time2str(unixTime), banCount, retryCount, (', Ban' if retryCount >= maxRetry else '')) # retryCount-1, because a ticket was already once incremented by filter self - retryCount = failManager.addFailure(ticket, retryCount - 1, True) + retryCount = jail.filter.failManager.addFailure(ticket, retryCount - 1, True) ticket.setBanCount(banCount) # after observe we have increased attempt count, compare it >= maxretry ... if retryCount >= maxRetry: # perform the banning of the IP now (again) - # [todo]: this code part will be used multiple times - optimize it later. - try: # pragma: no branch - exception is the only way out - while True: - ticket = failManager.toBan(ip) - jail.putFailTicket(ticket) - except FailManagerEmpty: - failManager.cleanup(MyTime.time()) + jail.filter.performBan(ip) except Exception as e: logSys.error('%s', e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) @@ -434,7 +435,7 @@ class ObserverThread(JailThread): if not jail.isAlive() or not jail.database: return banTime be = jail.getBanTimeExtra() - ip = ticket.getIP() + ip = ticket.getID() orgBanTime = banTime # check ip was already banned (increment time of ban): try: @@ -454,8 +455,8 @@ class ObserverThread(JailThread): # check current ticket time to prevent increasing for twice read tickets (restored from log file besides database after restart) if ticket.getTime() > timeOfBan: logSys.info('[%s] IP %s is bad: %s # last %s - incr %s to %s' % (jail.name, ip, banCount, - datetime.datetime.fromtimestamp(timeOfBan).strftime("%Y-%m-%d %H:%M:%S"), - datetime.timedelta(seconds=int(orgBanTime)), datetime.timedelta(seconds=int(banTime)))); + MyTime.time2str(timeOfBan), + MyTime.seconds2str(orgBanTime), MyTime.seconds2str(banTime))) else: ticket.restored = True break @@ -473,7 +474,7 @@ class ObserverThread(JailThread): return try: oldbtime = btime - ip = ticket.getIP() + ip = ticket.getID() logSys.debug("[%s] Observer: ban found %s, %s", jail.name, ip, btime) # if not permanent and ban time was not set - check time should be increased: if btime != -1 and ticket.getBanTime() is None: @@ -484,8 +485,7 @@ class ObserverThread(JailThread): # if not permanent if btime != -1: bendtime = ticket.getTime() + btime - logtime = (datetime.timedelta(seconds=int(btime)), - datetime.datetime.fromtimestamp(bendtime).strftime("%Y-%m-%d %H:%M:%S")) + logtime = (MyTime.seconds2str(btime), MyTime.time2str(bendtime)) # check ban is not too old : if bendtime < MyTime.time(): logSys.debug('Ignore old bantime %s', logtime[1]) @@ -514,7 +514,7 @@ class ObserverThread(JailThread): """ try: btime = ticket.getBanTime() - ip = ticket.getIP() + ip = ticket.getID() logSys.debug("[%s] Observer: prolong %s, %s", jail.name, ip, btime) # prolong ticket via actions that expected this: jail.actions._prolongBan(ticket) diff --git a/fail2ban/server/server.py b/fail2ban/server/server.py index dfbbd5d7..660f7918 100644 --- a/fail2ban/server/server.py +++ b/fail2ban/server/server.py @@ -34,12 +34,12 @@ import sys from .observer import Observers, ObserverThread from .jails import Jails -from .filter import FileFilter, JournalFilter +from .filter import DNSUtils, FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version from ..helpers import getLogger, _as_bool, extractOptions, str2LogLevel, \ - getVerbosityFormat, excepthook + getVerbosityFormat, excepthook, prctl_set_th_name # Gets the instance of the logger. logSys = getLogger(__name__) @@ -58,6 +58,23 @@ except ImportError: # pragma: no cover def _thread_name(): return threading.current_thread().__class__.__name__ +try: + FileExistsError +except NameError: # pragma: 3.x no cover + FileExistsError = OSError + +def _make_file_path(name): + """Creates path of file (last level only) on demand""" + name = os.path.dirname(name) + # only if it is absolute (e. g. important for socket, so if unix path): + if os.path.isabs(name): + # be sure path exists (create last level of directory on demand): + try: + os.mkdir(name) + except (OSError, FileExistsError) as e: + if e.errno != 17: # pragma: no cover - not EEXIST is not covered + raise + class Server: @@ -97,7 +114,7 @@ class Server: def start(self, sock, pidfile, force=False, observer=True, conf={}): # First set the mask to only allow access to owner - os.umask(0077) + os.umask(0o077) # Second daemonize before logging etc, because it will close all handles: if self.__daemon: # pragma: no cover logSys.info("Starting in daemon mode") @@ -111,6 +128,9 @@ class Server: logSys.error(err) raise ServerInitializationError(err) # We are daemon. + + # replace main thread (and process) name to identify server (for top/ps/pstree or diagnostic): + prctl_set_th_name(conf.get("pname", "fail2ban-server")) # Set all logging parameters (or use default if not specified): self.__verbose = conf.get("verbose", None) @@ -139,6 +159,7 @@ class Server: # Creates a PID file. try: logSys.debug("Creating PID file %s", pidfile) + _make_file_path(pidfile) pidFile = open(pidfile, 'w') pidFile.write("%s\n" % os.getpid()) pidFile.close() @@ -154,6 +175,7 @@ class Server: # Start the communication logSys.debug("Starting communication") try: + _make_file_path(sock) self.__asyncServer = AsyncServer(self.__transm) self.__asyncServer.onstart = conf.get('onstart') self.__asyncServer.start(sock, force) @@ -191,23 +213,26 @@ class Server: signal.signal(s, sh) # Give observer a small chance to complete its work before exit - if Observers.Main is not None: - Observers.Main.stop() + obsMain = Observers.Main + if obsMain is not None: + if obsMain.stop(forceQuit=False): + obsMain = None + Observers.Main = None # Now stop all the jails self.stopAllJail() + # Stop observer ultimately + if obsMain is not None: + obsMain.stop() + # Explicit close database (server can leave in a thread, # so delayed GC can prevent commiting changes) if self.__db: self.__db.close() self.__db = None - # Stop observer and exit - if Observers.Main is not None: - Observers.Main.stop() - Observers.Main = None - # Stop async + # Stop async and exit if self.__asyncServer is not None: self.__asyncServer.stop() self.__asyncServer = None @@ -268,6 +293,11 @@ class Server: for name in self.__jails.keys(): self.delJail(name, stop=False, join=True) + def clearCaches(self): + # we need to clear caches, to be able to recognize new IPs/families etc: + DNSUtils.CACHE_nameToIp.clear() + DNSUtils.CACHE_ipToName.clear() + def reloadJails(self, name, opts, begin): if begin: # begin reload: @@ -289,6 +319,8 @@ class Server: if "--restart" in opts: self.stopJail(name) else: + # invalidate caches by reload + self.clearCaches() # first unban all ips (will be not restored after (re)start): if "--unban" in opts: self.setUnbanIP() @@ -359,7 +391,7 @@ class Server: if isinstance(filter_, FileFilter): return filter_.getLogPaths() else: # pragma: systemd no cover - logSys.info("Jail %s is not a FileFilter instance" % name) + logSys.debug("Jail %s is not a FileFilter instance" % name) return [] def addJournalMatch(self, name, match): # pragma: systemd no cover @@ -377,7 +409,7 @@ class Server: if isinstance(filter_, JournalFilter): return filter_.getJournalMatch() else: - logSys.info("Jail %s is not a JournalFilter instance" % name) + logSys.debug("Jail %s is not a JournalFilter instance" % name) return [] def setLogEncoding(self, name, encoding): @@ -459,6 +491,12 @@ class Server: def getUseDns(self, name): return self.__jails[name].filter.getUseDns() + def setMaxMatches(self, name, value): + self.__jails[name].filter.failManager.maxMatches = value + + def getMaxMatches(self, name): + return self.__jails[name].filter.failManager.maxMatches + def setMaxRetry(self, name, value): self.__jails[name].filter.setMaxRetry(value) @@ -489,27 +527,70 @@ class Server: def setBanTime(self, name, value): self.__jails[name].actions.setBanTime(value) + def addAttemptIP(self, name, *args): + return self.__jails[name].filter.addAttempt(*args) + def setBanIP(self, name, value): - return self.__jails[name].filter.addBannedIP(value) - - def setUnbanIP(self, name=None, value=None): + return self.__jails[name].actions.addBannedIP(value) + + def setUnbanIP(self, name=None, value=None, ifexists=True): if name is not None: - # in all jails: + # single jail: jails = [self.__jails[name]] else: - # single jail: + # in all jails: jails = self.__jails.values() # unban given or all (if value is None): cnt = 0 + ifexists |= (name is None) for jail in jails: - cnt += jail.actions.removeBannedIP(value, ifexists=(name is None)) - if value and not cnt: - logSys.info("%s is not banned", value) + cnt += jail.actions.removeBannedIP(value, ifexists=ifexists) return cnt + def banned(self, name=None, ids=None): + if name is not None: + # single jail: + jails = [self.__jails[name]] + else: + # in all jails: + jails = self.__jails.values() + # check banned ids: + res = [] + if name is None and ids: + for ip in ids: + ret = [] + for jail in jails: + if jail.actions.getBanned([ip]): + ret.append(jail.name) + res.append(ret) + else: + for jail in jails: + ret = jail.actions.getBanned(ids) + if name is not None: + return ret + res.append(ret) + else: + res.append({jail.name: ret}) + return res + def getBanTime(self, name): return self.__jails[name].actions.getBanTime() + def getBanList(self, name, withTime=False): + """Returns the list of banned IP addresses for a jail. + + Parameters + ---------- + name : str + The name of a jail. + + Returns + ------- + list + The list of banned IP addresses. + """ + return self.__jails[name].actions.getBanList(withTime) + def setBanTimeExtra(self, name, opt, value): self.__jails[name].setBanTimeExtra(opt, value) @@ -597,7 +678,10 @@ class Server: return True padding = logOptions.get('padding') # set a format which is simpler for console use - if systarget == "SYSLOG": + if systarget == "SYSTEMD-JOURNAL": + from systemd.journal import JournalHandler + hdlr = JournalHandler(SYSLOG_IDENTIFIER='fail2ban') + elif systarget == "SYSLOG": facility = logOptions.get('facility', 'DAEMON').upper() # backwards compatibility - default no padding for syslog handler: if padding is None: padding = '0' @@ -647,9 +731,7 @@ class Server: except (ValueError, KeyError): # pragma: no cover # Is known to be thrown after logging was shutdown once # with older Pythons -- seems to be safe to ignore there - # At least it was still failing on 2.6.2-0ubuntu1 (jaunty) - if (2, 6, 3) <= sys.version_info < (3,) or \ - (3, 2) <= sys.version_info: + if sys.version_info < (3,) or sys.version_info >= (3, 2): raise # detailed format by deep log levels (as DEBUG=10): if logger.getEffectiveLevel() <= logging.DEBUG: # pragma: no cover @@ -675,7 +757,8 @@ class Server: verbose = self.__verbose-1 fmt = getVerbosityFormat(verbose, addtime=addtime, padding=padding) # tell the handler to use this format - hdlr.setFormatter(logging.Formatter(fmt)) + if target != "SYSTEMD-JOURNAL": + hdlr.setFormatter(logging.Formatter(fmt)) logger.addHandler(hdlr) # Does not display this message at startup. if self.__logTarget is not None: @@ -714,7 +797,7 @@ class Server: return self.__syslogSocket def flushLogs(self): - if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG']: + if self.__logTarget not in ['STDERR', 'STDOUT', 'SYSLOG', 'SYSTEMD-JOURNAL']: for handler in getLogger("fail2ban").handlers: try: handler.doRollover() @@ -729,6 +812,21 @@ class Server: logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + @staticmethod + def setIPv6IsAllowed(value): + value = _as_bool(value) if value != 'auto' else None + return DNSUtils.setIPv6IsAllowed(value) + + def setThreadOptions(self, value): + for o, v in value.iteritems(): + if o == 'stacksize': + threading.stack_size(int(v)*1024) + else: # pragma: no cover + raise KeyError("unknown option %r" % o) + + def getThreadOptions(self): + return {'stacksize': threading.stack_size() // 1024} + def setDatabase(self, filename): # if not changed - nothing to do if self.__db and self.__db.filename == filename: @@ -742,6 +840,7 @@ class Server: self.__db = None else: if Fail2BanDb is not None: + _make_file_path(filename) self.__db = Fail2BanDb(filename) self.__db.delAllJails() else: # pragma: no cover @@ -754,6 +853,26 @@ class Server: def getDatabase(self): return self.__db + @staticmethod + def __get_fdlist(): + """Generate a list of open file descriptors. + + This wouldn't work on some platforms, or if proc/fdescfs not mounted, or a chroot environment, + then it'd raise a FileExistsError. + """ + for path in ( + '/proc/self/fd', # Linux, Cygwin and NetBSD + '/proc/fd', # MacOS and FreeBSD + ): + if os.path.exists(path): + def fdlist(): + for name in os.listdir(path): + if name.isdigit(): + yield int(name) + return fdlist() + # other platform or unmounted, chroot etc: + raise FileExistsError("fd-list not found") + def __createDaemon(self): # pragma: no cover """ Detach a process from the controlling terminal and run it in the background as a daemon. @@ -811,25 +930,37 @@ class Server: # Signal to exit, parent of the first child. return None - # Close all open files. Try the system configuration variable, SC_OPEN_MAX, + # Close all open files. Try to obtain the range of open descriptors directly. + # As a fallback try the system configuration variable, SC_OPEN_MAX, # for the maximum number of open files to close. If it doesn't exist, use # the default value (configurable). try: - maxfd = os.sysconf("SC_OPEN_MAX") - except (AttributeError, ValueError): - maxfd = 256 # default maximum + fdlist = self.__get_fdlist() + maxfd = -1 + except: + try: + maxfd = os.sysconf("SC_OPEN_MAX") + except (AttributeError, ValueError): + maxfd = 256 # default maximum + fdlist = xrange(maxfd+1) # urandom should not be closed in Python 3.4.0. Fixed in 3.4.1 # http://bugs.python.org/issue21207 if sys.version_info[0:3] == (3, 4, 0): # pragma: no cover urandom_fd = os.open("/dev/urandom", os.O_RDONLY) - for fd in range(0, maxfd): + for fd in fdlist: try: if not os.path.sameopenfile(urandom_fd, fd): os.close(fd) except OSError: # ERROR (ignore) pass os.close(urandom_fd) + elif maxfd == -1: + for fd in fdlist: + try: + os.close(fd) + except OSError: # ERROR (ignore) + pass else: os.closerange(0, maxfd) diff --git a/fail2ban/server/strptime.py b/fail2ban/server/strptime.py index 498d284b..12be163a 100644 --- a/fail2ban/server/strptime.py +++ b/fail2ban/server/strptime.py @@ -30,17 +30,6 @@ locale_time = LocaleTime() TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?" FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,)) -def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): - """ Build century regex for last year and the next years (distance). - - Thereby respect possible run in the test-cases (alternate date used there) - """ - cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] - exprset = set( cent(now[0].year + i) for i in (-1, distance) ) - if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in (-1, distance) ) - return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset) - timeRE = TimeRE() # %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, @@ -63,20 +52,68 @@ timeRE['z'] = r"(?P<z>Z|UTC|GMT|[+-][01]\d(?::?\d{2})?)" timeRE['ExZ'] = r"(?P<Z>%s)" % (TZ_ABBR_RE,) timeRE['Exz'] = r"(?P<z>(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE) +# overwrite default patterns, since they can be non-optimal: +timeRE['d'] = r"(?P<d>[1-2]\d|[0 ]?[1-9]|3[0-1])" +timeRE['m'] = r"(?P<m>0?[1-9]|1[0-2])" +timeRE['Y'] = r"(?P<Y>\d{4})" +timeRE['H'] = r"(?P<H>[0-1]?\d|2[0-3])" +timeRE['M'] = r"(?P<M>[0-5]?\d)" +timeRE['S'] = r"(?P<S>[0-5]?\d|6[0-1])" + # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: -timeRE['Exd'] = r"(?P<d>3[0-1]|[1-2]\d|0[1-9])" -timeRE['Exm'] = r"(?P<m>1[0-2]|0[1-9])" -timeRE['ExH'] = r"(?P<H>2[0-3]|[0-1]\d)" -timeRE['Exk'] = r" ?(?P<H>2[0-3]|[0-1]\d|\d)" +timeRE['Exd'] = r"(?P<d>[1-2]\d|0[1-9]|3[0-1])" +timeRE['Exm'] = r"(?P<m>0[1-9]|1[0-2])" +timeRE['ExH'] = r"(?P<H>[0-1]\d|2[0-3])" +timeRE['Exk'] = r" ?(?P<H>[0-1]?\d|2[0-3])" timeRE['Exl'] = r" ?(?P<I>1[0-2]|\d)" timeRE['ExM'] = r"(?P<M>[0-5]\d)" -timeRE['ExS'] = r"(?P<S>6[0-1]|[0-5]\d)" -# more precise year patterns, within same century of last year and -# the next 3 years (for possible long uptime of fail2ban); thereby -# respect possible run in the test-cases (alternate date used there): -timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3) -timeRE['Exy'] = r"(?P<y>%s\d)" % _getYearCentRE(cent=(2,3), distance=3) +timeRE['ExS'] = r"(?P<S>[0-5]\d|6[0-1])" + +def _updateTimeRE(): + def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): + """ Build century regex for last year and the next years (distance). + + Thereby respect possible run in the test-cases (alternate date used there) + """ + cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) + exprset = set( cent(now[0].year + i) for i in (-1, distance) ) + if len(now) > 1 and now[1]: + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) + + # more precise year patterns, within same century of last year and + # the next 3 years (for possible long uptime of fail2ban); thereby + # consider possible run in the test-cases (alternate date used there), + # so accept years: 20xx (from test-date or 2001 up to current century) + timeRE['ExY'] = r"(?P<Y>%s\d)" % _getYearCentRE(cent=(0,3), distance=3, + now=(datetime.datetime.now(), datetime.datetime.fromtimestamp( + min(MyTime.alternateNowTime or 978393600, 978393600)) + ) + ) + timeRE['Exy'] = r"(?P<y>\d{2})" + +_updateTimeRE() def getTimePatternRE(): keys = timeRE.keys() @@ -168,9 +205,9 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): """ now = \ - year = month = day = hour = minute = tzoffset = \ + year = month = day = tzoffset = \ weekday = julian = week_of_year = None - second = fraction = 0 + hour = minute = second = fraction = 0 for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: @@ -234,16 +271,12 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): week_of_year = int(val) # U starts week on Sunday, W - on Monday week_of_year_start = 6 if key == 'U' else 0 - elif key == 'z': + elif key in ('z', 'Z'): z = val if z in ("Z", "UTC", "GMT"): tzoffset = 0 else: tzoffset = zone2offset(z, 0); # currently offset-based only - elif key == 'Z': - z = val - if z in ("UTC", "GMT"): - tzoffset = 0 # Fail2Ban will assume it's this year assume_year = False @@ -291,9 +324,8 @@ def reGroupDictStrptime(found_dict, msec=False, default_tz=None): date_result -= datetime.timedelta(days=1) if assume_year: if not now: now = MyTime.now() - if date_result > now: - # Could be last year? - # also reset month and day as it's not yesterday... + if date_result > now + datetime.timedelta(days=1): # ignore by timezone issues (+24h) + # assume last year - also reset month and day as it's not yesterday... date_result = date_result.replace( year=year-1, month=month, day=day) diff --git a/fail2ban/server/ticket.py b/fail2ban/server/ticket.py index 5debc79d..96e67773 100644 --- a/fail2ban/server/ticket.py +++ b/fail2ban/server/ticket.py @@ -33,7 +33,7 @@ logSys = getLogger(__name__) class Ticket(object): - __slots__ = ('_ip', '_flags', '_banCount', '_banTime', '_time', '_data', '_retry', '_lastReset') + __slots__ = ('_id', '_flags', '_banCount', '_banTime', '_time', '_data', '_retry', '_lastReset') MAX_TIME = 0X7FFFFFFFFFFF ;# 4461763-th year @@ -48,7 +48,7 @@ class Ticket(object): @param matches (log) lines caused the ticket """ - self.setIP(ip) + self.setID(ip) self._flags = 0; self._banCount = 0; self._banTime = None; @@ -65,7 +65,7 @@ class Ticket(object): def __str__(self): return "%s: ip=%s time=%s bantime=%s bancount=%s #attempts=%d matches=%r" % \ - (self.__class__.__name__.split('.')[-1], self._ip, self._time, + (self.__class__.__name__.split('.')[-1], self._id, self._time, self._banTime, self._banCount, self._data['failures'], self._data.get('matches', [])) @@ -74,7 +74,7 @@ class Ticket(object): def __eq__(self, other): try: - return self._ip == other._ip and \ + return self._id == other._id and \ round(self._time, 2) == round(other._time, 2) and \ self._data == other._data except AttributeError: @@ -86,18 +86,17 @@ class Ticket(object): if v is not None: setattr(self, n, v) - - def setIP(self, value): + def setID(self, value): # guarantee using IPAddr instead of unicode, str for the IP if isinstance(value, basestring): value = IPAddr(value) - self._ip = value + self._id = value def getID(self): - return self._data.get('fid', self._ip) + return self._id def getIP(self): - return self._ip + return self._data.get('ip', self._id) def setTime(self, value): self._time = value @@ -144,7 +143,13 @@ class Ticket(object): return self._data['failures'] def setMatches(self, matches): - self._data['matches'] = matches or [] + if matches: + self._data['matches'] = matches + else: + try: + del self._data['matches'] + except KeyError: + pass def getMatches(self): return [(line if not isinstance(line, (list, tuple)) else "".join(line)) \ @@ -209,20 +214,26 @@ class Ticket(object): # return single value of data: return self._data.get(key, default) + @property + def banEpoch(self): + return getattr(self, '_banEpoch', 0) + @banEpoch.setter + def banEpoch(self, value): + self._banEpoch = value + class FailTicket(Ticket): def __init__(self, ip=None, time=None, matches=None, data={}, ticket=None): # this class variables: - self._retry = 0 - self._lastReset = None + self._firstTime = None + self._retry = 1 # create/copy using default ticket constructor: Ticket.__init__(self, ip, time, matches, data, ticket) # init: - if ticket is None: - self._lastReset = time if time is not None else self.getTime() - if not self._retry: - self._retry = self._data['failures']; + if not isinstance(ticket, FailTicket): + self._firstTime = time if time is not None else self.getTime() + self._retry = self._data.get('failures', 1) def setRetry(self, value): """ Set artificial retry count, normally equal failures / attempt, @@ -239,7 +250,20 @@ class FailTicket(Ticket): """ Returns failures / attempt count or artificial retry count increased for bad IPs """ - return max(self._retry, self._data['failures']) + return self._retry + + def adjustTime(self, time, maxTime): + """ Adjust time of ticket and current attempts count considering given maxTime + as estimation from rate by previous known interval (if it exceeds the findTime) + """ + if time > self._time: + # expand current interval and attemps count (considering maxTime): + if self._firstTime < time - maxTime: + # adjust retry calculated as estimation from rate by previous known interval: + self._retry = int(round(self._retry / float(time - self._firstTime) * maxTime)) + self._firstTime = time - maxTime + # last time of failure: + self._time = time def inc(self, matches=None, attempt=1, count=1): self._retry += count @@ -251,19 +275,6 @@ class FailTicket(Ticket): else: self._data['matches'] = matches - def setLastTime(self, value): - if value > self._time: - self._time = value - - def getLastTime(self): - return self._time - - def getLastReset(self): - return self._lastReset - - def setLastReset(self, value): - self._lastReset = value - @staticmethod def wrap(o): o.__class__ = FailTicket diff --git a/fail2ban/server/transmitter.py b/fail2ban/server/transmitter.py index c24408c4..6de60f94 100644 --- a/fail2ban/server/transmitter.py +++ b/fail2ban/server/transmitter.py @@ -43,6 +43,7 @@ class Transmitter: def __init__(self, server): self.__server = server + self.__quiet = 0 ## # Proceeds a command. @@ -57,7 +58,7 @@ class Transmitter: ret = self.__commandHandler(command) ack = 0, ret except Exception as e: - logSys.warning("Command %r has failed. Received %r", + logSys.error("Command %r has failed. Received %r", command, e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) ack = 1, e @@ -69,9 +70,10 @@ class Transmitter: # def __commandHandler(self, command): - if command[0] == "ping": + name = command[0] + if name == "ping": return "pong" - elif command[0] == "add": + elif name == "add": name = command[1] if name == "--all": raise Exception("Reserved name %r" % (name,)) @@ -81,11 +83,15 @@ class Transmitter: backend = "auto" self.__server.addJail(name, backend) return name - elif command[0] == "start": + elif name == "multi-set": + return self.__commandSet(command[1:], True) + elif name == "set": + return self.__commandSet(command[1:]) + elif name == "start": name = command[1] self.__server.startJail(name) return None - elif command[0] == "stop": + elif name == "stop": if len(command) == 1: self.__server.quit() elif command[1] == "--all": @@ -94,47 +100,53 @@ class Transmitter: name = command[1] self.__server.stopJail(name) return None - elif command[0] == "reload": + elif name == "reload": opts = command[1:3] + self.__quiet = 1 try: self.__server.reloadJails(*opts, begin=True) for cmd in command[3]: self.__commandHandler(cmd) finally: + self.__quiet = 0 self.__server.reloadJails(*opts, begin=False) return 'OK' - elif len(command) >= 2 and command[0] == "unban": + elif name == "unban" and len(command) >= 2: # unban in all jails: value = command[1:] # if all ips: if len(value) == 1 and value[0] == "--all": return self.__server.setUnbanIP() - cnt = 0 - for value in value: - cnt += self.__server.setUnbanIP(None, value) - return cnt - elif command[0] == "echo": + return self.__server.setUnbanIP(None, value) + elif name == "banned": + # check IP is banned in all jails: + return self.__server.banned(None, command[1:]) + elif name == "echo": return command[1:] - elif command[0] == "server-status": + elif name == "server-status": logSys.debug("Status: ready") return "Server ready" - elif command[0] == "sleep": + elif name == "server-stream": + self.__quiet = 1 + try: + for cmd in command[1]: + self.__commandHandler(cmd) + finally: + self.__quiet = 0 + return None + elif name == "sleep": value = command[1] time.sleep(float(value)) return None - elif command[0] == "flushlogs": + elif name == "flushlogs": return self.__server.flushLogs() - elif command[0] == "multi-set": - return self.__commandSet(command[1:], True) - elif command[0] == "set": - return self.__commandSet(command[1:]) - elif command[0] == "get": + elif name == "get": return self.__commandGet(command[1:]) - elif command[0] == "status": + elif name == "status": return self.status(command[1:]) - elif command[0] == "version": + elif name == "version": return version.version - elif command[0] == "config-error": + elif name == "config-error": logSys.error(command[1]) return None raise Exception("Invalid command") @@ -145,19 +157,31 @@ class Transmitter: if name == "loglevel": value = command[1] self.__server.setLogLevel(value) + if self.__quiet: return return self.__server.getLogLevel() elif name == "logtarget": value = command[1] if self.__server.setLogTarget(value): + if self.__quiet: return return self.__server.getLogTarget() else: raise Exception("Failed to change log target") elif name == "syslogsocket": value = command[1] if self.__server.setSyslogSocket(value): + if self.__quiet: return return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + elif name == "allowipv6": + value = command[1] + self.__server.setIPv6IsAllowed(value) + if self.__quiet: return + return value + #Thread + elif name == "thread": + value = command[1] + return self.__server.setThreadOptions(value) #Database elif name == "dbfile": self.__server.setDatabase(command[1]) @@ -165,14 +189,25 @@ class Transmitter: if db is None: return None else: + if self.__quiet: return return db.filename + elif name == "dbmaxmatches": + db = self.__server.getDatabase() + if db is None: + logSys.log(logging.MSG, "dbmaxmatches setting was not in effect since no db yet") + return None + else: + db.maxMatches = int(command[1]) + if self.__quiet: return + return db.maxMatches elif name == "dbpurgeage": db = self.__server.getDatabase() if db is None: - logSys.warning("dbpurgeage setting was not in effect since no db yet") + logSys.log(logging.MSG, "dbpurgeage setting was not in effect since no db yet") return None else: db.purgeage = command[1] + if self.__quiet: return return db.purgeage # Jail elif command[1] == "idle": @@ -182,27 +217,33 @@ class Transmitter: self.__server.setIdleJail(name, False) else: raise Exception("Invalid idle option, must be 'on' or 'off'") + if self.__quiet: return return self.__server.getIdleJail(name) # Filter elif command[1] == "ignoreself": value = command[2] self.__server.setIgnoreSelf(name, value) + if self.__quiet: return return self.__server.getIgnoreSelf(name) elif command[1] == "addignoreip": - value = command[2] - self.__server.addIgnoreIP(name, value) + for value in command[2:]: + self.__server.addIgnoreIP(name, value) + if self.__quiet: return return self.__server.getIgnoreIP(name) elif command[1] == "delignoreip": value = command[2] self.__server.delIgnoreIP(name, value) + if self.__quiet: return return self.__server.getIgnoreIP(name) elif command[1] == "ignorecommand": value = command[2] self.__server.setIgnoreCommand(name, value) + if self.__quiet: return return self.__server.getIgnoreCommand(name) elif command[1] == "ignorecache": value = command[2] self.__server.setIgnoreCache(name, value) + if self.__quiet: return return self.__server.getIgnoreCache(name) elif command[1] == "addlogpath": value = command[2] @@ -215,93 +256,126 @@ class Transmitter: elif len(command) > 4: raise ValueError("Only one file can be added at a time") self.__server.addLogPath(name, value, tail) + if self.__quiet: return return self.__server.getLogPath(name) elif command[1] == "dellogpath": value = command[2] self.__server.delLogPath(name, value) + if self.__quiet: return return self.__server.getLogPath(name) elif command[1] == "logencoding": value = command[2] self.__server.setLogEncoding(name, value) + if self.__quiet: return return self.__server.getLogEncoding(name) elif command[1] == "addjournalmatch": # pragma: systemd no cover value = command[2:] self.__server.addJournalMatch(name, value) + if self.__quiet: return return self.__server.getJournalMatch(name) elif command[1] == "deljournalmatch": # pragma: systemd no cover value = command[2:] self.__server.delJournalMatch(name, value) + if self.__quiet: return return self.__server.getJournalMatch(name) elif command[1] == "prefregex": value = command[2] self.__server.setPrefRegex(name, value) - return self.__server.getPrefRegex(name) + if self.__quiet: return + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "addfailregex": value = command[2] self.__server.addFailRegex(name, value, multiple=multiple) if multiple: return True + if self.__quiet: return return self.__server.getFailRegex(name) elif command[1] == "delfailregex": value = int(command[2]) self.__server.delFailRegex(name, value) + if self.__quiet: return return self.__server.getFailRegex(name) elif command[1] == "addignoreregex": value = command[2] self.__server.addIgnoreRegex(name, value, multiple=multiple) if multiple: return True + if self.__quiet: return return self.__server.getIgnoreRegex(name) elif command[1] == "delignoreregex": value = int(command[2]) self.__server.delIgnoreRegex(name, value) + if self.__quiet: return return self.__server.getIgnoreRegex(name) elif command[1] == "usedns": value = command[2] self.__server.setUseDns(name, value) + if self.__quiet: return return self.__server.getUseDns(name) elif command[1] == "findtime": value = command[2] self.__server.setFindTime(name, value) + if self.__quiet: return return self.__server.getFindTime(name) elif command[1] == "datepattern": value = command[2] self.__server.setDatePattern(name, value) + if self.__quiet: return return self.__server.getDatePattern(name) elif command[1] == "logtimezone": value = command[2] self.__server.setLogTimeZone(name, value) + if self.__quiet: return return self.__server.getLogTimeZone(name) + elif command[1] == "maxmatches": + value = command[2] + self.__server.setMaxMatches(name, int(value)) + if self.__quiet: return + return self.__server.getMaxMatches(name) elif command[1] == "maxretry": value = command[2] self.__server.setMaxRetry(name, int(value)) + if self.__quiet: return return self.__server.getMaxRetry(name) elif command[1] == "maxlines": value = command[2] self.__server.setMaxLines(name, int(value)) + if self.__quiet: return return self.__server.getMaxLines(name) # command elif command[1] == "bantime": value = command[2] self.__server.setBanTime(name, value) + if self.__quiet: return return self.__server.getBanTime(name) + elif command[1] == "attempt": + value = command[2:] + if self.__quiet: return + return self.__server.addAttemptIP(name, *value) elif command[1].startswith("bantime."): value = command[2] opt = command[1][len("bantime."):] self.__server.setBanTimeExtra(name, opt, value) + if self.__quiet: return return self.__server.getBanTimeExtra(name, opt) elif command[1] == "banip": - value = command[2] + value = command[2:] return self.__server.setBanIP(name,value) elif command[1] == "unbanip": - value = command[2] - self.__server.setUnbanIP(name, value) - return value + ifexists = True + if command[2] != "--report-absent": + value = command[2:] + else: + ifexists = False + value = command[3:] + return self.__server.setUnbanIP(name, value, ifexists=ifexists) elif command[1] == "addaction": args = [command[2]] if len(command) > 3: args.extend([command[3], json.loads(command[4])]) self.__server.addAction(name, *args) + if self.__quiet: return return args[0] elif command[1] == "delaction": value = command[2] @@ -325,10 +399,12 @@ class Transmitter: actionkey = command[3] if callable(getattr(action, actionkey, None)): actionvalue = json.loads(command[4]) if len(command)>4 else {} + if self.__quiet: return return getattr(action, actionkey)(**actionvalue) else: actionvalue = command[4] setattr(action, actionkey, actionvalue) + if self.__quiet: return return getattr(action, actionkey) raise Exception("Invalid command %r (no set action or not yet implemented)" % (command[1],)) @@ -341,6 +417,9 @@ class Transmitter: return self.__server.getLogTarget() elif name == "syslogsocket": return self.__server.getSyslogSocket() + #Thread + elif name == "thread": + return self.__server.getThreadOptions() #Database elif name == "dbfile": db = self.__server.getDatabase() @@ -348,13 +427,22 @@ class Transmitter: return None else: return db.filename + elif name == "dbmaxmatches": + db = self.__server.getDatabase() + if db is None: + return None + else: + return db.maxMatches elif name == "dbpurgeage": db = self.__server.getDatabase() if db is None: return None else: return db.purgeage - # Filter + # Jail, Filter + elif command[1] == "banned": + # check IP is banned in all jails: + return self.__server.banned(name, command[2:]) elif command[1] == "logpath": return self.__server.getLogPath(name) elif command[1] == "logencoding": @@ -370,7 +458,8 @@ class Transmitter: elif command[1] == "ignorecache": return self.__server.getIgnoreCache(name) elif command[1] == "prefregex": - return self.__server.getPrefRegex(name) + v = self.__server.getPrefRegex(name) + return v.getRegex() if v else "" elif command[1] == "failregex": return self.__server.getFailRegex(name) elif command[1] == "ignoreregex": @@ -383,6 +472,8 @@ class Transmitter: return self.__server.getDatePattern(name) elif command[1] == "logtimezone": return self.__server.getLogTimeZone(name) + elif command[1] == "maxmatches": + return self.__server.getMaxMatches(name) elif command[1] == "maxretry": return self.__server.getMaxRetry(name) elif command[1] == "maxlines": @@ -390,6 +481,9 @@ class Transmitter: # Action elif command[1] == "bantime": return self.__server.getBanTime(name) + elif command[1] == "banip": + return self.__server.getBanList(name, + withTime=len(command) > 2 and command[2] == "--with-time") elif command[1].startswith("bantime."): opt = command[1][len("bantime."):] return self.__server.getBanTimeExtra(name, opt) diff --git a/fail2ban/server/utils.py b/fail2ban/server/utils.py index b59fb4e1..18073ea7 100644 --- a/fail2ban/server/utils.py +++ b/fail2ban/server/utils.py @@ -27,8 +27,10 @@ import os import signal import subprocess import sys +from threading import Lock import time from ..helpers import getLogger, _merge_dicts, uni_decode +from collections import OrderedDict if sys.version_info >= (3, 3): import importlib.machinery @@ -69,7 +71,8 @@ class Utils(): def __init__(self, *args, **kwargs): self.setOptions(*args, **kwargs) - self._cache = {} + self._cache = OrderedDict() + self.__lock = Lock() def setOptions(self, maxCount=1000, maxTime=60): self.maxCount = maxCount @@ -83,27 +86,32 @@ class Utils(): if v: if v[1] > time.time(): return v[0] - del self._cache[k] + self.unset(k) return defv def set(self, k, v): t = time.time() - cache = self._cache # for shorter local access - # clean cache if max count reached: - if len(cache) >= self.maxCount: - for (ck, cv) in cache.items(): - if cv[1] < t: - del cache[ck] - # if still max count - remove any one: + # avoid multiple modification of dict multi-threaded: + cache = self._cache + with self.__lock: + # clean cache if max count reached: if len(cache) >= self.maxCount: - cache.popitem() - cache[k] = (v, t + self.maxTime) + # ordered (so remove some from ahead, FIFO) + while cache: + (ck, cv) = cache.popitem(last=False) + # if not yet expired (but has free slot for new entry): + if cv[1] > t and len(cache) < self.maxCount: + break + # set now: + cache[k] = (v, t + self.maxTime) def unset(self, k): - try: - del self._cache[k] - except KeyError: - pass + with self.__lock: + self._cache.pop(k, None) + + def clear(self): + with self.__lock: + self._cache.clear() @staticmethod @@ -224,8 +232,8 @@ class Utils(): return False if not output else (False, stdout, stderr, retcode) std_level = logging.DEBUG if retcode in success_codes else logging.ERROR - if std_level > logSys.getEffectiveLevel(): - if logCmd: logCmd(std_level-1); logCmd = None + if std_level >= logSys.getEffectiveLevel(): + if logCmd: logCmd(std_level-1 if std_level == logging.DEBUG else logging.ERROR); logCmd = None # if we need output (to return or to log it): if output or std_level >= logSys.getEffectiveLevel(): @@ -240,7 +248,6 @@ class Utils(): if stdout is not None and stdout != '' and std_level >= logSys.getEffectiveLevel(): for l in stdout.splitlines(): logSys.log(std_level, "%x -- stdout: %r", realCmdId, uni_decode(l)) - popen.stdout.close() if popen.stderr: try: if retcode is None or retcode < 0: @@ -251,7 +258,9 @@ class Utils(): if stderr is not None and stderr != '' and std_level >= logSys.getEffectiveLevel(): for l in stderr.splitlines(): logSys.log(std_level, "%x -- stderr: %r", realCmdId, uni_decode(l)) - popen.stderr.close() + + if popen.stdout: popen.stdout.close() + if popen.stderr: popen.stderr.close() success = False if retcode in success_codes: @@ -307,11 +316,9 @@ class Utils(): timeout_expr = lambda: time.time() > time0 else: timeout_expr = timeout - if not interval: - interval = Utils.DEFAULT_SLEEP_INTERVAL if timeout_expr(): break - stm = min(stm + interval, Utils.DEFAULT_SLEEP_TIME) + stm = min(stm + (interval or Utils.DEFAULT_SLEEP_INTERVAL), Utils.DEFAULT_SLEEP_TIME) time.sleep(stm) return ret diff --git a/fail2ban/tests/action_d/test_badips.py b/fail2ban/tests/action_d/test_badips.py deleted file mode 100644 index 3ea1fc76..00000000 --- a/fail2ban/tests/action_d/test_badips.py +++ /dev/null @@ -1,156 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import unittest -import sys -from functools import wraps -from socket import timeout -from ssl import SSLError - -from ..actiontestcase import CallingMap -from ..dummyjail import DummyJail -from ..servertestcase import IPAddr -from ..utils import LogCaptureTestCase, CONFIG_DIR - -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.error import HTTPError, URLError -else: # pragma: 3.x no cover - from urllib2 import HTTPError, URLError - -def skip_if_not_available(f): - """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". - """ - @wraps(f) - def wrapper(self, *args): - try: - return f(self, *args) - except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues - if not isinstance(e, timeout) and 'timed out' not in str(e): - if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: - raise - raise unittest.SkipTest('Skip test because of %s' % e) - return wrapper - -if sys.version_info >= (2,7): # pragma: no cover - may be unavailable - class BadIPsActionTest(LogCaptureTestCase): - - available = True, None - pythonModule = None - modAction = None - - def setUp(self): - """Call before every test case.""" - super(BadIPsActionTest, self).setUp() - unittest.F2B.SkipIfNoNetwork() - - self.jail = DummyJail() - - self.jail.actions.add("test") - - pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py") - - # check availability (once if not alive, used shorter timeout as in test cases): - if BadIPsActionTest.available[0]: - if not BadIPsActionTest.modAction: - if not BadIPsActionTest.pythonModule: - BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action - self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) - if not BadIPsActionTest.available[0]: - raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) - - self.jail.actions.add("badips", pythonModuleName, initOpts={ - 'category': "ssh", - 'banaction': "test", - 'age': "2w", - 'score': 5, - 'key': "fail2ban-test-suite", - #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 60), - }) - self.action = self.jail.actions["badips"] - - def tearDown(self): - """Call after every test case.""" - # Must cancel timer! - if self.action._timer: - self.action._timer.cancel() - super(BadIPsActionTest, self).tearDown() - - @skip_if_not_available - def testCategory(self): - categories = self.action.getCategories() - self.assertIn("ssh", categories) - self.assertTrue(len(categories) >= 10) - - self.assertRaises( - ValueError, setattr, self.action, "category", - "invalid-category") - - # Not valid for reporting category... - self.assertRaises( - ValueError, setattr, self.action, "category", "mail") - # but valid for blacklisting. - self.action.bancategory = "mail" - - @skip_if_not_available - def testScore(self): - self.assertRaises(ValueError, setattr, self.action, "score", -5) - self.action.score = 3 - self.action.score = "3" - - @skip_if_not_available - def testBanaction(self): - self.assertRaises( - ValueError, setattr, self.action, "banaction", - "invalid-action") - self.action.banaction = "test" - - @skip_if_not_available - def testUpdateperiod(self): - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", -50) - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", 0) - self.action.updateperiod = 900 - self.action.updateperiod = "900" - - @skip_if_not_available - def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - - @skip_if_not_available - def testBanIP(self): - aInfo = CallingMap({ - 'ip': IPAddr('192.0.2.1') - }) - self.action.ban(aInfo) - self.assertLogged('badips.com: ban', wait=True) - self.pruneLog() - # produce an error using wrong category/IP: - self.action._category = 'f2b-this-category-dont-available-test-suite-only' - aInfo['ip'] = '' - self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo) - self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False) diff --git a/fail2ban/tests/actionstestcase.py b/fail2ban/tests/actionstestcase.py index 279290d1..9c9add65 100644 --- a/fail2ban/tests/actionstestcase.py +++ b/fail2ban/tests/actionstestcase.py @@ -28,11 +28,10 @@ import time import os import tempfile -from ..server.actions import Actions from ..server.ticket import FailTicket from ..server.utils import Utils from .dummyjail import DummyJail -from .utils import LogCaptureTestCase +from .utils import LogCaptureTestCase, with_alt_time, with_tmpdir, MyTime TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -43,21 +42,21 @@ class ExecuteActions(LogCaptureTestCase): """Call before every test case.""" super(ExecuteActions, self).setUp() self.__jail = DummyJail() - self.__actions = Actions(self.__jail) - self.__tmpfile, self.__tmpfilename = tempfile.mkstemp() + self.__actions = self.__jail.actions def tearDown(self): super(ExecuteActions, self).tearDown() - os.remove(self.__tmpfilename) - def defaultActions(self): + def defaultAction(self, o={}): self.__actions.add('ip') - self.__ip = self.__actions['ip'] - self.__ip.actionstart = 'echo ip start 64 >> "%s"' % self.__tmpfilename - self.__ip.actionban = 'echo ip ban <ip> >> "%s"' % self.__tmpfilename - self.__ip.actionunban = 'echo ip unban <ip> >> "%s"' % self.__tmpfilename - self.__ip.actioncheck = 'echo ip check <ip> >> "%s"' % self.__tmpfilename - self.__ip.actionstop = 'echo ip stop >> "%s"' % self.__tmpfilename + act = self.__actions['ip'] + act.actionstart = 'echo ip start'+o.get('start', '') + act.actionban = 'echo ip ban <ip>'+o.get('ban', '') + act.actionunban = 'echo ip unban <ip>'+o.get('unban', '') + act.actioncheck = 'echo ip check'+o.get('check', '') + act.actionflush = 'echo ip flush'+o.get('flush', '') + act.actionstop = 'echo ip stop'+o.get('stop', '') + return act def testActionsAddDuplicateName(self): self.__actions.add('test') @@ -78,16 +77,27 @@ class ExecuteActions(LogCaptureTestCase): self.assertEqual(self.__actions.getBanTime(),127) self.assertRaises(ValueError, self.__actions.removeBannedIP, '127.0.0.1') + def testAddBannedIP(self): + self.assertEqual(self.__actions.addBannedIP('192.0.2.1'), 1) + self.assertLogged('Ban 192.0.2.1') + self.pruneLog() + self.assertEqual(self.__actions.addBannedIP(['192.0.2.1', '192.0.2.2', '192.0.2.3']), 2) + self.assertLogged('192.0.2.1 already banned') + self.assertNotLogged('Ban 192.0.2.1') + self.assertLogged('Ban 192.0.2.2') + self.assertLogged('Ban 192.0.2.3') + def testActionsOutput(self): - self.defaultActions() + self.defaultAction() self.__actions.start() - with open(self.__tmpfilename) as f: - self.assertTrue( Utils.wait_for(lambda: (f.read() == "ip start 64\n"), 3) ) - + self.assertLogged("stdout: %r" % 'ip start', wait=True) self.__actions.stop() self.__actions.join() + self.assertLogged("stdout: %r" % 'ip flush', "stdout: %r" % 'ip stop') self.assertEqual(self.__actions.status(),[("Currently banned", 0 ), ("Total banned", 0 ), ("Banned IP list", [] )]) + self.assertEqual(self.__actions.status('short'),[("Currently banned", 0 ), + ("Total banned", 0 )]) def testAddActionPython(self): self.__actions.add( @@ -163,3 +173,340 @@ class ExecuteActions(LogCaptureTestCase): self.assertNotLogged("Failed to execute unban") self.assertLogged("action1 unban deleted aInfo IP") self.assertLogged("action2 unban deleted aInfo IP") + + @with_alt_time + def testUnbanOnBusyBanBombing(self): + # check unban happens in-between of "ban bombing" despite lower precedence, + # if it is not work, we'll not see "Unbanned 30" (rather "Unbanned 50") + # because then all the unbans occur earliest at flushing (after stop) + + # each 3rd ban we should see an unban check (and up to 5 tickets gets unbanned): + self.__actions.banPrecedence = 3 + self.__actions.unbanMaxCount = 5 + self.__actions.setBanTime(100) + + self.__actions.start() + + MyTime.setTime(0); # avoid "expired bantime" (in 0.11) + i = 0 + while i < 20: + ip = "192.0.2.%d" % i + self.__jail.putFailTicket(FailTicket(ip, 0)) + i += 1 + + # wait for last ban (all 20 tickets gets banned): + self.assertLogged(' / 20,', wait=True) + + MyTime.setTime(200); # unban time for 20 tickets reached + + while i < 50: + ip = "192.0.2.%d" % i + self.__jail.putFailTicket(FailTicket(ip, 200)) + i += 1 + + # wait for last ban (all 50 tickets gets banned): + self.assertLogged(' / 50,', wait=True) + self.__actions.stop() + self.__actions.join() + + self.assertLogged('Unbanned 30, 0 ticket(s)') + self.assertNotLogged('Unbanned 50, 0 ticket(s)') + + def testActionsConsistencyCheck(self): + act = self.defaultAction({'check':' <family>', 'flush':' <family>'}) + # flush for inet6 is intentionally "broken" here - test no unhandled except and invariant check: + act['actionflush?family=inet6'] = act.actionflush + '; exit 1' + act.actionstart_on_demand = True + # force errors via check in ban/unban: + act.actionban = "<actioncheck> ; " + act.actionban + act.actionunban = "<actioncheck> ; " + act.actionunban + self.__actions.start() + self.assertNotLogged("stdout: %r" % 'ip start') + + self.assertEqual(self.__actions.addBannedIP('192.0.2.1'), 1) + self.assertEqual(self.__actions.addBannedIP('2001:db8::1'), 1) + self.assertLogged('Ban 192.0.2.1', 'Ban 2001:db8::1', + "stdout: %r" % 'ip start', + "stdout: %r" % 'ip ban 192.0.2.1', + "stdout: %r" % 'ip ban 2001:db8::1', + all=True, wait=True) + + # check should fail (so cause stop/start): + self.pruneLog('[test-phase 1a] simulate inconsistent irreparable env by unban') + act['actioncheck?family=inet6'] = act.actioncheck + '; exit 1' + self.__actions.removeBannedIP('2001:db8::1') + self.assertLogged('Invariant check failed. Unban is impossible.', + wait=True) + self.pruneLog('[test-phase 1b] simulate inconsistent irreparable env by flush') + self.__actions._Actions__flushBan() + self.assertLogged( + "stdout: %r" % 'ip flush inet4', + "stdout: %r" % 'ip flush inet6', + 'Failed to flush bans', + 'No flush occurred, do consistency check', + 'Invariant check failed. Trying to restore a sane environment', + "stdout: %r" % 'ip stop', # same for both families + 'Failed to flush bans', + all=True, wait=True) + + # check succeeds: + self.pruneLog('[test-phase 2] consistent env') + act['actioncheck?family=inet6'] = act.actioncheck + self.assertEqual(self.__actions.addBannedIP('2001:db8::1'), 1) + self.assertLogged('Ban 2001:db8::1', + "stdout: %r" % 'ip start', # same for both families + "stdout: %r" % 'ip ban 2001:db8::1', + all=True, wait=True) + self.assertNotLogged("stdout: %r" % 'ip check inet4', + all=True) + + self.pruneLog('[test-phase 3] failed flush in consistent env') + self.__actions._Actions__flushBan() + self.assertLogged('Failed to flush bans', + 'No flush occurred, do consistency check', + "stdout: %r" % 'ip flush inet6', + "stdout: %r" % 'ip check inet6', + all=True, wait=True) + self.assertNotLogged( + "stdout: %r" % 'ip flush inet4', + "stdout: %r" % 'ip stop', + "stdout: %r" % 'ip start', + 'Unable to restore environment', + all=True) + + # stop, flush succeeds: + self.pruneLog('[test-phase end] flush successful') + act['actionflush?family=inet6'] = act.actionflush + self.__actions.stop() + self.__actions.join() + self.assertLogged( + "stdout: %r" % 'ip flush inet6', + "stdout: %r" % 'ip stop', # same for both families + 'action ip terminated', + all=True, wait=True) + # no flush for inet4 (already successfully flushed): + self.assertNotLogged("ERROR", + "stdout: %r" % 'ip flush inet4', + 'Unban tickets each individualy', + all=True) + + def testActionsConsistencyCheckDiffFam(self): + # same as testActionsConsistencyCheck, but different start/stop commands for both families and repair on unban + act = self.defaultAction({'start':' <family>', 'check':' <family>', 'flush':' <family>', 'stop':' <family>'}) + # flush for inet6 is intentionally "broken" here - test no unhandled except and invariant check: + act['actionflush?family=inet6'] = act.actionflush + '; exit 1' + act.actionstart_on_demand = True + act.actionrepair_on_unban = True + # force errors via check in ban/unban: + act.actionban = "<actioncheck> ; " + act.actionban + act.actionunban = "<actioncheck> ; " + act.actionunban + self.__actions.start() + self.assertNotLogged("stdout: %r" % 'ip start') + + self.assertEqual(self.__actions.addBannedIP('192.0.2.1'), 1) + self.assertEqual(self.__actions.addBannedIP('2001:db8::1'), 1) + self.assertLogged('Ban 192.0.2.1', 'Ban 2001:db8::1', + "stdout: %r" % 'ip start inet4', + "stdout: %r" % 'ip ban 192.0.2.1', + "stdout: %r" % 'ip start inet6', + "stdout: %r" % 'ip ban 2001:db8::1', + all=True, wait=True) + + # check should fail (so cause stop/start): + act['actioncheck?family=inet6'] = act.actioncheck + '; exit 1' + self.pruneLog('[test-phase 1a] simulate inconsistent irreparable env by unban') + self.__actions.removeBannedIP('2001:db8::1') + self.assertLogged('Invariant check failed. Trying to restore a sane environment', + "stdout: %r" % 'ip stop inet6', + all=True, wait=True) + self.assertNotLogged( + "stdout: %r" % 'ip start inet6', # start on demand (not on repair) + "stdout: %r" % 'ip stop inet4', # family inet4 is not affected + "stdout: %r" % 'ip start inet4', + all=True) + + self.pruneLog('[test-phase 1b] simulate inconsistent irreparable env by ban') + self.assertEqual(self.__actions.addBannedIP('2001:db8::1'), 1) + self.assertLogged('Invariant check failed. Trying to restore a sane environment', + "stdout: %r" % 'ip stop inet6', + "stdout: %r" % 'ip start inet6', + "stdout: %r" % 'ip check inet6', + 'Unable to restore environment', + 'Failed to execute ban', + all=True, wait=True) + self.assertNotLogged( + "stdout: %r" % 'ip stop inet4', # family inet4 is not affected + "stdout: %r" % 'ip start inet4', + all=True) + + act['actioncheck?family=inet6'] = act.actioncheck + self.assertEqual(self.__actions.addBannedIP('2001:db8::2'), 1) + act['actioncheck?family=inet6'] = act.actioncheck + '; exit 1' + self.pruneLog('[test-phase 1c] simulate inconsistent irreparable env by flush') + self.__actions._Actions__flushBan() + self.assertLogged( + "stdout: %r" % 'ip flush inet4', + "stdout: %r" % 'ip flush inet6', + 'Failed to flush bans', + 'No flush occurred, do consistency check', + 'Invariant check failed. Trying to restore a sane environment', + "stdout: %r" % 'ip stop inet6', + 'Failed to flush bans in jail', + all=True, wait=True) + # start/stop should be called for inet6 only: + self.assertNotLogged( + "stdout: %r" % 'ip stop inet4', + all=True) + + # check succeeds: + self.pruneLog('[test-phase 2] consistent env') + act['actioncheck?family=inet6'] = act.actioncheck + self.assertEqual(self.__actions.addBannedIP('2001:db8::1'), 1) + self.assertLogged('Ban 2001:db8::1', + "stdout: %r" % 'ip start inet6', + "stdout: %r" % 'ip ban 2001:db8::1', + all=True, wait=True) + self.assertNotLogged( + "stdout: %r" % 'ip check inet4', + "stdout: %r" % 'ip start inet4', + all=True) + + self.pruneLog('[test-phase 3] failed flush in consistent env') + act['actioncheck?family=inet6'] = act.actioncheck + self.__actions._Actions__flushBan() + self.assertLogged('Failed to flush bans', + 'No flush occurred, do consistency check', + "stdout: %r" % 'ip flush inet6', + "stdout: %r" % 'ip check inet6', + all=True, wait=True) + self.assertNotLogged( + "stdout: %r" % 'ip flush inet4', + "stdout: %r" % 'ip stop inet4', + "stdout: %r" % 'ip start inet4', + "stdout: %r" % 'ip stop inet6', + "stdout: %r" % 'ip start inet6', + all=True) + + # stop, flush succeeds: + self.pruneLog('[test-phase end] flush successful') + act['actionflush?family=inet6'] = act.actionflush + self.__actions.stop() + self.__actions.join() + self.assertLogged( + "stdout: %r" % 'ip flush inet6', + "stdout: %r" % 'ip stop inet4', + "stdout: %r" % 'ip stop inet6', + 'action ip terminated', + all=True, wait=True) + # no flush for inet4 (already successfully flushed): + self.assertNotLogged("ERROR", + "stdout: %r" % 'ip flush inet4', + 'Unban tickets each individualy', + all=True) + + @with_alt_time + @with_tmpdir + def testActionsRebanBrokenAfterRepair(self, tmp): + act = self.defaultAction({ + 'start':' <family>; touch "<FN>"', + 'check':' <family>; test -f "<FN>"', + 'flush':' <family>; echo -n "" > "<FN>"', + 'stop': ' <family>; rm -f "<FN>"', + 'ban': ' <family>; echo "<ip> <family>" >> "<FN>"', + }) + act['FN'] = tmp+'/<family>' + act.actionstart_on_demand = True + act.actionrepair = 'echo ip repair <family>; touch "<FN>"' + act.actionreban = 'echo ip reban <ip> <family>; echo "<ip> <family> -- rebanned" >> "<FN>"' + self.pruneLog('[test-phase 0] initial ban') + self.assertEqual(self.__actions.addBannedIP(['192.0.2.1', '2001:db8::1']), 2) + self.assertLogged('Ban 192.0.2.1', 'Ban 2001:db8::1', + "stdout: %r" % 'ip start inet4', + "stdout: %r" % 'ip ban 192.0.2.1 inet4', + "stdout: %r" % 'ip start inet6', + "stdout: %r" % 'ip ban 2001:db8::1 inet6', + all=True) + + self.pruneLog('[test-phase 1] check ban') + self.dumpFile(tmp+'/inet4') + self.assertLogged('192.0.2.1 inet4') + self.assertNotLogged('2001:db8::1 inet6') + self.pruneLog() + self.dumpFile(tmp+'/inet6') + self.assertLogged('2001:db8::1 inet6') + self.assertNotLogged('192.0.2.1 inet4') + + # simulate 3 seconds past: + MyTime.setTime(MyTime.time() + 4) + # already banned produces events: + self.pruneLog('[test-phase 2] check already banned') + self.assertEqual(self.__actions.addBannedIP(['192.0.2.1', '2001:db8::1', '2001:db8::2']), 1) + self.assertLogged( + '192.0.2.1 already banned', '2001:db8::1 already banned', 'Ban 2001:db8::2', + "stdout: %r" % 'ip check inet4', # both checks occurred + "stdout: %r" % 'ip check inet6', + all=True) + self.dumpFile(tmp+'/inet4') + self.dumpFile(tmp+'/inet6') + # no reban should occur: + self.assertNotLogged('Reban 192.0.2.1', 'Reban 2001:db8::1', + "stdout: %r" % 'ip ban 192.0.2.1 inet4', + "stdout: %r" % 'ip reban 192.0.2.1 inet4', + "stdout: %r" % 'ip ban 2001:db8::1 inet6', + "stdout: %r" % 'ip reban 2001:db8::1 inet6', + '192.0.2.1 inet4 -- repaired', + '2001:db8::1 inet6 -- repaired', + all=True) + + # simulate 3 seconds past: + MyTime.setTime(MyTime.time() + 4) + # break env (remove both files, so check would fail): + os.remove(tmp+'/inet4') + os.remove(tmp+'/inet6') + # test again already banned (it shall cause reban now): + self.pruneLog('[test-phase 3a] check reban after sane env repaired') + self.assertEqual(self.__actions.addBannedIP(['192.0.2.1', '2001:db8::1']), 2) + self.assertLogged( + "Invariant check failed. Trying to restore a sane environment", + "stdout: %r" % 'ip repair inet4', # both repairs occurred + "stdout: %r" % 'ip repair inet6', + "Reban 192.0.2.1, action 'ip'", "Reban 2001:db8::1, action 'ip'", # both rebans also + "stdout: %r" % 'ip reban 192.0.2.1 inet4', + "stdout: %r" % 'ip reban 2001:db8::1 inet6', + all=True) + + # now last IP (2001:db8::2) - no repair, but still old epoch of ticket, so it gets rebanned: + self.pruneLog('[test-phase 3a] check reban by epoch mismatch (without repair)') + self.assertEqual(self.__actions.addBannedIP('2001:db8::2'), 1) + self.assertLogged( + "Reban 2001:db8::2, action 'ip'", + "stdout: %r" % 'ip reban 2001:db8::2 inet6', + all=True) + self.assertNotLogged( + "Invariant check failed. Trying to restore a sane environment", + "stdout: %r" % 'ip repair inet4', # both repairs occurred + "stdout: %r" % 'ip repair inet6', + "Reban 192.0.2.1, action 'ip'", "Reban 2001:db8::1, action 'ip'", # both rebans also + "stdout: %r" % 'ip reban 192.0.2.1 inet4', + "stdout: %r" % 'ip reban 2001:db8::1 inet6', + all=True) + + # and bans present in files: + self.pruneLog('[test-phase 4] check reban') + self.dumpFile(tmp+'/inet4') + self.assertLogged('192.0.2.1 inet4 -- rebanned') + self.assertNotLogged('2001:db8::1 inet6 -- rebanned') + self.pruneLog() + self.dumpFile(tmp+'/inet6') + self.assertLogged( + '2001:db8::1 inet6 -- rebanned', + '2001:db8::2 inet6 -- rebanned', all=True) + self.assertNotLogged('192.0.2.1 inet4 -- rebanned') + + # coverage - intended error in reban (no unhandled exception, message logged): + act.actionreban = '' + act.actionban = 'exit 1' + self.assertEqual(self.__actions._Actions__reBan(FailTicket("192.0.2.1", 0)), 0) + self.assertLogged( + 'Failed to execute reban', + 'Error banning 192.0.2.1', all=True) diff --git a/fail2ban/tests/actiontestcase.py b/fail2ban/tests/actiontestcase.py index 5f4aa460..ce5de483 100644 --- a/fail2ban/tests/actiontestcase.py +++ b/fail2ban/tests/actiontestcase.py @@ -34,8 +34,8 @@ from ..server.actions import OrderedDict, Actions from ..server.utils import Utils from .dummyjail import DummyJail -from .utils import LogCaptureTestCase -from .utils import pid_exists +from .utils import pid_exists, with_tmpdir, LogCaptureTestCase + class CommandActionTest(LogCaptureTestCase): @@ -75,61 +75,59 @@ class CommandActionTest(LogCaptureTestCase): lambda: substituteRecursiveTags({'A': 'to=<B> fromip=<IP>', 'C': '<B>', 'B': '<C>', 'D': ''})) self.assertRaises(ValueError, lambda: substituteRecursiveTags({'failregex': 'to=<honeypot> fromip=<IP>', 'sweet': '<honeypot>', 'honeypot': '<sweet>', 'ignoreregex': ''})) - # We need here an ordered, because the sequence of iteration is very important for this test - if OrderedDict: - # No cyclic recursion, just multiple replacement of tag <T>, should be successful: - self.assertEqual(substituteRecursiveTags( OrderedDict( - (('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>'))) - ), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'} - ) - # No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful: - self.assertEqual(substituteRecursiveTags( OrderedDict( - (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>'))) - ), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'} - ) - # No cyclic recursion, just multiple replacement of same tags, should be successful: - self.assertEqual(substituteRecursiveTags( OrderedDict(( - ('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'), - ('ipmset', 'f2b-<name>'), - ('name', 'any'), - ('bantime', '600'), - ('ipsetfamily', 'inet'), - ('iptables', 'iptables <lockingopt>'), - ('lockingopt', '-w'), - ('chain', 'INPUT'), - ('actiontype', '<multiport>'), - ('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'), - ('protocol', 'tcp'), - ('port', 'ssh'), - ('blocktype', 'REJECT',), - )) - ), OrderedDict(( - ('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), - ('ipmset', 'f2b-any'), - ('name', 'any'), - ('bantime', '600'), - ('ipsetfamily', 'inet'), - ('iptables', 'iptables -w'), - ('lockingopt', '-w'), - ('chain', 'INPUT'), - ('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), - ('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), - ('protocol', 'tcp'), - ('port', 'ssh'), - ('blocktype', 'REJECT') - )) - ) - # Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle: - self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( - ('A', '<<B><C>>'), - ('B', 'D'), ('C', 'E'), - ('DE', 'cycle <A>'), - )) )) - self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( - ('DE', 'cycle <A>'), - ('A', '<<B><C>>'), - ('B', 'D'), ('C', 'E'), - )) )) + # No cyclic recursion, just multiple replacement of tag <T>, should be successful: + self.assertEqual(substituteRecursiveTags( OrderedDict( + (('X', 'x=x<T>'), ('T', '1'), ('Z', '<X> <T> <Y>'), ('Y', 'y=y<T>'))) + ), {'X': 'x=x1', 'T': '1', 'Y': 'y=y1', 'Z': 'x=x1 1 y=y1'} + ) + # No cyclic recursion, just multiple replacement of tag <T> in composite tags, should be successful: + self.assertEqual(substituteRecursiveTags( OrderedDict( + (('X', 'x=x<T> <Z> <<R1>> <<R2>>'), ('R1', 'Z'), ('R2', 'Y'), ('T', '1'), ('Z', '<T> <Y>'), ('Y', 'y=y<T>'))) + ), {'X': 'x=x1 1 y=y1 1 y=y1 y=y1', 'R1': 'Z', 'R2': 'Y', 'T': '1', 'Z': '1 y=y1', 'Y': 'y=y1'} + ) + # No cyclic recursion, just multiple replacement of same tags, should be successful: + self.assertEqual(substituteRecursiveTags( OrderedDict(( + ('actionstart', 'ipset create <ipmset> hash:ip timeout <bantime> family <ipsetfamily>\n<iptables> -I <chain> <actiontype>'), + ('ipmset', 'f2b-<name>'), + ('name', 'any'), + ('bantime', '600'), + ('ipsetfamily', 'inet'), + ('iptables', 'iptables <lockingopt>'), + ('lockingopt', '-w'), + ('chain', 'INPUT'), + ('actiontype', '<multiport>'), + ('multiport', '-p <protocol> -m multiport --dports <port> -m set --match-set <ipmset> src -j <blocktype>'), + ('protocol', 'tcp'), + ('port', 'ssh'), + ('blocktype', 'REJECT',), + )) + ), OrderedDict(( + ('actionstart', 'ipset create f2b-any hash:ip timeout 600 family inet\niptables -w -I INPUT -p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), + ('ipmset', 'f2b-any'), + ('name', 'any'), + ('bantime', '600'), + ('ipsetfamily', 'inet'), + ('iptables', 'iptables -w'), + ('lockingopt', '-w'), + ('chain', 'INPUT'), + ('actiontype', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), + ('multiport', '-p tcp -m multiport --dports ssh -m set --match-set f2b-any src -j REJECT'), + ('protocol', 'tcp'), + ('port', 'ssh'), + ('blocktype', 'REJECT') + )) + ) + # Cyclic recursion by composite tag creation, tags "create" another tag, that closes cycle: + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( + ('A', '<<B><C>>'), + ('B', 'D'), ('C', 'E'), + ('DE', 'cycle <A>'), + )) )) + self.assertRaises(ValueError, lambda: substituteRecursiveTags( OrderedDict(( + ('DE', 'cycle <A>'), + ('A', '<<B><C>>'), + ('B', 'D'), ('C', 'E'), + )) )) # missing tags are ok self.assertEqual(substituteRecursiveTags({'A': '<C>'}), {'A': '<C>'}) @@ -206,15 +204,15 @@ class CommandActionTest(LogCaptureTestCase): "Text 890 text 123 ABC") self.assertEqual( self.__action.replaceTag("<matches>", - {'matches': "some >char< should \< be[ escap}ed&\n"}), + {'matches': "some >char< should \\< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\\n") self.assertEqual( self.__action.replaceTag("<ipmatches>", - {'ipmatches': "some >char< should \< be[ escap}ed&\n"}), + {'ipmatches': "some >char< should \\< be[ escap}ed&\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\\n") self.assertEqual( self.__action.replaceTag("<ipjailmatches>", - {'ipjailmatches': "some >char< should \< be[ escap}ed&\r\n"}), + {'ipjailmatches': "some >char< should \\< be[ escap}ed&\r\n"}), "some \\>char\\< should \\\\\\< be\\[ escap\\}ed\\&\\r\\n") # Recursive @@ -252,7 +250,7 @@ class CommandActionTest(LogCaptureTestCase): delattr(self.__action, 'ac') # produce self-referencing query except: self.assertRaisesRegexp(ValueError, r"possible self referencing definitions in query", - lambda: self.__action.replaceTag("<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x<x>>>>>>>>>>>>>>>>>>>>>", + lambda: self.__action.replaceTag("<x"*30+">"*30, self.__action._properties, conditional="family=inet6") ) @@ -297,61 +295,128 @@ class CommandActionTest(LogCaptureTestCase): "Text 000-567 text 567 '567'") self.assertTrue(len(cache) >= 3) - - def testExecuteActionBan(self): - self.__action.actionstart = "touch /tmp/fail2ban.test" - self.assertEqual(self.__action.actionstart, "touch /tmp/fail2ban.test") - self.__action.actionstop = "rm -f /tmp/fail2ban.test" - self.assertEqual(self.__action.actionstop, 'rm -f /tmp/fail2ban.test') - self.__action.actionban = "echo -n" - self.assertEqual(self.__action.actionban, 'echo -n') - self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" - self.assertEqual(self.__action.actioncheck, '[ -e /tmp/fail2ban.test ]') + @with_tmpdir + def testExecuteActionBan(self, tmp): + tmp += "/fail2ban.test" + self.__action.actionstart = "touch '%s'" % tmp + self.__action.actionrepair = self.__action.actionstart + self.assertEqual(self.__action.actionstart, "touch '%s'" % tmp) + self.__action.actionstop = "rm -f '%s'" % tmp + self.assertEqual(self.__action.actionstop, "rm -f '%s'" % tmp) + self.__action.actionban = "<actioncheck> && echo -n" + self.assertEqual(self.__action.actionban, "<actioncheck> && echo -n") + self.__action.actioncheck = "[ -e '%s' ]" % tmp + self.assertEqual(self.__action.actioncheck, "[ -e '%s' ]" % tmp) self.__action.actionunban = "true" self.assertEqual(self.__action.actionunban, 'true') + self.pruneLog() self.assertNotLogged('returned') # no action was actually executed yet + # start on demand is false, so it should cause failure on first attempt of ban: self.__action.ban({'ip': None}) self.assertLogged('Invariant check failed') self.assertLogged('returned successfully') + self.__action.stop() + self.assertLogged(self.__action.actionstop) def testExecuteActionEmptyUnban(self): + # unban will be executed for actions with banned items only: + self.__action.actionban = "" self.__action.actionunban = "" + self.__action.actionflush = "echo -n 'flush'" + self.__action.actionstop = "echo -n 'stop'" + self.__action.start(); + self.__action.ban({}); + self.pruneLog() self.__action.unban({}) - self.assertLogged('Nothing to do') + self.assertLogged('Nothing to do', wait=True) + # same as above but with interim flush, so no unban anymore: + self.__action.ban({}); + self.pruneLog('[phase 2]') + self.__action.flush() + self.__action.unban({}) + self.__action.stop() + self.assertLogged('stop', wait=True) + self.assertNotLogged('Nothing to do') - def testExecuteActionStartCtags(self): + @with_tmpdir + def testExecuteActionStartCtags(self, tmp): + tmp += '/fail2ban.test' self.__action.HOST = "192.0.2.0" - self.__action.actionstart = "touch /tmp/fail2ban.test.<HOST>" - self.__action.actionstop = "rm -f /tmp/fail2ban.test.<HOST>" - self.__action.actioncheck = "[ -e /tmp/fail2ban.test.192.0.2.0 ]" + self.__action.actionstart = "touch '%s.<HOST>'" % tmp + self.__action.actionstop = "rm -f '%s.<HOST>'" % tmp + self.__action.actioncheck = "[ -e '%s.192.0.2.0' ]" % tmp self.__action.start() + self.__action.consistencyCheck() - def testExecuteActionCheckRestoreEnvironment(self): + @with_tmpdir + def testExecuteActionCheckRestoreEnvironment(self, tmp): + tmp += '/fail2ban.test' self.__action.actionstart = "" - self.__action.actionstop = "rm -f /tmp/fail2ban.test" - self.__action.actionban = "rm /tmp/fail2ban.test" - self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" + self.__action.actionstop = "rm -f '%s'" % tmp + self.__action.actionban = "rm '%s'" % tmp + self.__action.actioncheck = "[ -e '%s' ]" % tmp self.assertRaises(RuntimeError, self.__action.ban, {'ip': None}) self.assertLogged('Invariant check failed', 'Unable to restore environment', all=True) # 2nd time, try to restore with producing error in stop, but succeeded start hereafter: self.pruneLog('[phase 2]') - self.__action.actionstart = "touch /tmp/fail2ban.test" - self.__action.actionstop = "rm /tmp/fail2ban.test" - self.__action.actionban = 'printf "%%b\n" <ip> >> /tmp/fail2ban.test' - self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" + self.__action.actionstart = "touch '%s'" % tmp + self.__action.actionstop = "rm '%s'" % tmp + self.__action.actionban = """<actioncheck> && printf "%%%%b\n" <ip> >> '%s'""" % tmp + self.__action.actioncheck = "[ -e '%s' ]" % tmp self.__action.ban({'ip': None}) self.assertLogged('Invariant check failed') self.assertNotLogged('Unable to restore environment') - def testExecuteActionCheckRepairEnvironment(self): + @with_tmpdir + def testExecuteActionCheckOnBanFailure(self, tmp): + tmp += '/fail2ban.test' + self.__action.actionstart = "touch '%s'; echo 'started ...'" % tmp + self.__action.actionstop = "rm -f '%s'" % tmp + self.__action.actionban = "[ -e '%s' ] && echo 'banned '<ip>" % tmp + self.__action.actioncheck = "[ -e '%s' ] && echo 'check ok' || { echo 'check failed'; exit 1; }" % tmp + self.__action.actionrepair = "echo 'repair ...'; touch '%s'" % tmp + self.__action.actionstart_on_demand = False + self.__action.start() + # phase 1: with repair; + # phase 2: without repair (start/stop), not on demand; + # phase 3: without repair (start/stop), start on demand. + for i in (1, 2, 3): + self.pruneLog('[phase %s]' % i) + # 1st time with success ban: + self.__action.ban({'ip': '192.0.2.1'}) + self.assertLogged( + "stdout: %r" % 'banned 192.0.2.1', all=True) + self.assertNotLogged("Invariant check failed. Trying", + "stdout: %r" % 'check failed', + "stdout: %r" % ('repair ...' if self.__action.actionrepair else 'started ...'), + "stdout: %r" % 'check ok', all=True) + # force error in ban: + os.remove(tmp) + self.pruneLog() + # 2nd time with fail recognition, success repair, check and ban: + self.__action.ban({'ip': '192.0.2.2'}) + self.assertLogged("Invariant check failed. Trying", + "stdout: %r" % 'check failed', + "stdout: %r" % ('repair ...' if self.__action.actionrepair else 'started ...'), + "stdout: %r" % 'check ok', + "stdout: %r" % 'banned 192.0.2.2', all=True) + # repeat without repair (stop/start), herafter enable on demand: + if self.__action.actionrepair: + self.__action.actionrepair = "" + elif not self.__action.actionstart_on_demand: + self.__action.actionstart_on_demand = True + + @with_tmpdir + def testExecuteActionCheckRepairEnvironment(self, tmp): + tmp += '/fail2ban.test' self.__action.actionstart = "" self.__action.actionstop = "" - self.__action.actionban = "rm /tmp/fail2ban.test" - self.__action.actioncheck = "[ -e /tmp/fail2ban.test ]" - self.__action.actionrepair = "echo 'repair ...'; touch /tmp/fail2ban.test" + self.__action.actionban = "rm '%s'" % tmp + self.__action.actioncheck = "[ -e '%s' ]" % tmp + self.__action.actionrepair = "echo 'repair ...'; touch '%s'" % tmp # 1st time with success repair: self.__action.ban({'ip': None}) self.assertLogged("Invariant check failed. Trying", "echo 'repair ...'", all=True) @@ -379,13 +444,13 @@ class CommandActionTest(LogCaptureTestCase): 'user': "tester" } }) - self.__action.actionban = "touch /tmp/fail2ban.test.123; echo 'failure <F-ID> of <F-USER> -<F-TEST>- from <ip>:<F-PORT>'" - self.__action.actionunban = "rm /tmp/fail2ban.test.<ABC>; echo 'user <F-USER> unbanned'" + self.__action.actionban = "echo '<ABC>, failure <F-ID> of <F-USER> -<F-TEST>- from <ip>:<F-PORT>'" + self.__action.actionunban = "echo '<ABC>, user <F-USER> unbanned'" self.__action.ban(aInfo) self.__action.unban(aInfo) self.assertLogged( - " -- stdout: 'failure 111 of tester -- from 192.0.2.1:222'", - " -- stdout: 'user tester unbanned'", + " -- stdout: '123, failure 111 of tester -- from 192.0.2.1:222'", + " -- stdout: '123, user tester unbanned'", all=True ) @@ -560,6 +625,19 @@ class CommandActionTest(LogCaptureTestCase): self.assertEqual(len(m), 3) self.assertIn('c', m) self.assertEqual((m['a'], m['b'], m['c']), (5, 11, 'test')) + # immutability of copy: + m['d'] = 'dddd' + m2 = m.copy() + m2['c'] = lambda self: self['a'] + 7 + m2['a'] = 1 + del m2['b'] + del m2['d'] + self.assertTrue('b' in m) + self.assertTrue('d' in m) + self.assertFalse('b' in m2) + self.assertFalse('d' in m2) + self.assertEqual((m['a'], m['b'], m['c'], m['d']), (5, 11, 'test', 'dddd')) + self.assertEqual((m2['a'], m2['c']), (1, 8)) def testCallingMapRep(self): m = CallingMap({ diff --git a/fail2ban/tests/banmanagertestcase.py b/fail2ban/tests/banmanagertestcase.py index a5b37ef6..cf25ac0f 100644 --- a/fail2ban/tests/banmanagertestcase.py +++ b/fail2ban/tests/banmanagertestcase.py @@ -29,6 +29,7 @@ import unittest from .utils import setUpMyTime, tearDownMyTime from ..server.banmanager import BanManager +from ..server.ipdns import DNSUtils from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): @@ -99,23 +100,23 @@ class AddFailure(unittest.TestCase): self.assertFalse(self.__banManager._inBanList(ticket)) def testBanTimeIncr(self): - ticket = BanTicket(self.__ticket.getIP(), self.__ticket.getTime()) + ticket = BanTicket(self.__ticket.getID(), self.__ticket.getTime()) ## increase twice and at end permanent, check time/count increase: c = 0 for i in (1000, 2000, -1): self.__banManager.addBanTicket(self.__ticket); c += 1 ticket.setBanTime(i) self.assertFalse(self.__banManager.addBanTicket(ticket)); # no incr of c (already banned) - self.assertEqual(str(self.__banManager.getTicketByID(ticket.getIP())), - "BanTicket: ip=%s time=%s bantime=%s bancount=%s #attempts=0 matches=[]" % (ticket.getIP(), ticket.getTime(), i, c)) + self.assertEqual(str(self.__banManager.getTicketByID(ticket.getID())), + "BanTicket: ip=%s time=%s bantime=%s bancount=%s #attempts=0 matches=[]" % (ticket.getID(), ticket.getTime(), i, c)) ## after permanent, it should remain permanent ban time (-1): self.__banManager.addBanTicket(self.__ticket); c += 1 ticket.setBanTime(-1) self.assertFalse(self.__banManager.addBanTicket(ticket)); # no incr of c (already banned) ticket.setBanTime(1000) self.assertFalse(self.__banManager.addBanTicket(ticket)); # no incr of c (already banned) - self.assertEqual(str(self.__banManager.getTicketByID(ticket.getIP())), - "BanTicket: ip=%s time=%s bantime=%s bancount=%s #attempts=0 matches=[]" % (ticket.getIP(), ticket.getTime(), -1, c)) + self.assertEqual(str(self.__banManager.getTicketByID(ticket.getID())), + "BanTicket: ip=%s time=%s bantime=%s bancount=%s #attempts=0 matches=[]" % (ticket.getID(), ticket.getTime(), -1, c)) def testUnban(self): btime = self.__banManager.getBanTime() @@ -154,6 +155,21 @@ class AddFailure(unittest.TestCase): finally: self.__banManager.setBanTime(btime) + def testBanList(self): + tickets = [ + BanTicket('192.0.2.1', 1167605999.0), + BanTicket('192.0.2.2', 1167605999.0), + ] + tickets[1].setBanTime(-1) + for t in tickets: + self.__banManager.addBanTicket(t) + self.assertSortedEqual(self.__banManager.getBanList(ordered=True, withTime=True), + [ + '192.0.2.1 \t2006-12-31 23:59:59 + 600 = 2007-01-01 00:09:59', + '192.0.2.2 \t2006-12-31 23:59:59 + -1 = 9999-12-31 23:59:59' + ] + ) + class StatusExtendedCymruInfo(unittest.TestCase): def setUp(self): @@ -161,10 +177,10 @@ class StatusExtendedCymruInfo(unittest.TestCase): super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() setUpMyTime() - self.__ban_ip = "93.184.216.34" - self.__asn = "15133" - self.__country = "EU" - self.__rir = "ripencc" + self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() + self.__asn = "36692" + self.__country = "US" + self.__rir = "arin" ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() self.assertTrue(self.__banManager.addBanTicket(ticket)) diff --git a/fail2ban/tests/clientreadertestcase.py b/fail2ban/tests/clientreadertestcase.py index 184595ab..37083a06 100644 --- a/fail2ban/tests/clientreadertestcase.py +++ b/fail2ban/tests/clientreadertestcase.py @@ -28,9 +28,10 @@ import re import shutil import tempfile import unittest -from ..client.configreader import ConfigReader, ConfigReaderUnshared, NoSectionError +from ..client.configreader import ConfigReader, ConfigReaderUnshared, \ + DefinitionInitConfigReader, NoSectionError from ..client import configparserinc -from ..client.jailreader import JailReader, extractOptions +from ..client.jailreader import JailReader, extractOptions, splitWithOptions from ..client.filterreader import FilterReader from ..client.jailsreader import JailsReader from ..client.actionreader import ActionReader, CommandAction @@ -86,6 +87,21 @@ option = %s self.assertTrue(self.c.read(f)) # we got some now return self.c.getOptions('section', [("int", 'option')])['option'] + def testConvert(self): + self.c.add_section("Definition") + self.c.set("Definition", "a", "1") + self.c.set("Definition", "b", "1") + self.c.set("Definition", "c", "test") + opts = self.c.getOptions("Definition", + (('int', 'a', 0), ('bool', 'b', 0), ('int', 'c', 0))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + opts = self.c.getOptions("Definition", + (('int', 'a'), ('bool', 'b'), ('int', 'c'))) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': None}) + opts = self.c.getOptions("Definition", + {'a': ('int', 0), 'b': ('bool', 0), 'c': ('int', 0)}) + self.assertSortedEqual(opts, {'a': 1, 'b': True, 'c': 0}) + def testInaccessibleFile(self): f = os.path.join(self.d, "d.conf") # inaccessible file self._write('d.conf', 0) @@ -125,6 +141,54 @@ option = %s self._remove("c.d/90.conf") self.assertEqual(self._getoption(), 2) + def testLocalInIncludes(self): + self._write("c.conf", value=None, content=""" +[INCLUDES] +before = ib.conf +after = ia.conf +[Definition] +test = %(default/test)s +""") + self._write("ib.conf", value=None, content=""" +[DEFAULT] +test = A +[Definition] +option = 1 +""") + self._write("ib.local", value=None, content=""" +[DEFAULT] +test = B +[Definition] +option = 2 +""") + self._write("ia.conf", value=None, content=""" +[DEFAULT] +test = C +[Definition] +oafter = 3 +""") + self._write("ia.local", value=None, content=""" +[DEFAULT] +test = D +[Definition] +oafter = 4 +""") + class TestDefConfReader(DefinitionInitConfigReader): + _configOpts = { + "option": ["int", None], + "oafter": ["int", None], + "test": ["string", None], + } + self.c = TestDefConfReader('c', 'option', {}) + self.c.setBaseDir(self.d) + self.assertTrue(self.c.read()) + self.c.getOptions({}, all=True) + o = self.c.getCombined() + # test local wins (overwrite all options): + self.assertEqual(o.get('option'), 2) + self.assertEqual(o.get('oafter'), 4) + self.assertEqual(o.get('test'), 'D') + def testInterpolations(self): self.assertFalse(self.c.read('i')) # nothing is there yet self._write("i.conf", value=None, content=""" @@ -200,6 +264,17 @@ class JailReaderTest(LogCaptureTestCase): def __init__(self, *args, **kwargs): super(JailReaderTest, self).__init__(*args, **kwargs) + def testSplitWithOptions(self): + # covering all separators - new-line and spaces: + for sep in ('\n', '\t', ' '): + self.assertEqual(splitWithOptions('a%sb' % (sep,)), ['a', 'b']) + self.assertEqual(splitWithOptions('a[x=y]%sb' % (sep,)), ['a[x=y]', 'b']) + self.assertEqual(splitWithOptions('a[x=y][z=z]%sb' % (sep,)), ['a[x=y][z=z]', 'b']) + self.assertEqual(splitWithOptions('a[x="y][z"]%sb' % (sep,)), ['a[x="y][z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y z"]%sb' % (sep,)), ['a[x="y z"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\tz"]%sb' % (sep,)), ['a[x="y\tz"]', 'b']) + self.assertEqual(splitWithOptions('a[x="y\nz"]%sb' % (sep,)), ['a[x="y\nz"]', 'b']) + def testIncorrectJail(self): jail = JailReader('XXXABSENTXXX', basedir=CONFIG_DIR, share_config=CONFIG_DIR_SHARE_CFG) self.assertRaises(ValueError, jail.read) @@ -253,7 +328,48 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(jail.getName(), 'sshd') jail.setName('ssh-funky-blocker') self.assertEqual(jail.getName(), 'ssh-funky-blocker') - + + def testOverrideFilterOptInJail(self): + unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf + jail = JailReader('sshd-override-flt-opts', basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertTrue(jail.isEnabled()) + stream = jail.convert() + # check filter options are overriden with values specified directly in jail: + # prefregex: + self.assertEqual([['set', 'sshd-override-flt-opts', 'prefregex', '^Test']], + [o for o in stream if len(o) > 2 and o[2] == 'prefregex']) + # journalmatch: + self.assertEqual([['set', 'sshd-override-flt-opts', 'addjournalmatch', '_COMM=test']], + [o for o in stream if len(o) > 2 and o[2] == 'addjournalmatch']) + # maxlines: + self.assertEqual([['set', 'sshd-override-flt-opts', 'maxlines', 2]], + [o for o in stream if len(o) > 2 and o[2] == 'maxlines']) + # usedns should be before all regex in jail stream: + usednsidx = stream.index(['set', 'sshd-override-flt-opts', 'usedns', 'no']) + i = 0 + for o in stream: + self.assertFalse(len(o) > 2 and o[2].endswith('regex')) + i += 1 + if i > usednsidx: break + + def testLogTypeOfBackendInJail(self): + unittest.F2B.SkipIfCfgMissing(stock=True); # expected include of common.conf + # test twice to check cache works peoperly: + for i in (1, 2): + # backend-related, overwritten in definition, specified in init parameters: + for prefline in ('JRNL', 'FILE', 'TEST', 'INIT'): + jail = JailReader('checklogtype_'+prefline.lower(), basedir=IMPERFECT_CONFIG, + share_config=IMPERFECT_CONFIG_SHARE_CFG, force_enable=True) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + stream = jail.convert() + # 'JRNL' for systemd, 'FILE' for file backend, 'TEST' for custom logtype (overwrite it): + self.assertEqual([['set', jail.getName(), 'addfailregex', '^%s failure from <HOST>$' % prefline]], + [o for o in stream if len(o) > 2 and o[2] == 'addfailregex']) + def testSplitOption(self): # Simple example option = "mail-whois[name=SSH]" @@ -265,13 +381,16 @@ class JailReaderTest(LogCaptureTestCase): self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]")) self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is")) - self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']")) - #print(self.getLog()) - #self.assertLogged("Invalid argument ['s'] in ''s''") - self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']")) + self.assertEqual(('mail', {'a': 'b'}), extractOptions("mail[a=b, ]")) + + self.assertRaises(ValueError, extractOptions ,'mail-how[') + + self.assertRaises(ValueError, extractOptions, """mail[a="test with interim (wrong) "" quotes"]""") + self.assertRaises(ValueError, extractOptions, """mail[a='test with interim (wrong) '' quotes']""") + self.assertRaises(ValueError, extractOptions, """mail[a='x, y, z', b=x, y, z]""") - #self.assertRaises(ValueError, extractOptions ,'mail-how[') + self.assertRaises(ValueError, extractOptions, """mail['s']""") # Empty option option = "abc[]" @@ -304,6 +423,30 @@ class JailReaderTest(LogCaptureTestCase): ) self.assertEqual(expected2, result) + def testMultiLineOption(self): + jail = JailReader('multi-log', force_enable=True, basedir=IMPERFECT_CONFIG, share_config=IMPERFECT_CONFIG_SHARE_CFG) + self.assertTrue(jail.read()) + self.assertTrue(jail.getOptions()) + self.assertEqual(jail.options['logpath'], 'a.log\nb.log\nc.log') + self.assertEqual(jail.options['action'], 'action[actname=\'ban\']\naction[actname=\'log\', logpath="a.log\nb.log\nc.log\nd.log"]\naction[actname=\'test\']') + self.assertSortedEqual([a.convert() for a in jail._JailReader__actions], [ + [['set', 'multi-log', 'addaction', 'ban'], ['multi-set', 'multi-log', 'action', 'ban', [ + ['actionban', 'echo "name: ban, ban: <ip>, logs: a.log\nb.log\nc.log"'], + ['actname', 'ban'], + ['name', 'multi-log'] + ]]], + [['set', 'multi-log', 'addaction', 'log'], ['multi-set', 'multi-log', 'action', 'log', [ + ['actionban', 'echo "name: log, ban: <ip>, logs: a.log\nb.log\nc.log\nd.log"'], + ['actname', 'log'], + ['logpath', 'a.log\nb.log\nc.log\nd.log'], ['name', 'multi-log'] + ]]], + [['set', 'multi-log', 'addaction', 'test'], ['multi-set', 'multi-log', 'action', 'test', [ + ['actionban', 'echo "name: test, ban: <ip>, logs: a.log\nb.log\nc.log"'], + ['actname', 'test'], + ['name', 'multi-log'] + ]]] + ]) + def testVersionAgent(self): unittest.F2B.SkipIfCfgMissing(stock=True) jail = JailReader('blocklisttest', force_enable=True, basedir=CONFIG_DIR) @@ -315,8 +458,6 @@ class JailReaderTest(LogCaptureTestCase): ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), ('action', '%(action_blocklist_de)s\n' - '%(action_badips_report)s\n' - '%(action_badips)s\n' 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' ), )) @@ -330,16 +471,14 @@ class JailReaderTest(LogCaptureTestCase): if len(cmd) <= 4: continue # differentiate between set and multi-set (wrop it here to single set): - if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')): + if cmd[0] == 'set' and cmd[4] == 'agent': act.append(cmd) elif cmd[0] == 'multi-set': act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent']) useragent = 'Fail2Ban/%s' % version - self.assertEqual(len(act), 4) + self.assertEqual(len(act), 2) self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) - self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) - self.assertEqual(eval(act[2][5]).get('agent', '<wrong>'), useragent) - self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) @with_tmpdir def testGlob(self, d): @@ -369,14 +508,12 @@ class JailReaderTest(LogCaptureTestCase): self.assertRaises(NoSectionError, c.getOptions, 'test', {}) -class FilterReaderTest(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(FilterReaderTest, self).__init__(*args, **kwargs) - self.__share_cfg = {} +class FilterReaderTest(LogCaptureTestCase): def testConvert(self): - output = [['multi-set', 'testcase01', 'addfailregex', [ + output = [ + ['set', 'testcase01', 'maxlines', 1], + ['multi-set', 'testcase01', 'addfailregex', [ "^\\s*(?:\\S+ )?(?:kernel: \\[\\d+\\.\\d+\\] )?(?:@vserver_\\S+ )" "?(?:(?:\\[\\d+\\])?:\\s+[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?|" "[\\[\\(]?sshd(?:\\(\\S+\\))?[\\]\\)]?:?(?:\\[\\d+\\])?:)?\\s*(?:" @@ -398,7 +535,6 @@ class FilterReaderTest(unittest.TestCase): ['set', 'testcase01', 'addjournalmatch', "FIELD= with spaces ", "+", "AFIELD= with + char and spaces"], ['set', 'testcase01', 'datepattern', "%Y %m %d %H:%M:%S"], - ['set', 'testcase01', 'maxlines', 1], # Last for overide test ] filterReader = FilterReader("testcase01", "testcase01", {}) filterReader.setBaseDir(TEST_FILES_DIR) @@ -415,9 +551,18 @@ class FilterReaderTest(unittest.TestCase): filterReader.read() #filterReader.getOptions(["failregex", "ignoreregex"]) filterReader.getOptions(None) - output[-1][-1] = "5" + output[0][-1] = 5; # maxlines = 5 self.assertSortedEqual(filterReader.convert(), output) + def testConvertOptions(self): + filterReader = FilterReader("testcase01", "testcase01", {'maxlines': '<test>', 'test': 'X'}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined(); + self.assertNotEqual(opts['maxlines'], 'X'); # wrong int value 'X' for 'maxlines' + self.assertLogged("Wrong int value 'X' for 'maxlines'. Using default one:") + def testFilterReaderSubstitionDefault(self): output = [['set', 'jailname', 'addfailregex', 'to=sweet@example.com fromip=<IP>']] filterReader = FilterReader('substition', "jailname", {}, @@ -427,6 +572,17 @@ class FilterReaderTest(unittest.TestCase): c = filterReader.convert() self.assertSortedEqual(c, output) + def testFilterReaderSubstKnown(self): + # testcase02.conf + testcase02.local, test covering that known/option is not overridden + # with unmodified (not available) value of option from .local config file, so wouldn't + # cause self-recursion if option already has a reference to known/option in .conf file. + filterReader = FilterReader('testcase02', "jailname", {}, + share_config=TEST_FILES_DIR_SHARE_CFG, basedir=TEST_FILES_DIR) + filterReader.read() + filterReader.getOptions(None) + opts = filterReader.getCombined() + self.assertTrue('sshd' in opts['failregex']) + def testFilterReaderSubstitionSet(self): output = [['set', 'jailname', 'addfailregex', 'to=sour@example.com fromip=<IP>']] filterReader = FilterReader('substition', "jailname", {'honeypot': 'sour@example.com'}, @@ -542,7 +698,7 @@ class JailsReaderTestCache(LogCaptureTestCase): cnt = self._getLoggedReadCount(r'filter\.d/common\.conf') self.assertTrue(cnt == 1, "Unexpected count by reading of filter files, cnt = %s" % cnt) # same with action: - cnt = self._getLoggedReadCount(r'action\.d/iptables-common\.conf') + cnt = self._getLoggedReadCount(r'action\.d/iptables\.conf') self.assertTrue(cnt == 1, "Unexpected count by reading of action files, cnt = %s" % cnt) finally: configparserinc.logLevel = saved_ll @@ -595,9 +751,9 @@ class JailsReaderTest(LogCaptureTestCase): ['add', 'tz_correct', 'auto'], ['start', 'tz_correct'], ['config-error', - "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], + "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"], ['config-error', - "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"], + "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"], ['config-error', "Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"], ['config-error', @@ -679,7 +835,7 @@ class JailsReaderTest(LogCaptureTestCase): # somewhat duplicating here what is done in JailsReader if # the jail is enabled - for act in actions.split('\n'): + for act in splitWithOptions(actions): actName, actOpt = extractOptions(act) self.assertTrue(len(actName)) self.assertTrue(isinstance(actOpt, dict)) @@ -805,21 +961,24 @@ class JailsReaderTest(LogCaptureTestCase): % (option, commands)) # Set up of logging should come first - self.assertEqual(find_set('syslogsocket'), 0) - self.assertEqual(find_set('loglevel'), 1) - self.assertEqual(find_set('logtarget'), 2) - # then dbfile should be before dbpurgeage + self.assertTrue( + find_set('syslogsocket') < find_set('loglevel') < find_set('logtarget') + ) + # then dbfile should be before dbmaxmatches and dbpurgeage self.assertTrue(find_set('dbpurgeage') > find_set('dbfile')) + self.assertTrue(find_set('dbmaxmatches') > find_set('dbfile')) # and there is logging information left to be passed into the # server - self.assertSortedEqual(commands, - [['set', 'dbfile', - '/var/lib/fail2ban/fail2ban.sqlite3'], - ['set', 'dbpurgeage', '1d'], - ['set', 'loglevel', "INFO"], - ['set', 'logtarget', '/var/log/fail2ban.log'], - ['set', 'syslogsocket', 'auto']]) + self.assertSortedEqual(commands,[ + ['set', 'syslogsocket', 'auto'], + ['set', 'loglevel', "INFO"], + ['set', 'logtarget', '/var/log/fail2ban.log'], + ['set', 'allowipv6', 'auto'], + ['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'], + ['set', 'dbmaxmatches', 10], + ['set', 'dbpurgeage', '1d'], + ]) # and if we force change configurator's fail2ban's baseDir # there should be an error message (test visually ;) -- diff --git a/fail2ban/tests/config/action.d/action.conf b/fail2ban/tests/config/action.d/action.conf new file mode 100644 index 00000000..b26c00b8 --- /dev/null +++ b/fail2ban/tests/config/action.d/action.conf @@ -0,0 +1,4 @@ + +[Definition] + +actionban = echo "name: <actname>, ban: <ip>, logs: %(logpath)s" diff --git a/fail2ban/tests/config/filter.d/checklogtype.conf b/fail2ban/tests/config/filter.d/checklogtype.conf new file mode 100644 index 00000000..4d700fff --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype.conf @@ -0,0 +1,31 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = ../../../../config/filter.d/common.conf + +[Definition] + +_daemon = test + +failregex = ^<lt_<logtype>/__prefix_line> failure from <HOST>$ +ignoreregex = + +# following sections define prefix line considering logtype: + +# backend-related (retrieved from backend, overwrite default): +[lt_file] +__prefix_line = FILE + +[lt_journal] +__prefix_line = JRNL + +# specified in definition section of filter (see filter checklogtype_test.conf): +[lt_test] +__prefix_line = TEST + +# specified in init parameter of jail (see ../jail.conf, jail checklogtype_init): +[lt_init] +__prefix_line = INIT diff --git a/fail2ban/tests/config/filter.d/checklogtype_test.conf b/fail2ban/tests/config/filter.d/checklogtype_test.conf new file mode 100644 index 00000000..a76f5fcf --- /dev/null +++ b/fail2ban/tests/config/filter.d/checklogtype_test.conf @@ -0,0 +1,12 @@ +# Fail2Ban configuration file +# + +[INCLUDES] + +# Read common prefixes (logtype is set in default section) +before = checklogtype.conf + +[Definition] + +# overwrite logtype in definition (no backend anymore): +logtype = test
\ No newline at end of file diff --git a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf index 98fca7f5..ad8adeb6 100644 --- a/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf +++ b/fail2ban/tests/config/filter.d/zzz-sshd-obsolete-multiline.conf @@ -37,7 +37,7 @@ __pam_auth = pam_[a-z]+ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for .* from <HOST>( via \S+)?\s*%(__suff)s$ ^%(__prefix_line_sl)sUser not known to the underlying authentication module for .* from <HOST>\s*%(__suff)s$ ^%(__prefix_line_sl)sFailed \S+ for invalid user <F-USER>(?P<cond_user>\S+)|(?:(?! from ).)*?</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) - ^%(__prefix_line_sl)sFailed \b(?!publickey)\S+ for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) + ^%(__prefix_line_sl)sFailed (?:<F-NOFAIL>publickey</F-NOFAIL>|\S+) for (?P<cond_inv>invalid user )?<F-USER>(?P<cond_user>\S+)|(?(cond_inv)(?:(?! from ).)*?|[^:]+)</F-USER> from <HOST>%(__on_port_opt)s(?: ssh\d*)?(?(cond_user): |(?:(?:(?! from ).)*)$) ^%(__prefix_line_sl)sROOT LOGIN REFUSED FROM <HOST> ^%(__prefix_line_sl)s[iI](?:llegal|nvalid) user .*? from <HOST>%(__suff)s$ ^%(__prefix_line_sl)sUser .+ from <HOST> not allowed because not listed in AllowUsers\s*%(__suff)s$ @@ -56,14 +56,15 @@ cmnfailre = ^%(__prefix_line_sl)s[aA]uthentication (?:failure|error|failed) for mdre-normal = mdre-ddos = ^%(__prefix_line_sl)sDid not receive identification string from <HOST> - ^%(__prefix_line_sl)sConnection closed by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$ - ^%(__prefix_line_sl)sConnection reset by <HOST> + ^%(__prefix_line_sl)sBad protocol version identification '.*' from <HOST> + ^%(__prefix_line_sl)sConnection (?:closed|reset) by%(__authng_user)s <HOST>%(__on_port_opt)s\s+\[preauth\]\s*$ ^%(__prefix_line_ml1)sSSH: Server;Ltype: (?:Authname|Version|Kex);Remote: <HOST>-\d+;[A-Z]\w+:.*%(__prefix_line_ml2)sRead from socket failed: Connection reset by peer%(__suff)s$ -mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No supported authentication methods available +mdre-extra = ^%(__prefix_line_sl)sReceived disconnect from <HOST>%(__on_port_opt)s:\s*14: No(?: supported)? authentication methods available ^%(__prefix_line_sl)sUnable to negotiate with <HOST>%(__on_port_opt)s: no matching <__alg_match> found. ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sUnable to negotiate a <__alg_match> ^%(__prefix_line_ml1)sConnection from <HOST>%(__on_port_opt)s%(__prefix_line_ml2)sno matching <__alg_match> found: + ^%(__prefix_line_sl)sDisconnected(?: from)?(?: (?:invalid|authenticating)) user <F-USER>\S+</F-USER> <HOST>%(__on_port_opt)s \[preauth\]\s*$ mdre-aggressive = %(mdre-ddos)s %(mdre-extra)s diff --git a/fail2ban/tests/config/jail.conf b/fail2ban/tests/config/jail.conf index 3dcbf634..b1a1707b 100644 --- a/fail2ban/tests/config/jail.conf +++ b/fail2ban/tests/config/jail.conf @@ -51,3 +51,51 @@ action = [tz_correct] enabled = true logtimezone = UTC+0200 + +[multi-log] +enabled = false +filter = +logpath = a.log + b.log + c.log +log2nd = %(logpath)s + d.log +action = action[actname='ban'] + action[actname='log', logpath="%(log2nd)s"] + action[actname='test'] + +[sshd-override-flt-opts] +filter = zzz-sshd-obsolete-multiline[logtype=short] +backend = systemd +prefregex = ^Test +failregex = ^Test unused <ADDR>$ +ignoreregex = ^Test ignore <ADDR>$ +journalmatch = _COMM=test +maxlines = 2 +usedns = no +enabled = false + +[checklogtype_jrnl] +filter = checklogtype +backend = systemd +action = action +enabled = false + +[checklogtype_file] +filter = checklogtype +backend = polling +logpath = README.md +action = action +enabled = false + +[checklogtype_test] +filter = checklogtype_test +backend = systemd +action = action +enabled = false + +[checklogtype_init] +filter = checklogtype_test[logtype=init] +backend = systemd +action = action +enabled = false diff --git a/fail2ban/tests/databasetestcase.py b/fail2ban/tests/databasetestcase.py index 248ea9fd..8cc394be 100644 --- a/fail2ban/tests/databasetestcase.py +++ b/fail2ban/tests/databasetestcase.py @@ -29,7 +29,7 @@ import tempfile import sqlite3 import shutil -from ..server.filter import FileContainer +from ..server.filter import FileContainer, Filter from ..server.mytime import MyTime from ..server.ticket import FailTicket from ..server.actions import Actions, Utils @@ -192,7 +192,7 @@ class DatabaseTest(LogCaptureTestCase): ticket.setAttempt(3) self.assertEqual(bans[0], ticket) # second ban found also: - self.assertEqual(bans[1].getIP(), "1.2.3.8") + self.assertEqual(bans[1].getID(), "1.2.3.8") # updated ? self.assertEqual(self.db.updateDb(Fail2BanDb.__version__), Fail2BanDb.__version__) # check current bans (should find 2 tickets after upgrade): @@ -212,19 +212,20 @@ class DatabaseTest(LogCaptureTestCase): self.jail.name in self.db.getJailNames(True), "Jail not added to database") - def testAddLog(self): + def _testAddLog(self): self.testAddJail() # Jail required _, filename = tempfile.mkstemp(".log", "Fail2BanDb_") self.fileContainer = FileContainer(filename, "utf-8") - self.db.addLog(self.jail, self.fileContainer) + pos = self.db.addLog(self.jail, self.fileContainer) + self.assertTrue(pos is None); # unknown previously self.assertIn(filename, self.db.getLogPaths(self.jail)) os.remove(filename) def testUpdateLog(self): - self.testAddLog() # Add log file + self._testAddLog() # Add log file # Write some text filename = self.fileContainer.getFileName() @@ -262,6 +263,15 @@ class DatabaseTest(LogCaptureTestCase): self.db.addLog(self.jail, self.fileContainer), None) os.remove(filename) + def testUpdateJournal(self): + self.testAddJail() # Jail required + # not yet updated: + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), None) + # update 3 times (insert and 2 updates) and check it was set (and overwritten): + for t in (1500000000, 1500000001, 1500000002): + self.db.updateJournal(self.jail, 'systemd-journal', t, 'TEST'+str(t)) + self.assertEqual(self.db.getJournalPos(self.jail, 'systemd-journal'), t) + def testAddBan(self): self.testAddJail() ticket = FailTicket("127.0.0.1", 0, ["abc\n"]) @@ -302,7 +312,7 @@ class DatabaseTest(LogCaptureTestCase): for i, ticket in enumerate(tickets): DefLogSys.debug('readtickets[%d]: %r', i, readtickets[i].getData()) DefLogSys.debug(' == tickets[%d]: %r', i, ticket.getData()) - self.assertEqual(readtickets[i].getIP(), ticket.getIP()) + self.assertEqual(readtickets[i].getID(), ticket.getID()) self.assertEqual(len(readtickets[i].getMatches()), len(ticket.getMatches())) self.pruneLog('[test-phase 2] simulate errors') @@ -344,10 +354,10 @@ class DatabaseTest(LogCaptureTestCase): def testDelBan(self): tickets = self._testAdd3Bans() # delete single IP: - self.db.delBan(self.jail, tickets[0].getIP()) + self.db.delBan(self.jail, tickets[0].getID()) self.assertEqual(len(self.db.getBans(jail=self.jail)), 2) # delete two IPs: - self.db.delBan(self.jail, tickets[1].getIP(), tickets[2].getIP()) + self.db.delBan(self.jail, tickets[1].getID(), tickets[2].getID()) self.assertEqual(len(self.db.getBans(jail=self.jail)), 0) def testFlushBans(self): @@ -368,9 +378,9 @@ class DatabaseTest(LogCaptureTestCase): # be returned self.assertEqual(len(self.db.getBans(jail=self.jail,bantime=-1)), 2) - def testGetBansMerged_MaxEntries(self): + def testGetBansMerged_MaxMatches(self): self.testAddJail() - maxEntries = 2 + maxMatches = 2 failures = [ {"matches": ["abc\n"], "user": set(['test'])}, {"matches": ["123\n"], "user": set(['test'])}, @@ -386,29 +396,44 @@ class DatabaseTest(LogCaptureTestCase): ticket.setAttempt(1) self.db.addBan(self.jail, ticket) # should retrieve 2 matches only, but count of all attempts: - self.db.maxEntries = maxEntries; + self.db.maxMatches = maxMatches; ticket = self.db.getBansMerged("127.0.0.1") - self.assertEqual(ticket.getIP(), "127.0.0.1") + self.assertEqual(ticket.getID(), "127.0.0.1") self.assertEqual(ticket.getAttempt(), len(failures)) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), matches2find[-maxMatches:]) # add more failures at once: ticket = FailTicket("127.0.0.1", MyTime.time() - 10, matches2find, data={"user": set(['test', 'root'])}) ticket.setAttempt(len(failures)) self.db.addBan(self.jail, ticket) # should retrieve 2 matches only, but count of all attempts: - self.db.maxEntries = maxEntries; ticket = self.db.getBansMerged("127.0.0.1") self.assertEqual(ticket.getAttempt(), 2 * len(failures)) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), matches2find[-maxMatches:]) # also using getCurrentBans: ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100) self.assertTrue(ticket is not None) self.assertEqual(ticket.getAttempt(), len(failures)) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), matches2find[-maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), matches2find[-maxMatches:]) + # maxmatches of jail < dbmaxmatches (so read 1 match and 0 matches): + ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100, + maxmatches=1) + self.assertEqual(len(ticket.getMatches()), 1) + self.assertEqual(ticket.getMatches(), failures[3]['matches']) + ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100, + maxmatches=0) + self.assertEqual(len(ticket.getMatches()), 0) + # dbmaxmatches = 0, should retrieve 0 matches by last ban: + ticket.setMatches(["1","2","3"]) + self.db.maxMatches = 0; + self.db.addBan(self.jail, ticket) + ticket = self.db.getCurrentBans(self.jail, "127.0.0.1", fromtime=MyTime.time()-100) + self.assertTrue(ticket is not None) + self.assertEqual(ticket.getAttempt(), len(failures)) + self.assertEqual(len(ticket.getMatches()), 0) def testGetBansMerged(self): self.testAddJail() @@ -431,13 +456,13 @@ class DatabaseTest(LogCaptureTestCase): # All for IP 127.0.0.1 ticket = self.db.getBansMerged("127.0.0.1") - self.assertEqual(ticket.getIP(), "127.0.0.1") + self.assertEqual(ticket.getID(), "127.0.0.1") self.assertEqual(ticket.getAttempt(), 70) self.assertEqual(ticket.getMatches(), ["abc\n", "123\n", "ABC\n"]) # All for IP 127.0.0.1 for single jail ticket = self.db.getBansMerged("127.0.0.1", jail=self.jail) - self.assertEqual(ticket.getIP(), "127.0.0.1") + self.assertEqual(ticket.getID(), "127.0.0.1") self.assertEqual(ticket.getAttempt(), 30) self.assertEqual(ticket.getMatches(), ["abc\n", "123\n"]) @@ -465,8 +490,8 @@ class DatabaseTest(LogCaptureTestCase): tickets = self.db.getBansMerged() self.assertEqual(len(tickets), 2) self.assertSortedEqual( - list(set(ticket.getIP() for ticket in tickets)), - [ticket.getIP() for ticket in tickets]) + list(set(ticket.getID() for ticket in tickets)), + [ticket.getID() for ticket in tickets]) tickets = self.db.getBansMerged(jail=jail2) self.assertEqual(len(tickets), 1) @@ -485,7 +510,7 @@ class DatabaseTest(LogCaptureTestCase): tickets = self.db.getCurrentBans(jail=self.jail) self.assertEqual(len(tickets), 2) ticket = self.db.getCurrentBans(jail=None, ip="127.0.0.1"); - self.assertEqual(ticket.getIP(), "127.0.0.1") + self.assertEqual(ticket.getID(), "127.0.0.1") # positive case (1 ticket not yet expired): tickets = self.db.getCurrentBans(jail=self.jail, forbantime=15, @@ -519,17 +544,22 @@ class DatabaseTest(LogCaptureTestCase): # test action together with database functionality self.testAddJail() # Jail required self.jail.database = self.db - actions = Actions(self.jail) + self.db.addJail(self.jail) + actions = self.jail.actions actions.add( "action_checkainfo", os.path.join(TEST_FILES_DIR, "action.d/action_checkainfo.py"), {}) + actions.banManager.setBanTotal(20) + self.jail._Jail__filter = flt = Filter(self.jail) + flt.failManager.setFailTotal(50) ticket = FailTicket("1.2.3.4") ticket.setAttempt(5) ticket.setMatches(['test', 'test']) self.jail.putFailTicket(ticket) actions._Actions__checkBan() self.assertLogged("ban ainfo %s, %s, %s, %s" % (True, True, True, True)) + self.assertLogged("jail info %d, %d, %d, %d" % (1, 21, 0, 50)) def testDelAndAddJail(self): self.testAddJail() # Add jail diff --git a/fail2ban/tests/datedetectortestcase.py b/fail2ban/tests/datedetectortestcase.py index 36471489..bc33bc05 100644 --- a/fail2ban/tests/datedetectortestcase.py +++ b/fail2ban/tests/datedetectortestcase.py @@ -103,7 +103,7 @@ class DateDetectorTest(LogCaptureTestCase): def testGetEpochPattern(self): self.__datedetector = DateDetector() - self.__datedetector.appendTemplate('(?<=\|\s){LEPOCH}(?=\s\|)') + self.__datedetector.appendTemplate(r'(?<=\|\s){LEPOCH}(?=\s\|)') # correct short/long epoch time, using all variants: for fact in (1, 1000, 1000000): for dateUnix in (1138049999, 32535244799): @@ -119,6 +119,15 @@ class DateDetectorTest(LogCaptureTestCase): log = log % dateLong datelog = self.datedetector.getTime(log) self.assertFalse(datelog) + + def testGetEpochPatternCut(self): + self.__datedetector = DateDetector() + self.__datedetector.appendTemplate(r'^type=\S+ msg=audit\(({EPOCH})') + # correct epoch time and cut out epoch string only (captured group only, not the whole match): + line = "type=USER_AUTH msg=audit(1106513999.000:987)" + datelog = self.datedetector.getTime(line) + timeMatch = datelog[1] + self.assertEqual([int(datelog[0]), line[timeMatch.start(1):timeMatch.end(1)]], [1106513999, '1106513999.000']) def testGetTime(self): log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure" @@ -330,6 +339,27 @@ class DateDetectorTest(LogCaptureTestCase): dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt) + def testNotAnchoredCollision(self): + # try for patterns with and without word boundaries: + for dp in (r'%H:%M:%S', r'{UNB}%H:%M:%S'): + dd = DateDetector() + dd.appendTemplate(dp) + # boundary of timestamp changes right and left (and time is left and right in line): + for fmt in ('%s test', '%8s test', 'test %s', 'test %8s'): + for dt in ( + '00:01:02', + '00:01:2', + '00:1:2', + '0:1:2', + '00:1:2', + '00:01:2', + '00:01:02', + '0:1:2', + '00:01:02', + ): + t = dd.getTime(fmt % dt) + self.assertEqual((t[0], t[1].group()), (1123970462.0, dt)) + def testAmbiguousInOrderedTemplates(self): dd = self.datedetector for (debit, line, cnt) in ( @@ -385,7 +415,7 @@ class DateDetectorTest(LogCaptureTestCase): self.assertRaises(Exception, t.getDate, 'no date line') -iso8601 = DatePatternRegex("%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") +iso8601 = DatePatternRegex(r"%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): @@ -495,6 +525,9 @@ class CustomDateFormatsTest(unittest.TestCase): (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 UTC] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 UTC] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 Z] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 +0000] server ..."), + (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 Z] server ..."), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: @@ -530,6 +563,9 @@ class CustomDateFormatsTest(unittest.TestCase): (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), + # no time (only date) in pattern, assume local 00:00:00 for H:M:S : + (1093989600.0, "^%Y-%m-%d**", '2004-09-01'), + (1093996800.0, "^%Y-%m-%d%z**", '2004-09-01Z'), ): logSys.debug('== test: %r', (matched, dp, line)) dd = DateDetector() diff --git a/fail2ban/tests/dummyjail.py b/fail2ban/tests/dummyjail.py index ec960290..fdeced8f 100644 --- a/fail2ban/tests/dummyjail.py +++ b/fail2ban/tests/dummyjail.py @@ -40,7 +40,6 @@ class DummyJail(Jail): self.lock = Lock() self.queue = [] super(DummyJail, self).__init__(name=name, backend=backend) - self.__db = None self.__actions = DummyActions(self) def __len__(self): @@ -55,6 +54,10 @@ class DummyJail(Jail): with self.lock: return bool(self.queue) + @property + def hasFailTickets(self): + return bool(self.queue) + def putFailTicket(self, ticket): with self.lock: self.queue.append(ticket) @@ -75,14 +78,6 @@ class DummyJail(Jail): pass @property - def database(self): - return self.__db; - - @database.setter - def database(self, value): - self.__db = value; - - @property def actions(self): return self.__actions; diff --git a/fail2ban/tests/fail2banclienttestcase.py b/fail2ban/tests/fail2banclienttestcase.py index c120128b..bc93dae7 100644 --- a/fail2ban/tests/fail2banclienttestcase.py +++ b/fail2ban/tests/fail2banclienttestcase.py @@ -37,7 +37,7 @@ from threading import Thread from ..client import fail2banclient, fail2banserver, fail2bancmdline from ..client.fail2bancmdline import Fail2banCmdLine -from ..client.fail2banclient import exec_command_line as _exec_client, VisualWait +from ..client.fail2banclient import exec_command_line as _exec_client, CSocket, VisualWait from ..client.fail2banserver import Fail2banServer, exec_command_line as _exec_server from .. import protocol from ..server import server @@ -142,20 +142,11 @@ fail2banclient.input_command = _test_input_command fail2bancmdline.PRODUCTION = \ fail2banserver.PRODUCTION = False - -def _out_file(fn, handle=logSys.debug): - """Helper which outputs content of the file at HEAVYDEBUG loglevels""" - if (handle != logSys.debug or logSys.getEffectiveLevel() <= logging.DEBUG): - handle('---- ' + fn + ' ----') - for line in fileinput.input(fn): - line = line.rstrip('\n') - handle(line) - handle('-'*30) - +_out_file = LogCaptureTestCase.dumpFile def _write_file(fn, mode, *lines): f = open(fn, mode) - f.write('\n'.join(lines)) + f.write('\n'.join(lines)+('\n' if lines else '')) f.close() def _read_file(fn): @@ -169,11 +160,13 @@ def _read_file(fn): def _start_params(tmp, use_stock=False, use_stock_cfg=None, - logtarget="/dev/null", db=":memory:", jails=("",), create_before_start=None + logtarget="/dev/null", db=":memory:", f2b_local=(), jails=("",), + create_before_start=None, ): cfg = pjoin(tmp, "config") if db == 'auto': db = pjoin(tmp, "f2b-db.sqlite3") + j_conf = 'jail.conf' if use_stock and STOCK: # copy config (sub-directories as alias): def ig_dirs(dir, files): @@ -195,6 +188,8 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, if r.match(line): line = "backend = polling" print(line) + # jails to local: + j_conf = 'jail.local' if jails else '' else: # just empty config directory without anything (only fail2ban.conf/jail.conf): os.mkdir(cfg) @@ -207,25 +202,35 @@ def _start_params(tmp, use_stock=False, use_stock_cfg=None, "pidfile = " + pjoin(tmp, "f2b.pid"), "backend = polling", "dbfile = " + db, + "dbmaxmatches = 100", "dbpurgeage = 1d", "", ) - _write_file(pjoin(cfg, "jail.conf"), "w", + # write jails (local or conf): + if j_conf: + _write_file(pjoin(cfg, j_conf), "w", *(( "[INCLUDES]", "", "[DEFAULT]", "tmp = " + tmp, "", )+jails) ) - if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover - _out_file(pjoin(cfg, "fail2ban.conf")) - _out_file(pjoin(cfg, "jail.conf")) + if f2b_local: + _write_file(pjoin(cfg, "fail2ban.local"), "w", *f2b_local) + if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover + _out_file(pjoin(cfg, "fail2ban.conf")) + _out_file(pjoin(cfg, "jail.conf")) + if f2b_local: + _out_file(pjoin(cfg, "fail2ban.local")) + if j_conf and j_conf != "jail.conf": + _out_file(pjoin(cfg, j_conf)) + # link stock actions and filters: if use_stock_cfg and STOCK: for n in use_stock_cfg: os.symlink(os.path.abspath(pjoin(STOCK_CONF_DIR, n)), pjoin(cfg, n)) if create_before_start: for n in create_before_start: - _write_file(n % {'tmp': tmp}, 'w', '') + _write_file(n % {'tmp': tmp}, 'w') # parameters (sock/pid and config, increase verbosity, set log, etc.): vvv, llev = (), "INFO" if unittest.F2B.log_level < logging.INFO: # pragma: no cover @@ -338,6 +343,7 @@ def with_foreground_server_thread(startextra={}): # to wait for end of server, default accept any exit code, because multi-threaded, # thus server can exit in-between... def _stopAndWaitForServerEnd(code=(SUCCESS, FAILED)): + tearDownMyTime() # if seems to be down - try to catch end phase (wait a bit for end:True to recognize down state): if not phase.get('end', None) and not os.path.exists(pjoin(tmp, "f2b.pid")): Utils.wait_for(lambda: phase.get('end', None) is not None, MID_WAITTIME) @@ -447,6 +453,14 @@ class Fail2banClientServerBase(LogCaptureTestCase): self.assertRaises(exitType, self.exec_command_line[0], (self.exec_command_line[1:] + startparams + args)) + def execCmdDirect(self, startparams, *args): + sock = startparams[startparams.index('-s')+1] + s = CSocket(sock) + try: + return s.send(args) + finally: + s.close() + # # Common tests # @@ -462,13 +476,54 @@ class Fail2banClientServerBase(LogCaptureTestCase): phase['end'] = True logSys.debug("end of test worker") - @with_foreground_server_thread() + @with_foreground_server_thread(startextra={'f2b_local':( + "[Thread]", + "stacksize = 128" + "", + )}) def testStartForeground(self, tmp, startparams): + # check thread options were set: + self.pruneLog() + self.execCmd(SUCCESS, startparams, "get", "thread") + self.assertLogged("{'stacksize': 128}") # several commands to server: self.execCmd(SUCCESS, startparams, "ping") self.execCmd(FAILED, startparams, "~~unknown~cmd~failed~~") self.execCmd(SUCCESS, startparams, "echo", "TEST-ECHO") + @with_tmpdir + @with_kill_srv + def testStartFailsInForeground(self, tmp): + if not server.Fail2BanDb: # pragma: no cover + raise unittest.SkipTest('Skip test because no database') + dbname = pjoin(tmp,"tmp.db") + db = server.Fail2BanDb(dbname) + # set inappropriate DB version to simulate an irreparable error by start: + cur = db._db.cursor() + cur.executescript("UPDATE fail2banDb SET version = 555") + cur.close() + # timeout (thread will stop foreground server): + startparams = _start_params(tmp, db=dbname, logtarget='INHERITED') + phase = {'stop': True} + def _stopTimeout(startparams, phase): + if not Utils.wait_for(lambda: not phase['stop'], MAX_WAITTIME): + # print('==== STOP ====') + self.execCmdDirect(startparams, 'stop') + th = Thread( + name="_TestCaseWorker", + target=_stopTimeout, + args=(startparams, phase) + ) + th.start() + # test: + try: + self.execCmd(FAILED, ("-f",) + startparams, "start") + finally: + phase['stop'] = False + th.join() + self.assertLogged("Attempt to travel to future version of database", + "Exit with code 255", all=True) + class Fail2banClientTest(Fail2banClientServerBase): @@ -578,6 +633,11 @@ class Fail2banClientTest(Fail2banClientServerBase): os.kill(pid, signal.SIGCONT) self.assertLogged("timed out") self.pruneLog() + # check readline module available (expected by interactive client) + try: + import readline + except ImportError as e: + raise unittest.SkipTest('Skip test because of import error: %s' % e) # interactive client chat with started server: INTERACT += [ "echo INTERACT-ECHO", @@ -633,12 +693,6 @@ class Fail2banClientTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "--async", "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "miss/f2b.sock"), "start") - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## not running self.execCmd(FAILED, (), "-c", pjoin(tmp, "config"), "-s", pjoin(tmp, "f2b.sock"), "reload") @@ -734,12 +788,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged("Base configuration directory " + pjoin(tmp, "miss") + " does not exist") self.pruneLog() - ## wrong socket - self.execCmd(FAILED, (), - "-c", pjoin(tmp, "config"), "-x", "-s", pjoin(tmp, "miss/f2b.sock")) - self.assertLogged("There is no directory " + pjoin(tmp, "miss") + " to contain the socket file") - self.pruneLog() - ## already exists: open(pjoin(tmp, "f2b.sock"), 'a').close() self.execCmd(FAILED, (), @@ -853,7 +901,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "usedns = no", "maxretry = 3", "findtime = 10m", - "failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>", + r"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>", "datepattern = {^LN-BEG}EPOCH", "ignoreip = 127.0.0.1/8 ::1", # just to cover ignoreip in jailreader/transmitter "", @@ -869,8 +917,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "logpath = " + test1log, " " + test2log if 2 in enabled else "", " " + test3log if 2 in enabled else "", - "failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>", - " ^\s*error <F-ERRCODE>401|403</F-ERRCODE> from <HOST>" \ + r"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>", + r" ^\s*error <F-ERRCODE>401|403</F-ERRCODE> from <HOST>" \ if 2 in enabled else "", "enabled = true" if 1 in enabled else "", "", @@ -878,7 +926,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = ", " test-action2[name='%(__name__)s', restore='restored: <restored>', info=', err-code: <F-ERRCODE>']" \ if 2 in actions else "", - " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>']" + " test-action2[name='%(__name__)s', actname=test-action3, _exec_once=1, restore='restored: <restored>'," " actionflush=<_use_flush_>]" \ if 3 in actions else "", "logpath = " + test2log, @@ -927,10 +975,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'broken-jail' skipped, because of wrong configuration", all=True) # enable both jails, 3 logs for jail1, etc... - # truncate test-log - we should not find unban/ban again by reload: self.pruneLog("[test-phase 1b]") _write_jail_cfg(actions=[1,2]) - _write_file(test1log, "w+") if unittest.F2B.log_level < logging.DEBUG: # pragma: no cover _out_file(test1log) self.execCmd(SUCCESS, startparams, "reload") @@ -991,9 +1037,9 @@ class Fail2banServerTest(Fail2banClientServerBase): # leave action2 just to test restored interpolation: _write_jail_cfg(actions=[2,3]) - # write new failures: self.pruneLog("[test-phase 2b]") - _write_file(test2log, "w+", *( + # write new failures: + _write_file(test2log, "a+", *( (str(int(MyTime.time())) + " error 403 from 192.0.2.2: test 2",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.3: test 2",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.4: test 2",) * 3 + @@ -1005,13 +1051,19 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "2 ticket(s) in 'test-jail2", "5 ticket(s) in 'test-jail1", all=True, wait=MID_WAITTIME) + # ban manually to cover restore in restart (phase 2c): + self.execCmd(SUCCESS, startparams, + "set", "test-jail2", "banip", "192.0.2.9") + self.assertLogged( + "3 ticket(s) in 'test-jail2", wait=MID_WAITTIME) self.assertLogged( "[test-jail1] Ban 192.0.2.2", "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", "[test-jail1] Ban 192.0.2.8", "[test-jail2] Ban 192.0.2.4", - "[test-jail2] Ban 192.0.2.8", all=True) + "[test-jail2] Ban 192.0.2.8", + "[test-jail2] Ban 192.0.2.9", all=True) # test ips at all not visible for jail2: self.assertNotLogged( "[test-jail2] Found 192.0.2.2", @@ -1021,10 +1073,30 @@ class Fail2banServerTest(Fail2banClientServerBase): all=True) # if observer available wait for it becomes idle (write all tickets to db): _observer_wait_idle() - - # rotate logs: - _write_file(test1log, "w+") - _write_file(test2log, "w+") + # test banned command: + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned'), (0, [ + {'test-jail1': ['192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']}, + {'test-jail2': ['192.0.2.4', '192.0.2.9', '192.0.2.8']} + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'banned', '192.0.2.1', '192.0.2.4', '192.0.2.222'), (0, [ + ['test-jail1'], ['test-jail1', 'test-jail2'], [] + ] + )) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned')[1], [ + '192.0.2.4', '192.0.2.1', '192.0.2.8', '192.0.2.3', '192.0.2.2']) + self.assertSortedEqual(self.execCmdDirect(startparams, + 'get', 'test-jail2', 'banned')[1], [ + '192.0.2.4', '192.0.2.9', '192.0.2.8']) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3')[1], 1) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.9')[1], 0) + self.assertEqual(self.execCmdDirect(startparams, + 'get', 'test-jail1', 'banned', '192.0.2.3', '192.0.2.9')[1], [1, 0]) # restart jail without unban all: self.pruneLog("[test-phase 2c]") @@ -1033,15 +1105,17 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Reload finished.", "Restore Ban", - "2 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) + "3 ticket(s) in 'test-jail2", all=True, wait=MID_WAITTIME) # stop/start and unban/restore ban: self.assertLogged( - "Jail 'test-jail2' stopped", - "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", + "Jail 'test-jail2' stopped", + "Jail 'test-jail2' started", "[test-jail2] Restore Ban 192.0.2.4", - "[test-jail2] Restore Ban 192.0.2.8", all=True + "[test-jail2] Restore Ban 192.0.2.8", + "[test-jail2] Restore Ban 192.0.2.9", all=True ) # test restored is 1 (only test-action2): self.assertLogged( @@ -1064,6 +1138,17 @@ class Fail2banServerTest(Fail2banClientServerBase): "stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22", "stdout: '[test-jail2] test-action3: ++ ban 192.0.2.22 ", all=True, wait=MID_WAITTIME) + # get banned ips: + _observer_wait_idle() + self.pruneLog("[test-phase 2d.1]") + self.execCmd(SUCCESS, startparams, "get", "test-jail2", "banip", "\n") + self.assertLogged( + "192.0.2.4", "192.0.2.8", "192.0.2.21", "192.0.2.22", all=True, wait=MID_WAITTIME) + self.pruneLog("[test-phase 2d.2]") + self.execCmd(SUCCESS, startparams, "get", "test-jail1", "banip") + self.assertLogged( + "192.0.2.1", "192.0.2.2", "192.0.2.3", "192.0.2.4", "192.0.2.8", all=True, wait=MID_WAITTIME) + # restart jail with unban all: self.pruneLog("[test-phase 2e]") self.execCmd(SUCCESS, startparams, @@ -1075,7 +1160,8 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail2' stopped", "Jail 'test-jail2' started", "[test-jail2] Unban 192.0.2.4", - "[test-jail2] Unban 192.0.2.8", all=True + "[test-jail2] Unban 192.0.2.8", + "[test-jail2] Unban 192.0.2.9", all=True ) # test unban (action2): self.assertLogged( @@ -1129,7 +1215,7 @@ class Fail2banServerTest(Fail2banClientServerBase): # now write failures again and check already banned (jail1 was alive the whole time) and new bans occurred (jail1 was alive the whole time): self.pruneLog("[test-phase 5]") - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.1: test 5",) * 3 + (str(int(MyTime.time())) + " error 403 from 192.0.2.5: test 5",) * 3 + (str(int(MyTime.time())) + " failure 401 from 192.0.2.6: test 5",) * 3 @@ -1149,12 +1235,40 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertNotLogged("[test-jail1] Found 192.0.2.5") # unban single ips: - self.pruneLog("[test-phase 6]") + self.pruneLog("[test-phase 6a]") self.execCmd(SUCCESS, startparams, "--async", "unban", "192.0.2.5", "192.0.2.6") self.assertLogged( "192.0.2.5 is not banned", - "[test-jail1] Unban 192.0.2.6", all=True + "[test-jail1] Unban 192.0.2.6", all=True, wait=MID_WAITTIME + ) + # unban ips by subnet (cidr/mask): + self.pruneLog("[test-phase 6b]") + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.2/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.2", + "[test-jail1] Unban 192.0.2.3", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "unban", "192.0.2.8/31", "192.0.2.100/31") + self.assertLogged( + "[test-jail1] Unban 192.0.2.8", + "192.0.2.100/31 is not banned", all=True, wait=MID_WAITTIME) + + # ban/unban subnet(s): + self.pruneLog("[test-phase 6c]") + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "banip", "192.0.2.96/28", "192.0.2.112/28") + self.assertLogged( + "[test-jail1] Ban 192.0.2.96/28", + "[test-jail1] Ban 192.0.2.112/28", all=True, wait=MID_WAITTIME + ) + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "unbanip", "192.0.2.64/26"); # contains both subnets .96/28 and .112/28 + self.assertLogged( + "[test-jail1] Unban 192.0.2.96/28", + "[test-jail1] Unban 192.0.2.112/28", all=True, wait=MID_WAITTIME ) # reload all (one jail) with unban all: @@ -1166,8 +1280,6 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertLogged( "Jail 'test-jail1' reloaded", "[test-jail1] Unban 192.0.2.1", - "[test-jail1] Unban 192.0.2.2", - "[test-jail1] Unban 192.0.2.3", "[test-jail1] Unban 192.0.2.4", all=True ) # no restart occurred, no more ban (unbanned all using option "--unban"): @@ -1175,8 +1287,6 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", "[test-jail1] Ban 192.0.2.1", - "[test-jail1] Ban 192.0.2.2", - "[test-jail1] Ban 192.0.2.3", "[test-jail1] Ban 192.0.2.4", all=True ) @@ -1227,6 +1337,14 @@ class Fail2banServerTest(Fail2banClientServerBase): "Jail 'test-jail1' stopped", "Jail 'test-jail1' started", all=True, wait=MID_WAITTIME) + # Coverage for pickle of IPAddr (as string): + self.pruneLog("[test-phase end-3]") + self.execCmd(SUCCESS, startparams, + "--async", "set", "test-jail1", "addignoreip", "192.0.2.1/32", "2001:DB8::1/96") + self.execCmd(SUCCESS, startparams, + "--async", "get", "test-jail1", "ignoreip") + self.assertLogged("192.0.2.1/32", "2001:DB8::1/96", all=True) + # test action.d/nginx-block-map.conf -- @unittest.F2B.skip_if_cfg_missing(action="nginx-block-map") @with_foreground_server_thread(startextra={ @@ -1240,7 +1358,7 @@ class Fail2banServerTest(Fail2banClientServerBase): 'backend = polling', 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', - 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + 'action = nginx-block-map[srv_cmd="echo nginx", srv_pid="%(tmp)s/f2b.pid", blck_lst_file="%(tmp)s/blck-lst.map"]', ' blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; <Definition/actionban>\', email="Fail2Ban <fail2ban@localhost>", ' 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=<name>]', 'filter =', @@ -1280,6 +1398,8 @@ class Fail2banServerTest(Fail2banClientServerBase): self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check nginx reload is logged (pid of fail2ban is used to simulate success check nginx is running): + self.assertLogged("stdout: 'nginx -qt'", "stdout: 'nginx -s reload'", all=True) # check blocklist_de substitution (e. g. new-line after <matches>): self.assertLogged( "stdout: '*** curl --fail --data-urlencode server=Fail2Ban <fail2ban@localhost>" @@ -1308,6 +1428,126 @@ class Fail2banServerTest(Fail2banClientServerBase): mp = _read_file(mpfn) self.assertEqual(mp, '') + @unittest.F2B.skip_if_cfg_missing(filter="sendmail-auth") + @with_foreground_server_thread(startextra={ + # create log-file (avoid "not found" errors): + 'create_before_start': ('%(tmp)s/test.log',), + 'use_stock': True, + # fail2ban.local: + 'f2b_local': ( + '[DEFAULT]', + 'dbmaxmatches = 1' + ), + # jail.local config: + 'jails': ( + # default: + '''test_action = dummy[actionstart_on_demand=1, init="start: %(__name__)s", target="%(tmp)s/test.txt", + actionban='<known/actionban>; echo "found: <jail.found> / <jail.found_total>, banned: <jail.banned> / <jail.banned_total>" + echo "<matches>"; printf "=====\\n%%b\\n=====\\n\\n" "<matches>" >> <target>', + actionstop='<known/actionstop>; echo "stats <name> - found: <jail.found_total>, banned: <jail.banned_total>"']''', + # jail sendmail-auth: + '[sendmail-auth]', + 'backend = polling', + 'usedns = no', + 'logpath = %(tmp)s/test.log', + 'action = %(test_action)s', + 'filter = sendmail-auth[logtype=short]', + 'datepattern = ^Epoch', + 'maxretry = 3', + 'maxmatches = 2', + 'enabled = true', + # jail sendmail-reject: + '[sendmail-reject]', + 'backend = polling', + 'usedns = no', + 'logpath = %(tmp)s/test.log', + 'action = %(test_action)s', + 'filter = sendmail-reject[logtype=short]', + 'datepattern = ^Epoch', + 'maxretry = 3', + 'enabled = true', + ) + }) + def testServerJails_Sendmail(self, tmp, startparams): + cfg = pjoin(tmp, "config") + lgfn = '%(tmp)s/test.log' % {'tmp': tmp} + tofn = '%(tmp)s/test.txt' % {'tmp': tmp} + + smaut_msg = ( + str(int(MyTime.time())) + ' smtp1 sm-mta[5133]: s1000000000001: [192.0.2.1]: possible SMTP attack: command=AUTH, count=1', + str(int(MyTime.time())) + ' smtp1 sm-mta[5133]: s1000000000002: [192.0.2.1]: possible SMTP attack: command=AUTH, count=2', + str(int(MyTime.time())) + ' smtp1 sm-mta[5133]: s1000000000003: [192.0.2.1]: possible SMTP attack: command=AUTH, count=3', + ) + smrej_msg = ( + str(int(MyTime.time())) + ' smtp1 sm-mta[21134]: s2000000000001: ruleset=check_rcpt, arg1=<123@example.com>, relay=xxx.dynamic.example.com [192.0.2.2], reject=550 5.7.1 <123@example.com>... Relaying denied. Proper authentication required.', + str(int(MyTime.time())) + ' smtp1 sm-mta[21134]: s2000000000002: ruleset=check_rcpt, arg1=<345@example.com>, relay=xxx.dynamic.example.com [192.0.2.2], reject=550 5.7.1 <345@example.com>... Relaying denied. Proper authentication required.', + str(int(MyTime.time())) + ' smtp1 sm-mta[21134]: s3000000000003: ruleset=check_rcpt, arg1=<567@example.com>, relay=xxx.dynamic.example.com [192.0.2.2], reject=550 5.7.1 <567@example.com>... Relaying denied. Proper authentication required.', + ) + + self.pruneLog("[test-phase sendmail-auth]") + # write log: + _write_file(lgfn, "w+", *smaut_msg) + # wait and check it caused banned (and dump in the test-file): + self.assertLogged( + "[sendmail-auth] Ban 192.0.2.1", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) + _out_file(tofn) + td = _read_file(tofn) + # check matches (maxmatches = 2, so only 2 & 3 available): + m = smaut_msg[0] + self.assertNotIn(m, td) + for m in smaut_msg[1:]: + self.assertIn(m, td) + + self.pruneLog("[test-phase sendmail-reject]") + # write log: + _write_file(lgfn, "a+", *smrej_msg) + # wait and check it caused banned (and dump in the test-file): + self.assertLogged( + "[sendmail-reject] Ban 192.0.2.2", "stdout: 'found: 0 / 3, banned: 1 / 1'", + "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) + _out_file(tofn) + td = _read_file(tofn) + # check matches (no maxmatches, so all matched messages are available): + for m in smrej_msg: + self.assertIn(m, td) + + self.pruneLog("[test-phase restart sendmail-*]") + # restart jails (active ban-tickets should be restored): + self.execCmd(SUCCESS, startparams, + "reload", "--restart", "--all") + # wait a bit: + self.assertLogged( + "Reload finished.", + "stdout: 'stats sendmail-auth - found: 3, banned: 1'", + "stdout: 'stats sendmail-reject - found: 3, banned: 1'", + "[sendmail-auth] Restore Ban 192.0.2.1", "1 ticket(s) in 'sendmail-auth'", all=True, wait=MID_WAITTIME) + # check matches again - (dbmaxmatches = 1), so it should be only last match after restart: + td = _read_file(tofn) + m = smaut_msg[-1] + self.assertLogged(m) + self.assertIn(m, td) + for m in smaut_msg[0:-1]: + self.assertNotLogged(m) + self.assertNotIn(m, td) + # wait for restore of reject-jail: + self.assertLogged( + "[sendmail-reject] Restore Ban 192.0.2.2", "1 ticket(s) in 'sendmail-reject'", all=True, wait=MID_WAITTIME) + td = _read_file(tofn) + m = smrej_msg[-1] + self.assertLogged(m) + self.assertIn(m, td) + for m in smrej_msg[0:-1]: + self.assertNotLogged(m) + self.assertNotIn(m, td) + + self.pruneLog("[test-phase stop server]") + # stop server and wait for end: + self.stopAndWaitForServerEnd(SUCCESS) + + # just to debug actionstop: + self.assertFalse(exists(tofn)) + @with_foreground_server_thread() def testServerObserver(self, tmp, startparams): cfg = pjoin(tmp, "config") @@ -1343,7 +1583,7 @@ class Fail2banServerTest(Fail2banClientServerBase): "action = test-action1[name='%(__name__)s']", " test-action2[name='%(__name__)s']", "logpath = " + test1log, - "failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>:\s*<F-MSG>.*</F-MSG>$", + r"failregex = ^\s*failure <F-ERRCODE>401|403</F-ERRCODE> from <HOST>:\s*<F-MSG>.*</F-MSG>$", "enabled = true", "", ) @@ -1389,7 +1629,7 @@ class Fail2banServerTest(Fail2banClientServerBase): wakeObs = False _observer_wait_before_incrban(lambda: wakeObs) # write again (IP already bad): - _write_file(test1log, "w+", *( + _write_file(test1log, "a+", *( (str(int(MyTime.time())) + " failure 401 from 192.0.2.11: I'm very bad \"hacker\" `` $(echo test)",) * 2 )) # wait for ban: @@ -1397,6 +1637,11 @@ class Fail2banServerTest(Fail2banClientServerBase): "stdout: '[test-jail1] test-action1: ++ ban 192.0.2.11 -c 2 -t 300 : ", "stdout: '[test-jail1] test-action2: ++ ban 192.0.2.11 -c 2 -t 300 : ", all=True, wait=MID_WAITTIME) + # get banned ips with time: + self.pruneLog("[test-phase 2) time+10m - get-ips]") + self.execCmd(SUCCESS, startparams, "get", "test-jail1", "banip", "--with-time") + self.assertLogged( + "192.0.2.11", "+ 300 =", all=True, wait=MID_WAITTIME) # unblock observer here and wait it is done: wakeObs = True _observer_wait_idle() @@ -1411,6 +1656,44 @@ class Fail2banServerTest(Fail2banClientServerBase): "stdout: '[test-jail1] test-action2: ++ prolong 192.0.2.11 -c 2 -t 600 : ", all=True, wait=MID_WAITTIME) + # get banned ips with time: + _observer_wait_idle() + self.pruneLog("[test-phase 2) time+11m - get-ips]") + self.execCmd(SUCCESS, startparams, "get", "test-jail1", "banip", "--with-time") + self.assertLogged( + "192.0.2.11", "+ 600 =", all=True, wait=MID_WAITTIME) + + # test stop with busy observer: + self.pruneLog("[test-phase end) stop on busy observer]") + tearDownMyTime() + a = {'state': 0} + obsMain = Observers.Main + def _long_action(): + logSys.info('++ observer enters busy state ...') + a['state'] = 1 + Utils.wait_for(lambda: a['state'] == 2, MAX_WAITTIME) + obsMain.db_purge(); # does nothing (db is already None) + logSys.info('-- observer leaves busy state.') + obsMain.add('call', _long_action) + obsMain.add('call', lambda: None) + # wait observer enter busy state: + Utils.wait_for(lambda: a['state'] == 1, MAX_WAITTIME) + # overwrite default wait time (normally 5 seconds): + obsMain_stop = obsMain.stop + def _stop(wtime=(0.01 if unittest.F2B.fast else 0.1), forceQuit=True): + return obsMain_stop(wtime, forceQuit) + obsMain.stop = _stop + # stop server and wait for end: + self.stopAndWaitForServerEnd(SUCCESS) + # check observer and db state: + self.assertNotLogged('observer leaves busy state') + self.assertFalse(obsMain.idle) + self.assertEqual(obsMain._ObserverThread__db, None) + # server is exited without wait for observer, stop it now: + a['state'] = 2 + self.assertLogged('observer leaves busy state', wait=True) + obsMain.join() + # test multiple start/stop of the server (threaded in foreground) -- if False: # pragma: no cover @with_foreground_server_thread() diff --git a/fail2ban/tests/fail2banregextestcase.py b/fail2ban/tests/fail2banregextestcase.py index 44acfd35..213ea89b 100644 --- a/fail2ban/tests/fail2banregextestcase.py +++ b/fail2ban/tests/fail2banregextestcase.py @@ -25,6 +25,8 @@ __license__ = "GPL" import os import sys +import tempfile +import unittest from ..client import fail2banregex from ..client.fail2banregex import Fail2banRegex, get_opt_parser, exec_command_line, output, str2LogLevel @@ -34,7 +36,7 @@ from .utils import CONFIG_DIR fail2banregex.logSys = logSys def _test_output(*args): - logSys.notice(args[0]) + logSys.notice('output: %s', args[0]) fail2banregex.output = _test_output @@ -51,6 +53,10 @@ def _Fail2banRegex(*args): logSys.setLevel(str2LogLevel(opts.log_level)) return (opts, args, Fail2banRegex(opts)) +def _test_exec(*args): + (opts, args, fail2banRegex) = _Fail2banRegex(*args) + return fail2banRegex.start(args) + class ExitException(Exception): def __init__(self, code): self.code = code @@ -75,27 +81,54 @@ def _test_exec_command_line(*args): sys.stderr = _org['stderr'] return _exit_code +def _reset(): + # reset global warn-counter: + from ..server.filter import _decode_line_warn + _decode_line_warn.clear() -class Fail2banRegexTest(LogCaptureTestCase): +STR_00 = "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" +STR_00_NODT = "[sshd] error: PAM: Authentication failure for kevin from 192.0.2.0" + +RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>" +RE_00_ID = r"Authentication failure for <F-ID>.*?</F-ID> from <ADDR>$" +RE_00_USER = r"Authentication failure for <F-USER>.*?</F-USER> from <ADDR>$" + +FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") +FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") +FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") + +# STR_ML_SSHD -- multiline log-excerpt with two sessions: +# 192.0.2.1 (sshd[32307]) makes 2 failed attempts using public keys (without "Disconnecting: Too many authentication"), +# and delayed success on accepted (STR_ML_SSHD_OK) or no success by close on preauth phase (STR_ML_SSHD_FAIL) +# 192.0.2.2 (sshd[32310]) makes 2 failed attempts using public keys (with "Disconnecting: Too many authentication"), +# and closed on preauth phase +STR_ML_SSHD = """Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:03 srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth] +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.2 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:... +Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.2 [preauth]""" +STR_ML_SSHD_OK = "Nov 28 09:16:06 srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:..." +STR_ML_SSHD_FAIL = "Nov 28 09:16:06 srv sshd[32307]: Connection closed by 192.0.2.1 [preauth]" - RE_00 = r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>" - FILENAME_01 = os.path.join(TEST_FILES_DIR, "testcase01.log") - FILENAME_02 = os.path.join(TEST_FILES_DIR, "testcase02.log") - FILENAME_WRONGCHAR = os.path.join(TEST_FILES_DIR, "testcase-wrong-char.log") +FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") +FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') +FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log') +FILTER_ZZZ_SSHD = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-sshd-obsolete-multiline.conf') - FILENAME_SSHD = os.path.join(TEST_FILES_DIR, "logs", "sshd") - FILTER_SSHD = os.path.join(CONFIG_DIR, 'filter.d', 'sshd.conf') - FILENAME_ZZZ_SSHD = os.path.join(TEST_FILES_DIR, 'zzz-sshd-obsolete-multiline.log') - FILTER_ZZZ_SSHD = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-sshd-obsolete-multiline.conf') +FILENAME_ZZZ_GEN = os.path.join(TEST_FILES_DIR, "logs", "zzz-generic-example") +FILTER_ZZZ_GEN = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-generic-example.conf') - FILENAME_ZZZ_GEN = os.path.join(TEST_FILES_DIR, "logs", "zzz-generic-example") - FILTER_ZZZ_GEN = os.path.join(TEST_CONFIG_DIR, 'filter.d', 'zzz-generic-example.conf') + +class Fail2banRegexTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" LogCaptureTestCase.setUp(self) setUpMyTime() + _reset() def tearDown(self): """Call after every test case.""" @@ -103,58 +136,65 @@ class Fail2banRegexTest(LogCaptureTestCase): tearDownMyTime() def testWrongRE(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertFalse(_test_exec( "test", r".** from <HOST>$" - ) - self.assertFalse(fail2banRegex.start(args)) + )) + self.assertLogged("Unable to compile regular expression") + self.assertLogged("multiple repeat", "at position 2", all=False); # details of failed compilation + self.pruneLog() + self.assertFalse(_test_exec( + "test", r"^(?:(?P<type>A)|B)? (?(typo)...) from <ADDR>" + )) self.assertLogged("Unable to compile regular expression") + self.assertLogged("unknown group name: 'typo'", "at position 23", all=False); # details of failed compilation def testWrongIngnoreRE(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertFalse(_test_exec( "--datepattern", "{^LN-BEG}EPOCH", "test", r".*? from <HOST>$", r".**" - ) - self.assertFalse(fail2banRegex.start(args)) + )) self.assertLogged("Unable to compile regular expression") + self.assertLogged("multiple repeat", "at position 2", all=False); # details of failed compilation + + def testWrongFilterOptions(self): + self.assertFalse(_test_exec( + "test", "flt[a='x,y,z',b=z,y,x]" + )) + self.assertLogged("Wrong filter name or options", "wrong syntax at 14: y,x", all=True) def testDirectFound(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--print-no-missed", - "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + STR_00, r"Authentication failure for .*? from <HOST>$" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') def testDirectNotFound(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "--print-all-missed", - "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + STR_00, r"XYZ from <HOST>$" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: 1 lines, 0 ignored, 0 matched, 1 missed') def testDirectIgnored(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "--print-all-ignored", - "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", + STR_00, r"Authentication failure for .*? from <HOST>$", r"kevin from 192.0.2.0$" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: 1 lines, 1 ignored, 0 matched, 0 missed') def testDirectRE_1(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", - Fail2banRegexTest.FILENAME_01, - Fail2banRegexTest.RE_00 - ) - self.assertTrue(fail2banRegex.start(args)) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + FILENAME_01, RE_00 + )) + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') self.assertLogged('Error decoding line'); self.assertLogged('Continuing to process line ignoring invalid characters') @@ -163,69 +203,78 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10') def testDirectRE_1raw(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", - Fail2banRegexTest.FILENAME_01, - Fail2banRegexTest.RE_00 - ) - self.assertTrue(fail2banRegex.start(args)) - self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') + FILENAME_01, RE_00 + )) + self.assertLogged('Lines: 19 lines, 0 ignored, 19 matched, 0 missed') def testDirectRE_1raw_noDns(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", "--raw", "--usedns=no", - Fail2banRegexTest.FILENAME_01, - Fail2banRegexTest.RE_00 - ) - self.assertTrue(fail2banRegex.start(args)) - self.assertLogged('Lines: 19 lines, 0 ignored, 13 matched, 6 missed') + FILENAME_01, RE_00 + )) + self.assertLogged('Lines: 19 lines, 0 ignored, 16 matched, 3 missed') + # usage of <F-ID>\S+</F-ID> causes raw handling automatically: + self.pruneLog() + self.assertTrue(_test_exec( + "-d", "^Epoch", + "1490349000 test failed.dns.ch", "^\s*test <F-ID>\S+</F-ID>" + )) + self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed', all=True) + self.assertNotLogged('Unable to find a corresponding IP address') def testDirectRE_2(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--print-all-matched", - Fail2banRegexTest.FILENAME_02, - Fail2banRegexTest.RE_00 - ) - self.assertTrue(fail2banRegex.start(args)) + FILENAME_02, RE_00 + )) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') def testVerbose(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + self.assertTrue(_test_exec( + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--timezone", "UTC+0200", "--verbose", "--verbose-date", "--print-no-missed", - Fail2banRegexTest.FILENAME_02, - Fail2banRegexTest.RE_00 - ) - self.assertTrue(fail2banRegex.start(args)) + FILENAME_02, RE_00 + )) self.assertLogged('Lines: 13 lines, 0 ignored, 5 matched, 8 missed') self.assertLogged('141.3.81.106 Sun Aug 14 11:53:59 2005') self.assertLogged('141.3.81.106 Sun Aug 14 11:54:59 2005') def testVerboseFullSshd(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages "-v", "--verbose-date", "--print-all-matched", "--print-all-ignored", "-c", CONFIG_DIR, - Fail2banRegexTest.FILENAME_SSHD, "sshd" - ) - self.assertTrue(fail2banRegex.start(args)) + FILENAME_SSHD, "sshd" + )) # test failure line and not-failure lines both presents: self.assertLogged("[29116]: User root not allowed because account is locked", "[29116]: Received disconnect from 1.2.3.4", all=True) + self.pruneLog() + # show real options: + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "-vv", "-c", CONFIG_DIR, + "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.1", + "sshd[logtype=short]" + )) + # tet logtype is specified and set in real options: + self.assertLogged("Real filter options :", "'logtype': 'short'", all=True) + self.assertNotLogged("'logtype': 'file'", "'logtype': 'journal'", all=True) def testFastSshd(self): - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages "--print-all-matched", "-c", CONFIG_DIR, - Fail2banRegexTest.FILENAME_ZZZ_SSHD, "sshd.conf[mode=normal]" - ) - self.assertTrue(fail2banRegex.start(args)) + FILENAME_ZZZ_SSHD, "sshd.conf[mode=normal]" + )) # test failure line and all not-failure lines presents: self.assertLogged( "[29116]: Connection from 192.0.2.4", @@ -234,94 +283,280 @@ class Fail2banRegexTest(LogCaptureTestCase): def testMultilineSshd(self): # by the way test of missing lines by multiline in `for bufLine in orgLineBuffer[int(fullBuffer):]` - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages "--print-all-matched", "--print-all-missed", - "-c", os.path.dirname(Fail2banRegexTest.FILTER_ZZZ_SSHD), - Fail2banRegexTest.FILENAME_ZZZ_SSHD, os.path.basename(Fail2banRegexTest.FILTER_ZZZ_SSHD) - ) - self.assertTrue(fail2banRegex.start(args)) + "-c", os.path.dirname(FILTER_ZZZ_SSHD), + FILENAME_ZZZ_SSHD, os.path.basename(FILTER_ZZZ_SSHD) + )) # test "failure" line presents (2nd part only, because multiline fewer precise): self.assertLogged( "[29116]: Received disconnect from 192.0.2.4", all=True) def testFullGeneric(self): # by the way test of ignoreregex (specified in filter file)... - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages - Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILTER_ZZZ_GEN+"[mode=test]" - ) - self.assertTrue(fail2banRegex.start(args)) + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + FILENAME_ZZZ_GEN, FILTER_ZZZ_GEN+"[mode=test]" + )) def testDirectMultilineBuf(self): # test it with some pre-lines also to cover correct buffer scrolling (all multi-lines printed): for preLines in (0, 20): self.pruneLog("[test-phase %s]" % preLines) - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "--usedns", "no", "-d", "^Epoch", "--print-all-matched", "--maxlines", "5", ("1490349000 TEST-NL\n"*preLines) + "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: %s lines, 0 ignored, 2 matched, %s missed' % (preLines+4, preLines+2)) # both matched lines were printed: self.assertLogged("| 1490349000 FAIL", "| 1490349001 HOST 192.0.2.34", all=True) def testDirectMultilineBufDebuggex(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "--usedns", "no", "-d", "^Epoch", "--debuggex", "--print-all-matched", "--maxlines", "5", "1490349000 FAIL\n1490349000 TEST1\n1490349001 TEST2\n1490349001 HOST 192.0.2.34", r"^\s*FAIL\s*$<SKIPLINES>^\s*HOST <HOST>\s*$" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') # the sequence in args-dict is currently undefined (so can be 1st argument) self.assertLogged("&flags=m", "?flags=m") def testSinglelineWithNLinContent(self): # - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "--usedns", "no", "-d", "^Epoch", "--print-all-matched", - "1490349000 FAIL: failure\nhost: 192.0.2.35", + "-L", "2", "1490349000 FAIL: failure\nhost: 192.0.2.35", r"^\s*FAIL:\s*.*\nhost:\s+<HOST>$" - ) - self.assertTrue(fail2banRegex.start(args)) - self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') + )) + self.assertLogged('Lines: 2 lines, 0 ignored, 2 matched, 0 missed') def testRegexEpochPatterns(self): - (opts, args, fail2banRegex) = _Fail2banRegex( + self.assertTrue(_test_exec( "-r", "-d", r"^\[{LEPOCH}\]\s+", "--maxlines", "5", "[1516469849] 192.0.2.1 FAIL: failure\n" "[1516469849551] 192.0.2.2 FAIL: failure\n" "[1516469849551000] 192.0.2.3 FAIL: failure\n" "[1516469849551.000] 192.0.2.4 FAIL: failure", r"^<HOST> FAIL\b" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Lines: 4 lines, 0 ignored, 4 matched, 0 missed') - def testWrongFilterFile(self): - # use test log as filter file to cover eror cases... - (opts, args, fail2banRegex) = _Fail2banRegex( - Fail2banRegexTest.FILENAME_ZZZ_GEN, Fail2banRegexTest.FILENAME_ZZZ_GEN + def testRegexSubnet(self): + self.assertTrue(_test_exec( + "-vv", "-d", r"^\[{LEPOCH}\]\s+", "--maxlines", "5", + "[1516469849] 192.0.2.1 FAIL: failure\n" + "[1516469849] 192.0.2.1/24 FAIL: failure\n" + "[1516469849] 2001:DB8:FF:FF::1 FAIL: failure\n" + "[1516469849] 2001:DB8:FF:FF::1/60 FAIL: failure\n", + r"^<SUBNET> FAIL\b" + )) + self.assertLogged('Lines: 4 lines, 0 ignored, 4 matched, 0 missed') + self.assertLogged('192.0.2.0/24', '2001:db8:ff:f0::/60', all=True) + + def testFrmtOutput(self): + # id/ip only: + self.assertTrue(_test_exec('-o', 'id', STR_00, RE_00_ID)) + self.assertLogged('output: %s' % 'kevin') + self.pruneLog() + # multiple id combined to a tuple (id, tuple_id): + self.assertTrue(_test_exec('-o', 'id', '-d', '{^LN-BEG}EPOCH', + '1591983743.667 192.0.2.1 192.0.2.2', + r'^\s*<F-ID/> <F-TUPLE_ID>\S+</F-TUPLE_ID>')) + self.assertLogged('output: %s' % str(('192.0.2.1', '192.0.2.2'))) + self.pruneLog() + # multiple id combined to a tuple, id first - (id, tuple_id_1, tuple_id_2): + self.assertTrue(_test_exec('-o', 'id', '-d', '{^LN-BEG}EPOCH', + '1591983743.667 left 192.0.2.3 right', + r'^\s*<F-TUPLE_ID_1>\S+</F-TUPLE_ID_1> <F-ID/> <F-TUPLE_ID_2>\S+</F-TUPLE_ID_2>')) + self.assertLogged('output: %s' % str(('192.0.2.3', 'left', 'right'))) + self.pruneLog() + # id had higher precedence as ip-address: + self.assertTrue(_test_exec('-o', 'id', '-d', '{^LN-BEG}EPOCH', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*<F-TUPLE_ID_1>\S+</F-TUPLE_ID_1> <F-ID><ADDR>:<F-PORT/></F-ID> <F-TUPLE_ID_2>\S+</F-TUPLE_ID_2>')) + self.assertLogged('output: %s' % str(('[192.0.2.4]:12345', 'left', 'right'))) + self.pruneLog() + # ip is not id anymore (if IP-address deviates from ID): + self.assertTrue(_test_exec('-o', 'ip', '-d', '{^LN-BEG}EPOCH', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*<F-TUPLE_ID_1>\S+</F-TUPLE_ID_1> <F-ID><ADDR>:<F-PORT/></F-ID> <F-TUPLE_ID_2>\S+</F-TUPLE_ID_2>')) + self.assertNotLogged('output: %s' % str(('[192.0.2.4]:12345', 'left', 'right'))) + self.assertLogged('output: %s' % '192.0.2.4') + self.pruneLog() + self.assertTrue(_test_exec('-o', 'ID:<fid> | IP:<ip>', '-d', '{^LN-BEG}EPOCH', + '1591983743.667 left [192.0.2.4]:12345 right', + r'^\s*<F-TUPLE_ID_1>\S+</F-TUPLE_ID_1> <F-ID><ADDR>:<F-PORT/></F-ID> <F-TUPLE_ID_2>\S+</F-TUPLE_ID_2>')) + self.assertLogged('output: %s' % 'ID:'+str(('[192.0.2.4]:12345', 'left', 'right'))+' | IP:192.0.2.4') + self.pruneLog() + # row with id : + self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_ID)) + self.assertLogged('output: %s' % "['kevin'", "'ip4': '192.0.2.0'", "'fid': 'kevin'", all=True) + self.pruneLog() + # row with ip : + self.assertTrue(_test_exec('-o', 'row', STR_00, RE_00_USER)) + self.assertLogged('output: %s' % "['192.0.2.0'", "'ip4': '192.0.2.0'", "'user': 'kevin'", all=True) + self.pruneLog() + # log msg : + self.assertTrue(_test_exec('-o', 'msg', STR_00, RE_00_USER)) + self.assertLogged('output: %s' % STR_00) + self.pruneLog() + # item of match (user): + self.assertTrue(_test_exec('-o', 'user', STR_00, RE_00_USER)) + self.assertLogged('output: %s' % 'kevin') + self.pruneLog() + # complex substitution using tags (ip, user, family): + self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <family>', STR_00, RE_00_USER)) + self.assertLogged('output: %s' % '192.0.2.0, kevin, inet4') + self.pruneLog() + + def testStalledIPByNoFailFrmtOutput(self): + opts = ( + '-c', CONFIG_DIR, + "-d", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", ) - self.assertFalse(fail2banRegex.start(args)) + log = ( + 'May 27 00:16:33 host sshd[2364]: User root not allowed because account is locked\n' + 'May 27 00:16:33 host sshd[2364]: Received disconnect from 192.0.2.76 port 58846:11: Bye Bye [preauth]' + ) + _test = lambda *args: _test_exec(*(opts + args)) + # with MLFID from prefregex and IP after failure obtained from F-NOFAIL RE: + self.assertTrue(_test('-o', 'IP:<ip>', log, 'sshd')) + self.assertLogged('IP:192.0.2.76') + self.pruneLog() + # test diverse ID/IP constellations: + def _test_variants(flt="sshd", prefix=""): + # with different ID/IP from failregex (ID/User from first, IP from second message): + self.assertTrue(_test('-o', 'ID:"<fid>" | IP:<ip> | U:<F-USER>', log, + flt+'[failregex="' + '^'+prefix+'<F-ID>User <F-USER>\S+</F-USER></F-ID> not allowed\n' + '^'+prefix+'Received disconnect from <ADDR>' + '"]')) + self.assertLogged('ID:"User root" | IP:192.0.2.76 | U:root') + self.pruneLog() + # with different ID/IP from failregex (User from first, ID and IP from second message): + self.assertTrue(_test('-o', 'ID:"<fid>" | IP:<ip> | U:<F-USER>', log, + flt+'[failregex="' + '^'+prefix+'User <F-USER>\S+</F-USER> not allowed\n' + '^'+prefix+'Received disconnect from <F-ID><ADDR> port \d+</F-ID>' + '"]')) + self.assertLogged('ID:"192.0.2.76 port 58846" | IP:192.0.2.76 | U:root') + self.pruneLog() + # first with sshd and prefregex: + _test_variants() + # the same without prefregex and MLFID directly in failregex (no merge with prefregex groups): + _test_variants('common', prefix="\s*\S+ sshd\[<F-MLFID>\d+</F-MLFID>\]:\s+") + + def testNoDateTime(self): + # datepattern doesn't match: + self.assertTrue(_test_exec('-d', '{^LN-BEG}EPOCH', '-o', 'Found-ID:<F-ID>', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "Found-ID:kevin", all=True) + self.pruneLog() + # explicitly no datepattern: + self.assertTrue(_test_exec('-d', '{NONE}', '-o', 'Found-ID:<F-ID>', STR_00_NODT, RE_00_ID)) + self.assertLogged( + "Found-ID:kevin", all=True) + self.assertNotLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", all=True) + + def testIncompleteDateTime(self): + # datepattern in followed lines doesn't match previously known pattern + line is too short + # (logging break-off, no flush, etc): + self.assertTrue(_test_exec( + '-o', 'Found-ADDR:<ip>', + '192.0.2.1 - - [02/May/2021:18:40:55 +0100] "GET / HTTP/1.1" 302 328 "-" "Mozilla/5.0" "-"\n' + '192.0.2.2 - - [02/May/2021:18:40:55 +0100\n' + '192.0.2.3 - - [02/May/2021:18:40:55', + '^<ADDR>')) + self.assertLogged( + "Found-ADDR:192.0.2.1", "Found-ADDR:192.0.2.2", "Found-ADDR:192.0.2.3", all=True) + + def testFrmtOutputWrapML(self): + unittest.F2B.SkipIfCfgMissing(stock=True) + # complex substitution using tags and message (ip, user, msg): + self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, publickey=invalid]')) + # be sure we don't have IP in one line and have it in another: + lines = STR_ML_SSHD.split("\n") + self.assertTrue('192.0.2.2' not in lines[-2] and '192.0.2.2' in lines[-1]) + # but both are in output "merged" with IP and user: + self.assertLogged( + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (mode is not aggressive): + self.assertNotLogged('192.0.2.1, git, ') + + # test with publickey (nofail) - would not produce output for 192.0.2.1 because accepted: + self.pruneLog("[test-phase 1] mode=aggressive & publickey=nofail + OK (accepted)") + self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_OK, 'sshd[logtype=short, mode=aggressive]')) + self.assertLogged( + '192.0.2.2, git, '+lines[-4], + '192.0.2.2, git, '+lines[-3], + '192.0.2.2, git, '+lines[-2], + '192.0.2.2, git, '+lines[-1], + all=True) + # nothing should be found for 192.0.2.1 (access gained so failures ignored): + self.assertNotLogged('192.0.2.1, git, ') + + # now same test but "accepted" replaced with "closed" on preauth phase: + self.pruneLog("[test-phase 2] mode=aggressive & publickey=nofail + FAIL (closed on preauth)") + self.assertTrue(_test_exec('-o', '<ip>, <F-USER>, <msg>', + '-c', CONFIG_DIR, '--usedns', 'no', + STR_ML_SSHD + "\n" + STR_ML_SSHD_FAIL, 'sshd[logtype=short, mode=aggressive]')) + # 192.0.2.1 should be found for every failure (2x failed key + 1x closed): + lines = STR_ML_SSHD.split("\n")[0:2] + STR_ML_SSHD_FAIL.split("\n")[-1:] + self.assertLogged( + '192.0.2.1, git, '+lines[-3], + '192.0.2.1, git, '+lines[-2], + '192.0.2.1, git, '+lines[-1], + all=True) + + def testOutputNoPendingFailuresAfterGained(self): + unittest.F2B.SkipIfCfgMissing(stock=True) + # connect finished without authorization must generate a failure, because + # connect started will produce pending failure which gets reset by gained + # connect authorized. + self.assertTrue(_test_exec('-o', 'failure from == <ip> ==', + '-c', CONFIG_DIR, '-d', '{NONE}', + 'svc[1] connect started 192.0.2.3\n' + 'svc[1] connect finished 192.0.2.3\n' + 'svc[2] connect started 192.0.2.4\n' + 'svc[2] connect authorized 192.0.2.4\n' + 'svc[2] connect finished 192.0.2.4\n', + 'common[prefregex="^svc\[<F-MLFID>\d+</F-MLFID>\] connect <F-CONTENT>.+</F-CONTENT>$"' + ', failregex="' + '^started\n' + '^<F-NOFAIL><F-MLFFORGET>finished</F-MLFFORGET></F-NOFAIL> <ADDR>\n' + '^<F-MLFGAINED>authorized</F-MLFGAINED> <ADDR>' + '", maxlines=1]' + )) + self.assertLogged('failure from == 192.0.2.3 ==') + self.assertNotLogged('failure from == 192.0.2.4 ==') - def _reset(self): - # reset global warn-counter: - from ..server.filter import _decode_line_warn - _decode_line_warn.clear() + def testWrongFilterFile(self): + # use test log as filter file to cover eror cases... + self.assertFalse(_test_exec( + FILENAME_ZZZ_GEN, FILENAME_ZZZ_GEN + )) def testWronChar(self): - self._reset() - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", - Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD - ) - self.assertTrue(fail2banRegex.start(args)) + unittest.F2B.SkipIfCfgMissing(stock=True) + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + FILENAME_WRONGCHAR, FILTER_SSHD + )) self.assertLogged('Lines: 4 lines, 0 ignored, 2 matched, 2 missed') self.assertLogged('Error decoding line') @@ -331,20 +566,49 @@ class Fail2banRegexTest(LogCaptureTestCase): self.assertLogged('Nov 8 00:16:12 main sshd[32547]: pam_succeed_if(sshd:auth): error retrieving information about user llinco') def testWronCharDebuggex(self): - self._reset() - (opts, args, fail2banRegex) = _Fail2banRegex( - "-l", "notice", # put down log-level, because of too many debug-messages - "--datepattern", "^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", + unittest.F2B.SkipIfCfgMissing(stock=True) + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", "--debuggex", "--print-all-matched", - Fail2banRegexTest.FILENAME_WRONGCHAR, Fail2banRegexTest.FILTER_SSHD, + FILENAME_WRONGCHAR, FILTER_SSHD, r"llinco[^\\]" - ) - self.assertTrue(fail2banRegex.start(args)) + )) self.assertLogged('Error decoding line') self.assertLogged('Lines: 4 lines, 1 ignored, 2 matched, 1 missed') self.assertLogged('https://') + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'1490349000 \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n', + u'1490349000 \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' + ): + fout.write(l.encode(enc)) + fout.close() + + self.assertTrue(_test_exec( + "-l", "notice", # put down log-level, because of too many debug-messages + "--encoding", enc, + "--datepattern", r"^EPOCH", + fname, r"Failed .* from <HOST>", + )) + + self.assertLogged(" encoding : %s" % enc, + "Lines: 2 lines, 0 ignored, 2 matched, 0 missed", all=True) + self.assertNotLogged("Missed line(s)") + finally: + fout.close() + os.unlink(fname) + def testExecCmdLine_Usage(self): self.assertNotEqual(_test_exec_command_line(), 0) self.pruneLog() @@ -356,15 +620,48 @@ class Fail2banRegexTest(LogCaptureTestCase): def testExecCmdLine_Direct(self): self.assertEqual(_test_exec_command_line( '-l', 'info', - "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", - r"Authentication failure for .*? from <HOST>$" + STR_00, r"Authentication failure for .*? from <HOST>$" ), 0) self.assertLogged('Lines: 1 lines, 0 ignored, 1 matched, 0 missed') def testExecCmdLine_MissFailID(self): self.assertNotEqual(_test_exec_command_line( '-l', 'info', - "Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 192.0.2.0", - r"Authentication failure" + STR_00, r"Authentication failure" ), 0) self.assertLogged('No failure-id group in ') + + def testExecCmdLine_ErrorParam(self): + # single line error: + self.assertNotEqual(_test_exec_command_line( + '-l', 'notice', '-d', '%:%.%-', 'LOG', 'RE' + ), 0) + self.assertLogged('ERROR: Failed to set datepattern') + # verbose (traceback/callstack): + self.pruneLog() + self.assertNotEqual(_test_exec_command_line( + '-v', '-d', '%:%.%-', 'LOG', 'RE' + ), 0) + self.assertLogged('Failed to set datepattern') + + def testLogtypeSystemdJournal(self): # pragma: no cover + if not fail2banregex.FilterSystemd: + raise unittest.SkipTest('Skip test because no systemd backend available') + self.assertTrue(_test_exec( + "systemd-journal", FILTER_ZZZ_GEN + +'[journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' + +' failregex="^\x00\x01\x02dummy regex, never match <F-ID>xxx</F-ID>"]' + )) + self.assertLogged("'logtype': 'journal'") + self.assertNotLogged("'logtype': 'file'") + self.assertLogged('Lines: 0 lines, 0 ignored, 0 matched, 0 missed') + self.pruneLog() + # logtype specified explicitly (should win in filter): + self.assertTrue(_test_exec( + "systemd-journal", FILTER_ZZZ_GEN + +'[logtype=file,' + +' journalmatch="SYSLOG_IDENTIFIER=\x01\x02dummy\x02\x01",' + +' failregex="^\x00\x01\x02dummy regex, never match <F-ID>xxx</F-ID>"]' + )) + self.assertLogged("'logtype': 'file'") + self.assertNotLogged("'logtype': 'journal'") diff --git a/fail2ban/tests/failmanagertestcase.py b/fail2ban/tests/failmanagertestcase.py index ad89ec76..42b0fbd2 100644 --- a/fail2ban/tests/failmanagertestcase.py +++ b/fail2ban/tests/failmanagertestcase.py @@ -69,9 +69,9 @@ class AddFailure(unittest.TestCase): self.assertEqual(self.__failManager.getFailTotal(), 0) self.__failManager.setFailTotal(13) - def testFailManagerAdd_MaxEntries(self): - maxEntries = 2 - self.__failManager.maxEntries = maxEntries + def testFailManagerAdd_MaxMatches(self): + maxMatches = 2 + self.__failManager.maxMatches = maxMatches failures = ["abc\n", "123\n", "ABC\n", "1234\n"] # add failures sequential: i = 80 @@ -86,8 +86,8 @@ class AddFailure(unittest.TestCase): ticket = manFailList["127.0.0.1"] # should retrieve 2 matches only, but count of all attempts (4): self.assertEqual(ticket.getAttempt(), len(failures)) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxMatches:]) # add more failures at once: ticket = FailTicket("127.0.0.1", 1000002000 - 10, failures) ticket.setAttempt(len(failures)) @@ -98,8 +98,8 @@ class AddFailure(unittest.TestCase): ticket = manFailList["127.0.0.1"] # should retrieve 2 matches only, but count of all attempts (8): self.assertEqual(ticket.getAttempt(), 2 * len(failures)) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxMatches:]) # add self ticket again: self.__failManager.addFailure(ticket) # @@ -108,8 +108,16 @@ class AddFailure(unittest.TestCase): ticket = manFailList["127.0.0.1"] # same matches, but +1 attempt (9) self.assertEqual(ticket.getAttempt(), 2 * len(failures) + 1) - self.assertEqual(len(ticket.getMatches()), maxEntries) - self.assertEqual(ticket.getMatches(), failures[len(failures) - maxEntries:]) + self.assertEqual(len(ticket.getMatches()), maxMatches) + self.assertEqual(ticket.getMatches(), failures[len(failures) - maxMatches:]) + # no matches by maxMatches == 0 : + self.__failManager.maxMatches = 0 + self.__failManager.addFailure(ticket) + manFailList = self.__failManager._FailManager__failList + ticket = manFailList["127.0.0.1"] + self.assertEqual(len(ticket.getMatches()), 0) + # test set matches None to None: + ticket.setMatches(None) def testFailManagerMaxTime(self): self._addDefItems() @@ -142,8 +150,8 @@ class AddFailure(unittest.TestCase): self.__failManager.setMaxRetry(5) #ticket = FailTicket('193.168.0.128', None) ticket = self.__failManager.toBan() - self.assertEqual(ticket.getIP(), "193.168.0.128") - self.assertTrue(isinstance(ticket.getIP(), (str, IPAddr))) + self.assertEqual(ticket.getID(), "193.168.0.128") + self.assertTrue(isinstance(ticket.getID(), (str, IPAddr))) # finish with rudimentary tests of the ticket # verify consistent str @@ -172,9 +180,9 @@ class AddFailure(unittest.TestCase): def testWindow(self): self._addDefItems() ticket = self.__failManager.toBan() - self.assertNotEqual(ticket.getIP(), "100.100.10.10") + self.assertNotEqual(ticket.getID(), "100.100.10.10") ticket = self.__failManager.toBan() - self.assertNotEqual(ticket.getIP(), "100.100.10.10") + self.assertNotEqual(ticket.getID(), "100.100.10.10") self.assertRaises(FailManagerEmpty, self.__failManager.toBan) def testBgService(self): diff --git a/fail2ban/tests/files/action.d/action_checkainfo.py b/fail2ban/tests/files/action.d/action_checkainfo.py index 63dd4f5b..c5eaf0f8 100644 --- a/fail2ban/tests/files/action.d/action_checkainfo.py +++ b/fail2ban/tests/files/action.d/action_checkainfo.py @@ -8,6 +8,9 @@ class TestAction(ActionBase): self._logSys.info("ban ainfo %s, %s, %s, %s", aInfo["ipmatches"] != '', aInfo["ipjailmatches"] != '', aInfo["ipfailures"] > 0, aInfo["ipjailfailures"] > 0 ) + self._logSys.info("jail info %d, %d, %d, %d", + aInfo["jail.banned"], aInfo["jail.banned_total"], aInfo["jail.found"], aInfo["jail.found_total"] + ) def unban(self, aInfo): pass diff --git a/fail2ban/tests/files/action.d/action_modifyainfo.py b/fail2ban/tests/files/action.d/action_modifyainfo.py index b003edef..58787dd7 100644 --- a/fail2ban/tests/files/action.d/action_modifyainfo.py +++ b/fail2ban/tests/files/action.d/action_modifyainfo.py @@ -12,4 +12,9 @@ class TestAction(ActionBase): del aInfo['ip'] self._logSys.info("%s unban deleted aInfo IP", self._name) + def flush(self): + # intended error to cover no unhandled exception occurs in flush + # as well as unbans are done individually after errored flush. + raise ValueError("intended error") + Action = TestAction diff --git a/fail2ban/tests/files/filter.d/testcase02.conf b/fail2ban/tests/files/filter.d/testcase02.conf new file mode 100644 index 00000000..99b3bb45 --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.conf @@ -0,0 +1,12 @@ +[INCLUDES] + +# Read common prefixes. If any customizations available -- read them from +# common.local +before = testcase-common.conf + +[Definition] + +_daemon = sshd +__prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? + +failregex = %(__prefix_line)s test
\ No newline at end of file diff --git a/fail2ban/tests/files/filter.d/testcase02.local b/fail2ban/tests/files/filter.d/testcase02.local new file mode 100644 index 00000000..bfc81d4b --- /dev/null +++ b/fail2ban/tests/files/filter.d/testcase02.local @@ -0,0 +1,4 @@ +[Definition] + +# no options here, coverage for testFilterReaderSubstKnown: +# avoid to overwrite known/option with unmodified (not available) value of option from .local config file
\ No newline at end of file diff --git a/fail2ban/tests/files/logs/apache-auth b/fail2ban/tests/files/logs/apache-auth index 93040b1d..fd3bae82 100644 --- a/fail2ban/tests/files/logs/apache-auth +++ b/fail2ban/tests/files/logs/apache-auth @@ -134,6 +134,14 @@ # failJSON: { "time": "2018-03-28T01:31:42", "match": true , "host": "91.49.82.139" } [Wed Mar 28 01:31:42.355210 2018] [ssl:error] [pid 6586] [client 91.49.82.139:58028] AH02033: No hostname was provided via SNI for a name based virtual host +# failJSON: { "match": false, "desc": "ignore mod_evasive errors in normal mode (gh-2548)" } +[Thu Oct 17 18:43:40.160521 2019] [evasive20:error] [pid 22589] [client 192.0.2.1:56175] client denied by server configuration: /path/index.php, referer: https://hostname/path/ + +# filterOptions: {"mode": "aggressive"} + +# failJSON: { "time": "2019-10-17T18:43:40", "match": true, "host": "192.0.2.1", "desc": "accept mod_evasive errors in aggressive mode (gh-2548)" } +[Thu Oct 17 18:43:40.160521 2019] [evasive20:error] [pid 22589] [client 192.0.2.1:56175] client denied by server configuration: /path/index.php, referer: https://hostname/path/ + # filterOptions: {"logging": "syslog"} # failJSON: { "time": "2005-02-15T16:23:00", "match": true , "host": "192.0.2.1", "desc": "using syslog (ErrorLog syslog)" } diff --git a/fail2ban/tests/files/logs/apache-modsecurity b/fail2ban/tests/files/logs/apache-modsecurity index 3ca2e074..7e2f8c86 100644 --- a/fail2ban/tests/files/logs/apache-modsecurity +++ b/fail2ban/tests/files/logs/apache-modsecurity @@ -3,3 +3,9 @@ # failJSON: { "time": "2013-12-28T09:18:05", "match": true , "host": "32.65.254.69", "desc": "additional entry (and exact one space)" } [Sat Dec 28 09:18:05 2013] [error] [client 32.65.254.69] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] + +# failJSON: { "time": "2018-09-28T09:18:06", "match": true , "host": "192.0.2.1", "desc": "two client entries in message (gh-2247)" } +[Sat Sep 28 09:18:06 2018] [error] [client 192.0.2.1:55555] [client 192.0.2.1] ModSecurity: [file "/etc/httpd/modsecurity.d/10_asl_rules.conf"] [line "635"] [id "340069"] [rev "4"] [msg "Atomicorp.com UNSUPPORTED DELAYED Rules: Web vulnerability scanner"] [severity "CRITICAL"] Access denied with code 403 (phase 2). Pattern match "(?:nessus(?:_is_probing_you_|test)|^/w00tw00t\\\\.at\\\\.)" at REQUEST_URI. [hostname "192.81.249.191"] [uri "/w00tw00t.at.blackhats.romanian.anti-sec:)"] [unique_id "4Q6RdsBR@b4AAA65LRUAAAAA"] + +# failJSON: { "time": "2020-05-09T00:35:52", "match": true , "host": "192.0.2.2", "desc": "new format - apache 2.4 and php-fpm (gh-2717)" } +[Sat May 09 00:35:52.389262 2020] [:error] [pid 22406:tid 139985298601728] [client 192.0.2.2:47762] [client 192.0.2.2] ModSecurity: Access denied with code 401 (phase 2). Operator EQ matched 1 at IP:blocked. [file "/etc/httpd/modsecurity.d/activated_rules/modsecurity_wp_login.conf"] [line "14"] [id "500000"] [msg "Ip address blocked for 15 minutes, more than 5 login attempts in 3 minutes."] [hostname "example.com"] [uri "/wp-login.php"] [unique_id "XrYlGL5IY3I@EoLOgAAAA8"], referer: https://example.com/wp-login.php diff --git a/fail2ban/tests/files/logs/apache-noscript b/fail2ban/tests/files/logs/apache-noscript index 91c55571..eb78e8e7 100644 --- a/fail2ban/tests/files/logs/apache-noscript +++ b/fail2ban/tests/files/logs/apache-noscript @@ -17,4 +17,9 @@ # failJSON: { "time": "2013-12-23T07:49:01", "match": true , "host": "204.232.202.107" } [Mon Dec 23 07:49:01.981912 2013] [:error] [pid 3790] [client 204.232.202.107:46301] script '/var/www/timthumb.php' not found or unable to stat # failJSON: { "time": "2018-03-11T08:56:20", "match": true , "host": "192.0.2.106", "desc": "php-fpm error" } -[Sun Mar 11 08:56:20.913548 2018] [proxy_fcgi:error] [pid 742:tid 140142593419008] [client 192.0.2.106:50900] AH01071: Got error 'Primary script unknown\n'
\ No newline at end of file +[Sun Mar 11 08:56:20.913548 2018] [proxy_fcgi:error] [pid 742:tid 140142593419008] [client 192.0.2.106:50900] AH01071: Got error 'Primary script unknown\n' +# failJSON: { "time": "2019-07-09T14:27:42", "match": true , "host": "127.0.0.1", "desc": "script unknown, without \n (gh-2466)" } +[Tue Jul 09 14:27:42.650548 2019] [proxy_fcgi:error] [pid 22075:tid 140322524440320] [client 127.0.0.1] AH01071: Got error 'Primary script unknown' + +# failJSON: { "time": "2020-08-11T08:56:17", "match": true , "host": "192.0.2.1", "desc": "script not found with AH02811 and cgi-bin path segment in script (gh-2805)" } +[Tue Aug 11 08:56:17.580412 2020] [cgi:error] [pid 27550:tid 140110750279424] [client 192.0.2.1:18071] AH02811: script not found or unable to stat: /usr/lib/cgi-bin/kerbynet diff --git a/fail2ban/tests/files/logs/apache-overflows b/fail2ban/tests/files/logs/apache-overflows index 376114c4..4be013eb 100644 --- a/fail2ban/tests/files/logs/apache-overflows +++ b/fail2ban/tests/files/logs/apache-overflows @@ -3,6 +3,8 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# failJSON: { "time": "2010-03-15T16:04:06", "match": true , "host": "192.0.2.1", "desc": "AH00126 failure, gh-2908" } +[Sat Mar 15 16:04:06.105212 2010] [core:error] [pid 17408] [client 192.0.2.1:55280] AH00126: Invalid URI in request GET /static/../../../a/../../../../etc/passwd HTTP/1.1 # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } diff --git a/fail2ban/tests/files/logs/asterisk b/fail2ban/tests/files/logs/asterisk index 7bd011fc..ab31fa6f 100644 --- a/fail2ban/tests/files/logs/asterisk +++ b/fail2ban/tests/files/logs/asterisk @@ -19,6 +19,8 @@ [2012-02-13 17:44:26] NOTICE[1638] chan_iax2.c: Host 1.2.3.4 failed MD5 authentication for 'Fail2ban' (e7df7cd2ca07f4f1ab415d457a6e1c13 != 53ac4bc41ee4ec77888ed4aa50677247) # failJSON: { "time": "2013-02-05T23:44:42", "match": true , "host": "1.2.3.4" } [2013-02-05 23:44:42] NOTICE[436][C-00000fa9] chan_sip.c: Call from '' (1.2.3.4:10836) to extension '0972598285108' rejected because extension not found in context 'default'. +# failJSON: { "time": "2005-01-18T17:39:50", "match": true , "host": "1.2.3.4" } +[Jan 18 17:39:50] NOTICE[12049]: res_pjsip_session.c:2337 new_invite: Call from 'anonymous' (TCP:[1.2.3.4]:61470) to extension '9011+442037690237' rejected because extension not found in context 'default'. # failJSON: { "time": "2013-03-26T15:47:54", "match": true , "host": "1.2.3.4" } [2013-03-26 15:47:54] NOTICE[1237] chan_sip.c: Registration from '"100"sip:100@1.2.3.4' failed for '1.2.3.4:23930' - No matching peer found # failJSON: { "time": "2013-05-13T07:10:53", "match": true , "host": "1.2.3.4" } @@ -35,7 +37,12 @@ # failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "192.168.55.152" } [2013-11-11 14:33:38] WARNING[6756][C-0000001d] Ext. s: "Rejecting unknown SIP connection from 192.168.55.152" - +# failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "192.168.55.152" } +[2013-11-11 14:33:38] WARNING[8447][C-00000244] Ext. s: "Rejecting unknown SIP connection from 192.168.55.152:52126" +# failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "2001:db8::1" } +[2013-11-11 14:33:38] WARNING[12124][C-00000001] Ext. s: "Rejecting unknown SIP connection from 2001:db8::1" +# failJSON: { "time": "2013-11-11T14:33:38", "match": true , "host": "2001:db8::1" } +[2013-11-11 14:33:38] WARNING[12124][C-00000001] Ext. s: "Rejecting unknown SIP connection from [2001:db8::1]:5060" # failJSON: { "time": "2004-11-04T18:30:40", "match": true , "host": "192.168.200.100" } Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in handle_request_register: Registration from '<sip:301@example.com>' failed for '192.168.200.100:36998' - Wrong password @@ -45,6 +52,8 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # failed authentication attempt on INVITE using PJSIP # failJSON: { "time": "2015-05-24T08:42:16", "match": true, "host": "10.250.251.252" } [2015-05-24 08:42:16] SECURITY[4583] res_security_log.c: SecurityEvent="ChallengeResponseFailed",EventTV="2015-05-24T08:42:16.296+0300",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="<unknown>",SessionID="17a483d-eb8cc0-556164ab@1.2.3.4",LocalAddress="IPV4/UDP/1.2.3.4/5060",RemoteAddress="IPV4/UDP/10.250.251.252/5060",Challenge="1432446136/6d16ccf29ff59d423c6d548af00bf9b4",Response="849dfcf133d8156f77ef11a9194119df",ExpectedResponse="" +# failJSON: { "time": "2019-09-20T19:12:43", "match": true, "host": "192.0.2.2", "desc": "TLS before address, gh-2531" } +[2019-09-20 19:12:43] SECURITY[1724] res_security_log.c: SecurityEvent="ChallengeResponseFailed",EventTV="2019-09-20T19:12:43.659-0500",Severity="Error",Service="PJSIP",EventVersion="1",AccountID="<unknown>",SessionID="3686a690-f8ccac10-5677c924-51b54926",LocalAddress="IPV4/TLS/1.2.3.4/5062",RemoteAddress="IPV4/TLS/192.0.2.2/30245",Challenge="1569024763/510a7e1ed568b93ce283d1b16bc17a15",Response="8e181448412899ccb20ea585efc8bab0",ExpectedResponse="" # SessionID may contain any special characters and spaces # failJSON: { "time": "2015-05-25T07:19:19", "match": true, "host": "10.250.251.252" } @@ -109,3 +118,10 @@ Nov 4 18:30:40 localhost asterisk[32229]: NOTICE[32257]: chan_sip.c:23417 in han # failJSON: { "time": "2005-03-01T15:35:53", "match": true , "host": "192.0.2.2", "desc": "log over remote syslog server" } Mar 1 15:35:53 pbx asterisk[2350]: WARNING[1195][C-00000b43]: Ext. s:6 in @ from-sip-external: "Rejecting unknown SIP connection from 192.0.2.2" + +# filterOptions: [{"logtype": "journal", "test.prefix-line": "server asterisk[123]: "}] + +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" } +NOTICE[566]: chan_sip.c:28926 handle_request_register: Registration from '"28" <sip:28@127.0.0.100>' failed for '192.0.2.1:7998' - Wrong password +# failJSON: { "match": true , "host": "192.0.2.2", "desc": "systemd-journal entry (with additional timestamp in message)" } +[Mar 27 10:06:14] NOTICE[566]: chan_sip.c:28926 handle_request_register: Registration from '"1000" <sip:1000@127.0.0.100>' failed for '192.0.2.2:7998' - Wrong password diff --git a/fail2ban/tests/files/logs/bitwarden b/fail2ban/tests/files/logs/bitwarden new file mode 100644 index 00000000..0fede6c6 --- /dev/null +++ b/fail2ban/tests/files/logs/bitwarden @@ -0,0 +1,11 @@ +# failJSON: { "time": "2019-11-25T18:04:49", "match": true , "host": "192.168.0.16" } +2019-11-26 01:04:49.008 +08:00 [WRN] Failed login attempt. 192.168.0.16 + +# failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } +2019-11-25 21:39:58.464 +01:00 [WRN] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-11-25T21:39:58", "match": true , "host": "192.168.0.21" } +2019-11-25 21:39:58.464 +01:00 [Warning] Failed login attempt, 2FA invalid. 192.168.0.21 + +# failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.168.0.23" } +2019-09-24T13:16:50 e5a81dbf7fd1 Bitwarden-Identity[1]: [Bit.Core.IdentityServer.ResourceOwnerPasswordValidator] Failed login attempt. 192.168.0.23 diff --git a/fail2ban/tests/files/logs/centreon b/fail2ban/tests/files/logs/centreon new file mode 100644 index 00000000..fc6fe4fe --- /dev/null +++ b/fail2ban/tests/files/logs/centreon @@ -0,0 +1,4 @@ +# Access of unauthorized host in /var/log/centreon/login.log +# failJSON: { "time": "2019-10-21T18:55:15", "match": true , "host": "50.97.225.132" } +2019-10-21 18:55:15|-1|0|0|[WEB] [50.97.225.132] Authentication failed for 'admin' : password mismatch + diff --git a/fail2ban/tests/files/logs/courier-auth b/fail2ban/tests/files/logs/courier-auth index 3505e109..8a20a27f 100644 --- a/fail2ban/tests/files/logs/courier-auth +++ b/fail2ban/tests/files/logs/courier-auth @@ -8,3 +8,5 @@ Nov 13 08:11:53 server imapd-ssl: LOGIN FAILED, user=user@domain.tld, ip=[::ffff Apr 17 19:17:11 SERVER courierpop3login: LOGIN FAILED, user=USER@EXAMPLE.org, ip=[::ffff:1.2.3.4] # failJSON: { "time": "2005-04-17T19:17:12", "match": true , "host": "192.0.2.4" } Apr 17 19:17:12 server imapd-ssl: LOGIN FAILED, method=PLAIN, ip=[::ffff:192.0.2.4] +# failJSON: { "time": "2005-04-27T09:00:00", "match": true , "user": "tester", "host": "192.0.2.5" } +Apr 27 09:00:00 servername imapd: LOGIN FAILED, user=tester, ip=[::ffff:192.0.2.5], port=[255] diff --git a/fail2ban/tests/files/logs/courier-smtp b/fail2ban/tests/files/logs/courier-smtp index ab99d322..6da0d0a4 100644 --- a/fail2ban/tests/files/logs/courier-smtp +++ b/fail2ban/tests/files/logs/courier-smtp @@ -8,7 +8,9 @@ Jul 4 18:39:39 mail courieresmtpd: error,relay=::ffff:1.2.3.4,from=<picaro@astr Jul 6 03:42:28 whistler courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<admin at memcpy>: 550 User unknown. # failJSON: { "time": "2004-11-21T23:16:17", "match": true , "host": "1.2.3.4" } Nov 21 23:16:17 server courieresmtpd: error,relay=::ffff:1.2.3.4,from=<>,to=<>: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 HOSTNAME courieresmtpd: error,relay=::ffff:1.2.3.4,from=<firozquarl@aclunc.org>,to=<BOGUSUSER@HOSTEDDOMAIN.org>: 550 User unknown. -# failJSON: { "time": "2004-08-14T12:51:04", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2005-08-14T12:51:04", "match": true , "host": "1.2.3.4" } Aug 14 12:51:04 mail.server courieresmtpd[26762]: error,relay=::ffff:1.2.3.4,msg="535 Authentication failed.",cmd: AUTH PLAIN AAAAABBBBCCCCWxlZA== admin +# failJSON: { "time": "2005-08-14T12:51:05", "match": true , "host": "192.0.2.3" } +Aug 14 12:51:05 mail.server courieresmtpd[425070]: error,relay=::ffff:192.0.2.3,port=43632,msg="535 Authentication failed.",cmd: AUTH LOGIN PlcmSpIp@example.com diff --git a/fail2ban/tests/files/logs/dante b/fail2ban/tests/files/logs/dante new file mode 100644 index 00000000..80d6744f --- /dev/null +++ b/fail2ban/tests/files/logs/dante @@ -0,0 +1,6 @@ +# failJSON: { "time": "2005-04-14T15:35:03", "match": true , "host": "1.2.3.4" } +Apr 14 15:35:03 vps111111 danted[17969]: info: block(1): tcp/accept ]: 1.2.3.4.50550 0.0.0.0.1080: error after reading 35 bytes in 0 seconds: could not access user "roooooooot"'s records in the system password file: no system error +# failJSON: { "time": "2005-04-14T15:44:26", "match": true , "host": "1.2.3.4" } +Apr 14 15:44:26 vps111111 danted[1846]: info: block(1): tcp/accept ]: 1.2.3.4.57178 0.0.0.0.1080: error after reading 18 bytes in 0 seconds: system password authentication failed for user "aland" +# failJSON: { "time": "2005-04-14T15:44:26", "match": true , "host": "1.2.3.4" } +Apr 14 15:44:26 vps111111 danted[1846]: info: block(1): tcp/accept ]: 1.2.3.4.57178 0.0.0.0.1080: error after reading 1 byte in 1 second: system password authentication failed for user "aland" diff --git a/fail2ban/tests/files/logs/domino-smtp b/fail2ban/tests/files/logs/domino-smtp index 4987e7ea..957d593f 100644 --- a/fail2ban/tests/files/logs/domino-smtp +++ b/fail2ban/tests/files/logs/domino-smtp @@ -6,3 +6,8 @@ 08-09-2014 06:14:27 smtp: postmaster [1.2.3.4] authentication failure using internet password # failJSON: { "time": "2016-11-07T22:21:20", "match": true , "host": "1.2.3.4" } 2016-11-07 22:21:20 smtp: postmaster [1.2.3.4] authentication failure using internet password + +# failJSON: { "time": "2018-09-19T17:25:50", "match": true , "host": "192.0.2.1", "desc":"different log-format" } +2018-09-19 17:25:50 SMTP Server [0D14:0027-1334] Authentication failed for user Bad Hacker ; connecting host [192.0.2.1] +# failJSON: { "time": "2018-09-19T17:25:52", "match": true , "host": "192.0.2.2", "desc":"gh-2228, rejected for policy reasons" } +2018-09-19 17:25:52 SMTP Server [000527:000013-0000001227564800] Connection from [192.0.2.2] rejected for policy reasons. IP address of connecting host not found in reverse DNS lookup. diff --git a/fail2ban/tests/files/logs/dovecot b/fail2ban/tests/files/logs/dovecot index dfa6199e..0e332961 100644 --- a/fail2ban/tests/files/logs/dovecot +++ b/fail2ban/tests/files/logs/dovecot @@ -34,6 +34,9 @@ Jul 02 13:49:32 hostname dovecot[442]: dovecot: auth(default): pam(account@MYSER # failJSON: { "time": "2005-01-29T05:32:50", "match": true , "host": "1.2.3.4" } Jan 29 05:32:50 mail dovecot: auth-worker(304): pam(username,1.2.3.4): pam_authenticate() failed: Authentication failure (password mismatch?) +# failJSON: { "time": "2005-01-29T18:55:55", "match": true , "host": "192.0.2.4", "desc": "Password mismatch (title case, gh-2880)" } +Jan 29 18:55:55 mail dovecot: auth-worker(12182): pam(user,192.0.2.4): pam_authenticate() failed: Authentication failure (Password mismatch?) + # failJSON: { "time": "2005-01-29T05:13:40", "match": true , "host": "1.2.3.4" } Jan 29 05:13:40 mail dovecot: auth-worker(31326): pam(username,1.2.3.4): unknown user @@ -43,9 +46,25 @@ Jan 29 05:13:50 mail dovecot: auth: passwd-file(username,1.2.3.4): unknown user # failJSON: { "time": "2005-01-29T13:54:06", "match": true , "host": "192.0.2.5" } Jan 29 13:54:06 auth-worker(22401): Info: sql(admin@example.de,192.0.2.5,<n4JLdHNVngZGpV2j>): unknown user +#failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573" } +Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,<LZmGp6mZaMrAqLIZ>): unknown user (SHA1 of given password: f638ff) + #failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.144.226" } Jun 11 13:57:17 main dovecot: auth: sql(admin@example.ru,192.168.144.226,<6rXunFtu493AqJDi>): Password mismatch +#failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573" } +Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,<LZmGp6mZaMrAqLIZ>): Password mismatch (for LDAP bind) (SHA1 of given password: f638ff) + +# failJSON: { "time": "2005-06-12T11:48:12", "match": true , "host": "192.0.2.6" } +Jun 12 11:48:12 auth-worker(80180): Info: conn unix:auth-worker (uid=143): auth-worker<13247>: sql(support,192.0.2.6): unknown user +# failJSON: { "time": "2005-06-12T23:06:05", "match": true , "host": "192.0.2.7" } +Jun 12 23:06:05 auth-worker(57065): Info: conn unix:auth-worker (uid=143): auth-worker<225622>: sql(user@domain.com,192.0.2.7,<Yx7+W8+Io>): Password mismatch + +# failJSON: { "time": "2005-06-15T11:28:21", "match": true , "host": "192.0.2.7" } +Jun 15 11:28:21 hostname dovecot: auth-worker(5787): conn unix:auth-worker (pid=27359,uid=97): auth-worker<55>: pam(webapps,192.0.2.7): unknown user +# failJSON: { "time": "2005-06-15T13:57:41", "match": true , "host": "192.0.2.7" } +Jun 15 13:57:41 hostname dovecot: auth-worker(3270): conn unix:auth-worker (pid=27359,uid=97): auth-worker<128>: pam(webapps,192.0.2.7): pam_authenticate() failed: Authentication failure (Password mismatch?) + # failJSON: { "time": "2005-01-29T14:38:51", "match": true , "host": "192.0.2.6", "desc": "PAM Permission denied (gh-1897)" } Jan 29 14:38:51 example.com dovecot[24941]: auth-worker(30165): pam(user@example.com,192.0.2.6,<PNHQq8pZhqIKAQGd>): pam_authenticate() failed: Permission denied @@ -93,6 +112,26 @@ Jul 26 11:12:19 hostname dovecot: imap-login: Disconnected: Too many invalid com # failJSON: { "time": "2004-08-28T06:38:51", "match": true , "host": "192.0.2.3" } Aug 28 06:38:51 s166-62-100-187 dovecot: imap-login: Disconnected (auth failed, 1 attempts in 9 secs): user=<administrator@example.com>, method=PLAIN, rip=192.0.2.3, lip=192.168.1.2, TLS: Disconnected, TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits) +# failJSON: { "time": "2004-08-28T06:38:52", "match": true , "host": "192.0.2.4", "desc": "open parenthesis in optional part between Disconnected and (auth failed ...), gh-3210" } +Aug 28 06:38:52 s166-62-100-187 dovecot: imap-login: Disconnected: Connection closed: read(size=1003) failed: Connection reset by peer (auth failed, 1 attempts in 0 secs): user=<test@example.com>, rip=192.0.2.4, lip=127.0.0.19, session=<Lsz0Oo7WXti3b7xe> + +# failJSON: { "time": "2004-08-29T01:49:33", "match": false , "desc": "avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: read(size=1026) failed: Connection reset by peer (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: read(size=1026) failed: Connection reset by peer +# failJSON: { "time": "2004-08-29T01:49:33", "match": false , "desc": "avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number +# failJSON: { "time": "2004-08-29T01:49:33", "match": false , "desc": "avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: managesieve-login: Disconnected: Too many invalid commands. (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1 +# failJSON: { "time": "2004-08-29T01:49:33", "match": false , "desc": "avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: managesieve-login: Disconnected: Connection closed: read(size=1007) failed: Connection reset by peer (no auth attempts in 1 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1 +# failJSON: { "time": "2004-08-29T01:49:33", "match": false , "desc": "avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[472]: imap-login: Disconnected: Connection closed: SSL_accept() failed: error:14209102:SSL routines:tls_early_post_process_client_hello:unsupported protocol (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: SSL_accept() failed: error:14209102:SSL routines:tls_early_post_process_client_hello:unsupported protocol + +# failJSON: { "time": "2004-08-29T03:17:18", "match": true , "host": "192.0.2.133" } +Aug 29 03:17:18 server dovecot: submission-login: Client has quit the connection (auth failed, 1 attempts in 2 secs): user=<user1>, method=LOGIN, rip=192.0.2.133, lip=0.0.0.0 +# failJSON: { "time": "2004-08-29T03:53:52", "match": true , "host": "192.0.2.169" } +Aug 29 03:53:52 server dovecot: submission-login: Remote closed connection (auth failed, 1 attempts in 2 secs): user=<user4>, method=PLAIN, rip=192.0.2.169, lip=0.0.0.0 +# failJSON: { "time": "2004-08-29T15:33:53", "match": true , "host": "192.0.2.100" } +Aug 29 15:33:53 server dovecot: managesieve-login: Disconnected: Too many invalid commands. (auth failed, 1 attempts in 2 secs): user=<myself>, method=PLAIN, rip=192.0.2.100, lip=0.0.0.0, TLS, TLSv1.3 with cipher TLS_CHACHA20_POLY1305_SHA256 (256/256 bits) # --------------------------------------- # Test-cases of aggressive mode: @@ -100,6 +139,17 @@ Aug 28 06:38:51 s166-62-100-187 dovecot: imap-login: Disconnected (auth failed, # filterOptions: [{"mode": "aggressive"}] +# failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: read(size=1026) failed: Connection reset by peer (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: read(size=1026) failed: Connection reset by peer +# failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: imap-login: Disconnected: Connection closed: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: SSL_accept() failed: error:1408F10B:SSL routines:ssl3_get_record:wrong version number +# failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: managesieve-login: Disconnected: Too many invalid commands. (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1 +# failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[459]: managesieve-login: Disconnected: Connection closed: read(size=1007) failed: Connection reset by peer (no auth attempts in 1 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1 +# failJSON: { "time": "2004-08-29T01:49:33", "match": true , "host": "192.0.2.5", "desc": "matches in aggressive mode, avoid slow RE, gh-3370" } +Aug 29 01:49:33 server dovecot[472]: imap-login: Disconnected: Connection closed: SSL_accept() failed: error:14209102:SSL routines:tls_early_post_process_client_hello:unsupported protocol (no auth attempts in 0 secs): user=<>, rip=192.0.2.5, lip=127.0.0.1, TLS handshaking: SSL_accept() failed: error:14209102:SSL routines:tls_early_post_process_client_hello:unsupported protocol + # failJSON: { "time": "2004-08-29T16:06:58", "match": true , "host": "192.0.2.5" } Aug 29 16:06:58 s166-62-100-187 dovecot: imap-login: Disconnected (disconnected before auth was ready, waited 0 secs): user=<>, rip=192.0.2.5, lip=192.168.1.2, TLS handshaking: SSL_accept() syscall failed: Connection reset by peer # failJSON: { "time": "2004-08-31T16:15:10", "match": true , "host": "192.0.2.6" } diff --git a/fail2ban/tests/files/logs/drupal-auth b/fail2ban/tests/files/logs/drupal-auth index 5e7194d9..4d063e55 100644 --- a/fail2ban/tests/files/logs/drupal-auth +++ b/fail2ban/tests/files/logs/drupal-auth @@ -3,5 +3,15 @@ Apr 26 13:15:25 webserver example.com: https://example.com|1430068525|user|1.2.3 # failJSON: { "time": "2005-04-26T13:15:25", "match": true , "host": "1.2.3.4" } Apr 26 13:15:25 webserver example.com: https://example.com/subdir|1430068525|user|1.2.3.4|https://example.com/subdir/user|https://example.com/subdir/user|0||Login attempt failed for drupaladmin. -# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4" } +# failJSON: { "time": "2005-04-26T13:19:08", "match": false , "host": "1.2.3.4", "user": "drupaladmin" } Apr 26 13:19:08 webserver example.com: https://example.com|1430068748|user|1.2.3.4|https://example.com/user|https://example.com/user|1||Session opened for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:00", "match": false, "desc": "attempt to inject on URI (pipe, login failed for), not a failure, gh-2742" } +Apr 26 13:20:00 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed for tester|2||Session revisited for drupaladmin. + +# failJSON: { "time": "2005-04-26T13:20:01", "match": true , "host": "192.0.2.7", "user": "Jack Sparrow", "desc": "log-format change - for -> from, user name with space, gh-2742" } +Apr 26 13:20:01 mweb drupal_site[24864]: https://www.example.com|1613058599|user|192.0.2.7|https://www.example.com/en/user/login|https://www.example.com/en/user/login|0||Login attempt failed from Jack Sparrow. +# failJSON: { "time": "2005-04-26T13:20:02", "match": true , "host": "192.0.2.4", "desc": "attempt to inject on URI (pipe), login failed, gh-2742" } +Apr 26 13:20:02 host drupal-site: https://example.com|1613063581|user|192.0.2.4|https://example.com/user/login?test=%7C&test2=%7C|https://example.com/user/login?test=|&test2=||0||Login attempt failed from 192.0.2.4. +# failJSON: { "time": "2005-04-26T13:20:03", "match": false, "desc": "attempt to inject on URI (pipe, login failed from), not a failure, gh-2742" } +Apr 26 13:20:03 host drupal-site: https://example.com|1613063581|user|192.0.2.5|https://example.com/user/login?test=%7C&test2=%7C...|https://example.com/user/login?test=|&test2=|0||Login attempt failed from 1.2.3.4|2||Session revisited for drupaladmin. diff --git a/fail2ban/tests/files/logs/exim b/fail2ban/tests/files/logs/exim index 79437a90..e88f06ef 100644 --- a/fail2ban/tests/files/logs/exim +++ b/fail2ban/tests/files/logs/exim @@ -43,6 +43,9 @@ # failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" } 2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner) +# failJSON: { "time": "2019-10-22T03:39:17", "match": true , "host": "192.0.2.37", "desc": "pid-prefix in form of 'mx1 exim[...]:', gh-2553" } +2019-10-22 03:39:17 mx1 exim[29786]: dovecot_login authenticator failed for (User) [192.0.2.37]: 535 Incorrect authentication data (set_id=test@domain.com) + # failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" } 2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster) diff --git a/fail2ban/tests/files/logs/gitlab b/fail2ban/tests/files/logs/gitlab new file mode 100644 index 00000000..70ddc0e8 --- /dev/null +++ b/fail2ban/tests/files/logs/gitlab @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/gitlab/gitlab-rails/application.log +# failJSON: { "time": "2020-04-09T16:04:00", "match": true , "host": "80.10.11.12" } +2020-04-09T14:04:00.667Z: Failed Login: username=admin ip=80.10.11.12 +# failJSON: { "time": "2020-04-09T16:15:09", "match": true , "host": "80.10.11.12" } +2020-04-09T14:15:09.344Z: Failed Login: username=user name ip=80.10.11.12 diff --git a/fail2ban/tests/files/logs/grafana b/fail2ban/tests/files/logs/grafana new file mode 100644 index 00000000..aac86ebc --- /dev/null +++ b/fail2ban/tests/files/logs/grafana @@ -0,0 +1,5 @@ +# Access of unauthorized host in /var/log/grafana/grafana.log +# failJSON: { "time": "2020-10-19T17:44:33", "match": true , "host": "182.56.23.12" } +t=2020-10-19T17:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="Invalid Username or Password" remote_addr=182.56.23.12 +# failJSON: { "time": "2020-10-19T18:44:33", "match": true , "host": "182.56.23.13" } +t=2020-10-19T18:44:33+0200 lvl=eror msg="Invalid username or password" logger=context userId=0 orgId=0 uname= error="User not found" remote_addr=182.56.23.13 diff --git a/fail2ban/tests/files/logs/guacamole b/fail2ban/tests/files/logs/guacamole index 3de67454..ebb7afb0 100644 --- a/fail2ban/tests/files/logs/guacamole +++ b/fail2ban/tests/files/logs/guacamole @@ -10,3 +10,8 @@ WARNING: Authentication attempt from 192.0.2.0 for user "null" failed. apr 16, 2013 8:32:28 AM org.slf4j.impl.JCLLoggerAdapter warn # failJSON: { "time": "2013-04-16T08:32:28", "match": true , "host": "192.0.2.0" } WARNING: Authentication attempt from 192.0.2.0 for user "pippo" failed. + +# filterOptions: {"logging": "webapp"} + +# failJSON: { "time": "2005-08-13T12:57:32", "match": true , "host": "182.23.72.36" } +12:57:32.907 [http-nio-8080-exec-10] WARN o.a.g.r.auth.AuthenticationService - Authentication attempt from 182.23.72.36 for user "guacadmin" failed. diff --git a/fail2ban/tests/files/logs/lighttpd-auth b/fail2ban/tests/files/logs/lighttpd-auth index 184dba33..c8a922b5 100644 --- a/fail2ban/tests/files/logs/lighttpd-auth +++ b/fail2ban/tests/files/logs/lighttpd-auth @@ -1,4 +1,3 @@ -#authentification failure (mod_auth) # failJSON: { "time": "2011-12-25T17:09:20", "match": true , "host": "4.4.4.4" } 2011-12-25 17:09:20: (http_auth.c.875) password doesn't match for /gitweb/ username: francois, IP: 4.4.4.4 # failJSON: { "time": "2012-09-26T10:24:35", "match": true , "host": "4.4.4.4" } @@ -7,3 +6,9 @@ 2013-08-25 00:24:55: (http_auth.c.877) get_password failed, IP: 4.4.4.4 # failJSON: { "time": "2018-01-16T14:10:32", "match": true , "host": "192.0.2.1", "desc": "http_auth -> mod_auth, gh-2018" } 2018-01-16 14:10:32: (mod_auth.c.525) password doesn't match for /test-url username: test, IP: 192.0.2.1 +# failJSON: { "time": "2021-09-30T16:05:33", "match": true , "host": "192.0.2.2", "user":"test", "desc": "gh-3116" } +2021-09-30 16:05:33: mod_auth.c.828) password doesn't match for /secure/ username: test IP: 192.0.2.2 +# failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.3", "user":"tester", "desc": "gh-3116" } +2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed for tester : wrong password, IP: 192.0.2.3 +# failJSON: { "time": "2021-09-30T17:44:37", "match": true , "host": "192.0.2.4", "desc": "gh-3116" } +2021-09-30 17:44:37: (mod_auth.c.791) digest: auth failed: uri mismatch (/uri1 != /uri2), IP: 192.0.2.4 diff --git a/fail2ban/tests/files/logs/monit b/fail2ban/tests/files/logs/monit index 57437046..36f1c1e4 100644 --- a/fail2ban/tests/files/logs/monit +++ b/fail2ban/tests/files/logs/monit @@ -1,7 +1,7 @@ # Previous version -- -# failJSON: { "time": "2005-04-16T21:05:29", "match": true , "host": "69.93.127.111" } +# failJSON: { "time": "2005-04-17T06:05:29", "match": true , "host": "69.93.127.111" } [PDT Apr 16 21:05:29] error : Warning: Client '69.93.127.111' supplied unknown user 'foo' accessing monit httpd -# failJSON: { "time": "2005-04-16T20:59:33", "match": true , "host": "97.113.189.111" } +# failJSON: { "time": "2005-04-17T05:59:33", "match": true , "host": "97.113.189.111" } [PDT Apr 16 20:59:33] error : Warning: Client '97.113.189.111' supplied wrong password for user 'admin' accessing monit httpd # Current version -- corresponding "https://bitbucket.org/tildeslash/monit/src/6905335aa903d425cae732cab766bd88ea5f2d1d/src/http/processor.c?at=master&fileviewer=file-view-default#processor.c-728" @@ -19,3 +19,6 @@ Mar 9 09:18:32 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3 Mar 9 09:18:33 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3.4: unknown user 'test1' # failJSON: { "time": "2005-03-09T09:18:34", "match": true, "host": "1.2.3.4", "desc": "wrong password try" } Mar 9 09:18:34 hostname monit[5731]: HttpRequest: access denied -- client 1.2.3.4: wrong password for user 'test2' + +# failJSON: { "time": "2005-08-06T10:14:52", "match": true, "host": "192.168.1.85", "desc": "IP in brackets, gh-2494" } +[CEST Aug 6 10:14:52] error : HttpRequest: access denied -- client [192.168.1.85]: wrong password for user 'root' diff --git a/fail2ban/tests/files/logs/monitorix b/fail2ban/tests/files/logs/monitorix new file mode 100644 index 00000000..e6ad6dc6 --- /dev/null +++ b/fail2ban/tests/files/logs/monitorix @@ -0,0 +1,8 @@ +# failJSON: { "time": "2021-04-14T08:11:01", "match": false, "desc": "should be ignored: successful request" } +Wed Apr 14 08:11:01 2021 - OK - [127.0.0.1] "GET /monitorix-cgi/monitorix.cgi - Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" +# failJSON: { "time": "2021-04-14T08:54:22", "match": true, "host": "127.0.0.1", "desc": "file does not exist" } +Wed Apr 14 08:54:22 2021 - NOTEXIST - [127.0.0.1] File does not exist: /manager/html +# failJSON: { "time": "2021-04-14T11:24:31", "match": true, "host": "127.0.0.1", "desc": "access not allowed" } +Wed Apr 14 11:24:31 2021 - NOTALLOWED - [127.0.0.1] Access not allowed: /monitorix/ +# failJSON: { "time": "2021-04-14T11:26:08", "match": true, "host": "127.0.0.1", "desc": "authentication error" } +Wed Apr 14 11:26:08 2021 - AUTHERR - [127.0.0.1] Authentication error: /monitorix/ diff --git a/fail2ban/tests/files/logs/mssql-auth b/fail2ban/tests/files/logs/mssql-auth new file mode 100644 index 00000000..1c9b65ec --- /dev/null +++ b/fail2ban/tests/files/logs/mssql-auth @@ -0,0 +1,11 @@ +# failJSON: { "time": "2020-02-24T16:05:21", "match": true , "host": "192.0.2.1" } +2020-02-24 16:05:21.00 Logon Login failed for user 'Backend'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.1] +# failJSON: { "time": "2020-02-24T16:30:25", "match": true , "host": "192.0.2.2" } +2020-02-24 16:30:25.88 Logon Login failed for user '===)jf02hüas9ä##22f'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.2] +# failJSON: { "time": "2020-02-24T16:31:12", "match": true , "host": "192.0.2.3" } +2020-02-24 16:31:12.20 Logon Login failed for user ''. Reason: An attempt to login using SQL authentication failed. Server is configured for Integrated authentication only. [CLIENT: 192.0.2.3] + +# failJSON: { "time": "2020-02-24T16:31:26", "match": true , "host": "192.0.2.4", "user":"O'Leary" } +2020-02-24 16:31:26.01 Logon Login failed for user 'O'Leary'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.4] +# failJSON: { "time": "2020-02-24T16:31:26", "match": false, "desc": "test injection in possibly unescaped foreign input" } +2020-02-24 16:31:26.02 Wrong data received: Logon Login failed for user 'test'. Reason: Could not find a login matching the name provided. [CLIENT: 192.0.2.5] diff --git a/fail2ban/tests/files/logs/murmur b/fail2ban/tests/files/logs/murmur index bc18b7ea..ee15a9c7 100644 --- a/fail2ban/tests/files/logs/murmur +++ b/fail2ban/tests/files/logs/murmur @@ -3,3 +3,8 @@ # failJSON: { "time": "2015-11-29T17:18:20", "match": true , "host": "192.168.1.2" } <W>2015-11-29 17:18:20.962 1 => <8:testUsernameTwo(-1)> Rejected connection from 192.168.1.2:29761: Wrong certificate or password for existing user + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" } +Test murmurd[2064]: <W>2019-09-08 13:00:05.615 1 => <10:Test(-1)> Rejected connection from 192.0.2.1:31752: Invalid server password diff --git a/fail2ban/tests/files/logs/mysqld-auth b/fail2ban/tests/files/logs/mysqld-auth index ebb8c0c4..29faeb71 100644 --- a/fail2ban/tests/files/logs/mysqld-auth +++ b/fail2ban/tests/files/logs/mysqld-auth @@ -25,3 +25,15 @@ Sep 16 21:30:32 catinthehat mysqld: 130916 21:30:32 [Warning] Access denied for # failJSON: { "time": "2016-02-24T15:26:18", "match": false , "host": "localhost", "desc": "A hypothetical example of injection having full log line first (for paranoid yoh)" } 2016-02-24T15:26:18.237955 6 [Note] Access denied for user 'root'@'localhost' (using password: YES) condition lead to a hypothetical failure + +# failJSON: { "time": "2019-01-03T09:50:04", "match": true , "host": "192.0.2.1", "desc": "mysql 8.0.13 logging with details, (log-error-verbosity = 3, gh-2314)" } +2019-01-03T08:50:04.634875Z 113 [Note] [MY-010926] [Server] Access denied for user 'root'@'192.0.2.1' (using password: NO) + +# failJSON: { "time": "2019-09-06T01:45:18", "match": true , "host": "192.0.2.2", "desc": "ISO timestamp within log message" } +2019-09-06T01:45:18 srv mysqld: 2019-09-06 1:45:18 140581192722176 [Warning] Access denied for user 'global'@'192.0.2.2' (using password: YES) +# failJSON: { "time": "2019-09-24T13:16:50", "match": true , "host": "192.0.2.3", "desc": "ISO timestamp within log message" } +2019-09-24T13:16:50 srv mysqld[1234]: 2019-09-24 13:16:50 8756 [Warning] Access denied for user 'root'@'192.0.2.3' (using password: YES) + +# filterOptions: [{"logtype": "file"}, {"logtype": "short"}, {"logtype": "journal"}] +# failJSON: { "match": true , "host": "192.0.2.1", "user":"root", "desc": "mariadb 10.4 log format, gh-2611" } +2020-01-16 21:34:14 4644 [Warning] Access denied for user 'root'@'192.0.2.1' (using password: YES) diff --git a/fail2ban/tests/files/logs/named-refused b/fail2ban/tests/files/logs/named-refused index 7414a1b3..5ef42074 100644 --- a/fail2ban/tests/files/logs/named-refused +++ b/fail2ban/tests/files/logs/named-refused @@ -23,3 +23,16 @@ Aug 17 08:20:22 catinthehat named[2954]: client 223.252.23.219#56275: zone trans 27-Aug-2013 17:49:45.330 client 59.167.242.100#44281 (watt.kiev.ua): zone transfer 'watt.kiev.ua/AXFR/IN' denied # failJSON: { "time": "2004-08-27T16:58:31", "match": true , "host": "176.9.92.38" } Aug 27 16:58:31 vhost1-ua named[29206]: client 176.9.92.38#42592 (simmarket.com.ua): bad zone transfer request: 'simmarket.com.ua/IN': non-authoritative zone (NOTAUTH) + +# failJSON: { "time": "2004-08-27T16:59:00", "match": true , "host": "192.0.2.1", "desc": "new log format, 9.11.0 (#2406)" } +Aug 27 16:59:00 host named[28098]: client @0x7f6450002ef0 192.0.2.1#23332 (example.com): bad zone transfer request: 'test.com/IN': non-authoritative zone (NOTAUTH) + +# failJSON: { "match": true , "host": "192.0.2.8", "desc": "log message with category (security), gh-3388" } +Oct 23 02:06:39 security: info: client @0x7f4e446fd6e8 192.0.2.8#53 (example.io): query (cache) 'example.io/A/IN' denied +# failJSON: { "match": true , "host": "192.0.2.237", "desc": "log message with category, gh-3388" } +Oct 23 03:35:40 update-security: error: client @0x7f4e45c07a48 192.0.2.237#55956 (example.ca): zone transfer 'example.ca/AXFR/IN' denied + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" } +atom named[1806]: client @0x7fb13400eec0 192.0.2.1#61977 (.): query (cache) './ANY/IN' denied diff --git a/fail2ban/tests/files/logs/nginx-bad-request b/fail2ban/tests/files/logs/nginx-bad-request new file mode 100644 index 00000000..a9ff6497 --- /dev/null +++ b/fail2ban/tests/files/logs/nginx-bad-request @@ -0,0 +1,23 @@ +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - root [20/Jan/2015:19:53:28 +0100] "" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "\x03\x00\x00/*\xE0\x00\x00\x00\x00\x00Cookie: mstshash=Administr" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:53:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:53:28 +0100] "GET //admin/pma/scripts/setup.php HTTP/1.1" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:54:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:54:28 +0100] "HELP" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T19:55:28", "match": true , "host": "12.34.56.78" } +12.34.56.78 - - [20/Jan/2015:19:55:28 +0100] "batman" 400 47 "-" "-" "-" + +# failJSON: { "time": "2015-01-20T01:17:07", "match": true , "host": "7.8.9.10" } +7.8.9.10 - root [20/Jan/2015:01:17:07 +0100] "CONNECT 123.123.123.123 HTTP/1.1" 400 162 "-" "-" "-" + +# failJSON: { "time": "2014-12-12T22:59:02", "match": true , "host": "2.5.2.5" } +2.5.2.5 - tomcat [12/Dec/2014:22:59:02 +0100] "GET /cgi-bin/tools/tools.pl HTTP/1.1" 400 162 "-" "-" "-"
\ No newline at end of file diff --git a/fail2ban/tests/files/logs/nginx-http-auth b/fail2ban/tests/files/logs/nginx-http-auth index c9c96807..fb24b242 100644 --- a/fail2ban/tests/files/logs/nginx-http-auth +++ b/fail2ban/tests/files/logs/nginx-http-auth @@ -1,3 +1,4 @@ +# filterOptions: [{"mode": "normal"}, {"mode": "auth"}] # failJSON: { "time": "2012-04-09T11:53:29", "match": true , "host": "192.0.43.10" } 2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" @@ -11,3 +12,20 @@ 2014/04/03 22:20:38 [error] 30708#0: *3 user "scriben dio": password mismatch, client: 192.0.2.1, server: , request: "GET / HTTP/1.1", host: "localhost:8443" # failJSON: { "time": "2014-04-03T22:20:40", "match": true, "host": "192.0.2.2", "desc": "trying injection on user name"} 2014/04/03 22:20:40 [error] 30708#0: *3 user "test": password mismatch, client: 127.0.0.1, server: test, request: "GET / HTTP/1.1", host: "localhost:8443"": was not found in "/etc/nginx/.htpasswd", client: 192.0.2.2, server: , request: "GET / HTTP/1.1", host: "localhost:8443" + +# filterOptions: [{"mode": "fallback"}] + +# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } +2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T15:47:47", "match": true , "host": "80.191.166.166" } +2020/11/25 15:47:47 [crit] 76952#76952: *5062354 SSL_do_handshake() failed (SSL: error:1408F0A0:SSL routines:ssl3_get_record:length too short) while SSL handshaking, client: 80.191.166.166, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:48:08", "match": true , "host": "5.126.32.148" } +2020/11/25 16:48:08 [crit] 76952#76952: *7976400 SSL_do_handshake() failed (SSL: error:1408F096:SSL routines:ssl3_get_record:encrypted length too long) while SSL handshaking, client: 5.126.32.148, server: 0.0.0.0:443 +# failJSON: { "time": "2020-11-25T16:02:45", "match": false } +2020/11/25 16:02:45 [error] 76952#76952: *5645766 connect() failed (111: Connection refused) while connecting to upstream, client: 5.126.32.148, server: www.google.de, request: "GET /admin/config HTTP/2.0", upstream: "http://127.0.0.1:3000/admin/config", host: "www.google.de" + +# filterOptions: [{"mode": "aggressive"}] +# failJSON: { "time": "2020-11-25T14:42:16", "match": true , "host": "142.93.180.14" } +2020/11/25 14:42:16 [crit] 76952#76952: *2454307 SSL_do_handshake() failed (SSL: error:1408F0C6:SSL routines:ssl3_get_record:packet length too long) while SSL handshaking, client: 142.93.180.14, server: 0.0.0.0:443 +# failJSON: { "time": "2012-04-09T11:53:29", "match": true , "host": "192.0.43.10" } +2012/04/09 11:53:29 [error] 2865#0: *66647 user "xyz" was not found in "/var/www/.htpasswd", client: 192.0.43.10, server: www.myhost.com, request: "GET / HTTP/1.1", host: "www.myhost.com" diff --git a/fail2ban/tests/files/logs/nsd b/fail2ban/tests/files/logs/nsd index a33a52a9..63c162e9 100644 --- a/fail2ban/tests/files/logs/nsd +++ b/fail2ban/tests/files/logs/nsd @@ -2,3 +2,5 @@ [1387288694] nsd[7745]: info: ratelimit block example.com. type any target 192.0.2.0/24 query 192.0.2.105 TYPE255 # failJSON: { "time": "2013-12-18T07:42:15", "match": true , "host": "192.0.2.115" } [1387348935] nsd[23600]: info: axfr for zone domain.nl. from client 192.0.2.115 refused, no acl matches. +# failJSON: { "time": "2021-03-05T05:25:14", "match": true , "host": "192.0.2.32", "desc": "new format, no client after from, no dot at end, gh-2965" } +[2021-03-05 05:25:14.562] nsd[160800]: info: axfr for example.com. from 192.0.2.32 refused, no acl matches diff --git a/fail2ban/tests/files/logs/postfix b/fail2ban/tests/files/logs/postfix index b56619a4..d1e534e3 100644 --- a/fail2ban/tests/files/logs/postfix +++ b/fail2ban/tests/files/logs/postfix @@ -12,7 +12,11 @@ Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.1 Jul 18 23:12:56 xxx postfix/smtpd[8738]: NOQUEUE: reject: RCPT from foo[192.51.100.43]: 554 5.7.1 <foo@bad.domain>: Sender address rejected: match bad.domain; from=<foo@bad.domain> to=<foo@porcupine.org> proto=SMTP helo=<192.51.100.43> # failJSON: { "time": "2005-08-10T10:55:38", "match": true , "host": "72.53.132.234" } Aug 10 10:55:38 f-vanier-bourgeois postfix/smtpd[2162]: NOQUEUE: reject: VRFY from 72-53-132-234.cpe.distributel.net[72.53.132.234]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient tab +# failJSON: { "time": "2005-08-13T15:45:46", "match": true , "host": "192.0.2.1" } +Aug 13 15:45:46 server postfix/smtpd[13844]: 00ADB3C0899: reject: RCPT from example.com[192.0.2.1]: 550 5.1.1 <sales@server.com>: Recipient address rejected: User unknown in local recipient table; from=<xxxxxx@example.com> to=<sales@server.com> proto=ESMTP helo=<mail.example.com> +# failJSON: { "time": "2005-05-19T00:00:30", "match": true , "host": "192.0.2.2", "desc": "undeliverable address (sender/recipient verification, gh-3039)" } +May 19 00:00:30 proxy2 postfix/smtpd[16123]: NOQUEUE: reject: RCPT from example.net[192.0.2.2]: 550 5.1.1 <user1@example.com>: Recipient address rejected: undeliverable address: verification failed; from=<user2@example.org> to=<user1@example.com> proto=ESMTP helo=<example.net> # failJSON: { "time": "2005-01-12T11:07:49", "match": true , "host": "181.21.131.88" } Jan 12 11:07:49 emf1pt2-2-35-70 postfix/smtpd[13767]: improper command pipelining after DATA from unknown[181.21.131.88]: @@ -25,6 +29,8 @@ Dec 18 02:05:46 platypus postfix/smtpd[16349]: improper command pipelining after # failJSON: { "time": "2004-12-21T21:17:29", "match": true , "host": "93.184.216.34" } Dec 21 21:17:29 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.1 Client host rejected: cannot find your hostname, [93.184.216.34]; from=<badactor@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badserver.example.com> +# failJSON: { "time": "2004-12-21T21:17:30", "match": true , "host": "93.184.216.34", "desc": "variable status code suffix, gh-2442" } +Dec 21 21:17:30 xxx postfix/smtpd[7150]: NOQUEUE: reject: RCPT from badserver.example.com[93.184.216.34]: 450 4.7.25 Client host rejected: cannot find your hostname, [93.184.216.34]; from=<badactor@example.com> to=<goodguy@example.com> proto=ESMTP helo=<badserver.example.com> # failJSON: { "time": "2004-11-22T22:33:44", "match": true , "host": "1.2.3.4" } Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.example.com[1.2.3.4]: 450 4.1.8 <some@nonexistant.tld>: Sender address rejected: Domain not found; from=<some@nonexistant.tld> to=<goodguy@example.com> proto=ESMTP helo=<1-2-3-4.example.com> @@ -32,6 +38,16 @@ Nov 22 22:33:44 xxx postfix/smtpd[11111]: NOQUEUE: reject: RCPT from 1-2-3-4.exa # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 <User>: Helo command rejected: need fully-qualified hostname; proto=SMTP helo=<User> +# failJSON: { "time": "2005-03-7T02:09:33", "match": true , "host": "192.0.2.151", "desc": "reject: DATA from, gh-2927" } +Mar 7 02:09:33 server postfix/smtpd[27246]: 1D8CC1CA0A7F: milter-reject: DATA from 66-220-155-151.mail-mail.facebook.com[192.0.2.151]: 550 5.7.1 Command rejected; from=<security@mail.example.com> to=<hostmaster@example.com> proto=ESMTP helo=<192-0-2-151.mail-mail.example.com> +# failJSON: { "time": "2005-03-11T23:27:54", "match": true , "host": "192.0.2.109", "desc": "reject: BDAT from, gh-2927" } +Mar 11 23:27:54 server postfix-smo/submission/smtpd[22427]: 44JCRG5tYPzCqt2: reject: BDAT from signing-milter.example.com[192.0.2.109]: 550 5.5.3 <DATA>: Data command rejected: Multi-recipient bounce; from=<> to=<some@example.com> proto=ESMTP helo=<domain.tld> + +# failJSON: { "time": "2005-04-06T13:05:01", "match": true , "host": "192.0.2.116", "desc": "RCPT from unknown, gh-2995" } +Apr 6 13:05:01 server postfix/smtpd[20589]: NOQUEUE: reject: RCPT from unknown[192.0.2.116]: 504 5.5.2 <WIN-6A0KEE6QVP5>: Helo command rejected: need fully-qualified hostname; from=<spameri@example.com> to=<spameri@example.com> proto=ESMTP helo=<WIN-6A0KEE6QVP5> +# failJSON: { "time": "2005-04-07T03:10:56", "match": true , "host": "192.0.2.246", "desc": "550 5.7.25 Client host rejected, gh-2996" } +Apr 7 03:10:56 server postfix/smtpd[7754]: NOQUEUE: reject: RCPT from unknown[192.0.2.246]: 550 5.7.25 Client host rejected: cannot find your hostname, [192.0.2.246]; from=<laqqubtbyop@example.com> to=<sxhcpltqhpex@example.com> proto=ESMTP helo=<[192.0.2.246]> + # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 <User>: Helo command rejected: need fully-qualified hostname; proto=SMTP helo=<User> @@ -47,13 +63,20 @@ Jun 12 08:58:35 xxx postfix/smtpd[2931]: NOQUEUE: reject: RCPT from unknown[1.2. # failJSON: { "time": "2005-06-12T08:58:35", "match": true , "host": "1.2.3.4" } Jun 12 08:58:35 xxx postfix/smtpd[13533]: improper command pipelining after AUTH from unknown[1.2.3.4]: QUIT -# --------------------------------------- -# Test-cases of postfix-postscreen: -# --------------------------------------- - # failJSON: { "time": "2005-05-05T15:51:11", "match": true , "host": "216.245.194.173", "desc": "postfix postscreen / gh-1764" } May 5 15:51:11 xxx postfix/postscreen[1148]: NOQUEUE: reject: RCPT from [216.245.194.173]:60591: 550 5.7.1 Service unavailable; client [216.245.194.173] blocked using rbl.example.com; from=<spammer@example.com>, to=<goodguy@example.com>, proto=ESMTP, helo=<badguy.example.com> +# failJSON: { "time": "2005-06-03T06:25:43", "match": true , "host": "192.0.2.11", "desc": "too many errors / gh-2439" } +Jun 3 06:25:43 srv postfix/smtpd[29306]: too many errors after RCPT from example.com[192.0.2.11] + +# filterOptions: [{"mode": "errors"}] + +# failJSON: { "match": false, "desc": "ignore normal messages, jail for too many errors only" } +Jun 12 08:58:35 srv postfix/smtpd[29306]: improper command pipelining after AUTH from unknown[192.0.2.11]: QUIT + +# failJSON: { "time": "2005-06-03T06:25:43", "match": true , "host": "192.0.2.11", "desc": "too many errors / gh-2439" } +Jun 3 06:25:43 srv postfix/smtpd[29306]: too many errors after RCPT from example.com[192.0.2.11] + # --------------------------------------- # Test-cases of postfix-rbl: # --------------------------------------- @@ -127,6 +150,11 @@ Jan 14 16:18:16 xxx postfix/smtpd[14933]: warning: host[192.0.2.5]: SASL CRAM-MD # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.1" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.1] helo=1 auth=0/1 quit=1 commands=2/3 +# failJSON: { "time": "2005-02-10T13:26:34", "match": true , "host": "192.0.2.2" } +Feb 10 13:26:34 srv postfix/smtpd[123]: disconnect from unknown[192.0.2.2] ehlo=1 auth=0/1 rset=1 quit=1 commands=3/4 + # failJSON: { "time": "2005-02-18T09:45:10", "match": true , "host": "192.0.2.10" } Feb 18 09:45:10 xxx postfix/smtpd[42]: lost connection after CONNECT from spammer.example.com[192.0.2.10] # failJSON: { "time": "2005-02-18T09:45:12", "match": true , "host": "192.0.2.42" } @@ -136,6 +164,17 @@ Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[1 # failJSON: { "time": "2005-02-18T09:48:04", "match": true , "host": "192.0.2.23" } Feb 18 09:48:04 xxx postfix/smtpd[23]: lost connection after AUTH from unknown[192.0.2.23] +# failJSON: { "time": "2004-12-23T19:39:13", "match": true , "host": "192.0.2.2" } +Dec 23 19:39:13 xxx postfix/postscreen[21057]: PREGREET 14 after 0.08 from [192.0.2.2]:59415: EHLO ylmf-pc\r\n +# failJSON: { "time": "2004-12-24T00:54:36", "match": true , "host": "192.0.2.3" } +Dec 24 00:54:36 xxx postfix/postscreen[22515]: HANGUP after 16 from [192.0.2.3]:48119 in tests after SMTP handshake + +# failJSON: { "time": "2005-06-08T23:14:28", "match": true , "host": "192.0.2.77", "desc": "abusive clients hitting command limit, see see http://www.postfix.org/POSTSCREEN_README.html (gh-3040)" } +Jun 8 23:14:28 proxy2 postfix/postscreen[473]: COMMAND TIME LIMIT from [192.0.2.77]:3608 after CONNECT +# failJSON: { "time": "2005-06-08T23:14:54", "match": true , "host": "192.0.2.26", "desc": "abusive clients hitting command limit (gh-3040)" } +Jun 8 23:14:54 proxy2 postfix/postscreen[473]: COMMAND COUNT LIMIT from [192.0.2.26]:15592 after RCPT + + # filterOptions: [{}, {"mode": "ddos"}, {"mode": "aggressive"}] # failJSON: { "match": false, "desc": "don't affect lawful data (sporadical connection aborts within DATA-phase, see gh-1813 for discussion)" } Feb 18 09:50:05 xxx postfix/smtpd[42]: lost connection after DATA from good-host.example.com[192.0.2.10] diff --git a/fail2ban/tests/files/logs/proftpd b/fail2ban/tests/files/logs/proftpd index b255e91e..8d0d571c 100644 --- a/fail2ban/tests/files/logs/proftpd +++ b/fail2ban/tests/files/logs/proftpd @@ -1,6 +1,6 @@ -# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-01-10T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Jan 10 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username (Login failed): User in /etc/ftpusers -# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123" } +# failJSON: { "time": "2005-02-01T00:00:00", "match": true , "host": "123.123.123.123", "user": "username" } Feb 1 00:00:00 myhost proftpd[12345] myhost.domain.com (123.123.123.123[123.123.123.123]): USER username: no such user found from 123.123.123.123 [123.123.123.123] to 234.234.234.234:21 # failJSON: { "time": "2005-06-09T07:30:58", "match": true , "host": "67.227.224.66" } Jun 09 07:30:58 platypus.ace-hosting.com.au proftpd[11864] platypus.ace-hosting.com.au (mail.bloodymonster.net[::ffff:67.227.224.66]): USER username (Login failed): Incorrect password. @@ -12,7 +12,9 @@ Jun 13 22:07:23 platypus.ace-hosting.com.au proftpd[15719] platypus.ace-hosting. Jun 14 00:09:59 platypus.ace-hosting.com.au proftpd[17839] platypus.ace-hosting.com.au (::ffff:59.167.242.100[::ffff:59.167.242.100]): USER platypus.ace-hosting.com.au proftpd[17424] platypus.ace-hosting.com.au (hihoinjection[1.2.3.44]): no such user found from ::ffff:59.167.242.100 [::ffff:59.167.242.100] to ::ffff:113.212.99.194:21 # failJSON: { "time": "2005-05-31T10:53:25", "match": true , "host": "1.2.3.4" } May 31 10:53:25 mail proftpd[15302]: xxxxxxxxxx (::ffff:1.2.3.4[::ffff:1.2.3.4]) - Maximum login attempts (3) exceeded -# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4" } +# failJSON: { "time": "2004-10-02T15:45:44", "match": true , "host": "192.0.2.13", "user": "Root", "desc": "dot at end is optional (mod_sftp, gh-2246)" } +Oct 2 15:45:44 ftp01 proftpd[5517]: 192.0.2.13 (192.0.2.13[192.0.2.13]) - SECURITY VIOLATION: Root login attempted +# failJSON: { "time": "2004-12-05T15:44:32", "match": true , "host": "1.2.3.4", "user": "jtittle@domain.org" } Dec 5 15:44:32 serv1 proftpd[70944]: serv1.domain.com (example.com[1.2.3.4]) - USER jtittle@domain.org: no such user found from example.com [1.2.3.4] to 1.2.3.4:21 # failJSON: { "time": "2013-11-16T21:59:30", "match": true , "host": "1.2.3.4", "desc": "proftpd-basic 1.3.5~rc3-2.1 on Debian uses date format with milliseconds if logging under /var/log/proftpd/proftpd.log" } -2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21 +2013-11-16 21:59:30,121 novo proftpd[25891] localhost (andy[1.2.3.4]): USER kjsad: no such user found from andy [1.2.3.5] to ::ffff:192.168.1.14:21
\ No newline at end of file diff --git a/fail2ban/tests/files/logs/scanlogd b/fail2ban/tests/files/logs/scanlogd new file mode 100644 index 00000000..5a97c578 --- /dev/null +++ b/fail2ban/tests/files/logs/scanlogd @@ -0,0 +1,8 @@ +# failJSON: { "time": "2005-03-05T21:44:43", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:43 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 80, 81, 83, 88, 99, 443, 1080, 3128, ..., f????uxy, TOS 00, TTL 49 @20:44:43 +# failJSON: { "time": "2005-03-05T21:44:44", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:44 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 497, 515, 544, 543, 464, 513, ..., fSrpauxy, TOS 00 @09:04:25 +# failJSON: { "time": "2005-03-05T21:44:45", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:45 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 593, 548, 636, 646, 625, 631, ..., fSrpauxy, TOS 00, TTL 239 @17:34:00 +# failJSON: { "time": "2005-03-05T21:44:46", "match": true , "host": "192.0.2.123" } +Mar 5 21:44:46 srv scanlogd: 192.0.2.123 to 192.0.2.1 ports 22, 26, 37, 80, 25, 79, ..., fSrpauxy, TOS 00 @22:38:37 diff --git a/fail2ban/tests/files/logs/selinux-ssh b/fail2ban/tests/files/logs/selinux-ssh index f9e1b828..6ba552fe 100644 --- a/fail2ban/tests/files/logs/selinux-ssh +++ b/fail2ban/tests/files/logs/selinux-ssh @@ -27,3 +27,6 @@ type=USER_AUTH msg=audit(1383116263.000:603): pid=12887 uid=0 auid=4294967295 se # failJSON: { "time": "2013-10-30T07:54:08", "match": false , "host": "192.168.3.100" } type=USER_LOGIN msg=audit(1383116048.000:595): pid=12354 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=login acct="dan" exe="/usr/sbin/sshd" hostname=? addr=192.168.3.100 terminal=ssh res=failed' + +# failJSON: { "time": "2022-11-14T00:11:11", "match": true , "host": "192.0.2.111" } +type=USER_AUTH msg=audit(1668381071.000:373474): pid=173582 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:sshd_t:s0-s0:c0.c1023 msg='op=PAM:authentication grantors=? acct="root" exe="/usr/sbin/sshd" hostname=192.0.2.111 addr=192.0.2.111 terminal=ssh res=failed' UID="root" AUID="unset" diff --git a/fail2ban/tests/files/logs/sendmail-auth b/fail2ban/tests/files/logs/sendmail-auth index 835508f6..c7cc28de 100644 --- a/fail2ban/tests/files/logs/sendmail-auth +++ b/fail2ban/tests/files/logs/sendmail-auth @@ -14,3 +14,23 @@ Feb 24 13:00:17 kismet sm-acceptingconnections[1499]: s1OHxxSn001499: 192.241.70 # gh-1632, Fedora 24/RHEL - the daemon name is "sendmail": # failJSON: { "time": "2005-02-24T14:00:00", "match": true , "host": "192.0.2.1" } Feb 24 14:00:00 server sendmail[26592]: u0CB32qX026592: [192.0.2.1]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-24T14:00:01", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } +Feb 24 14:00:01 server sendmail[3529566]: xA32R2PQ3529566: [192.0.2.2]: possible SMTP attack: command=AUTH, count=5 + +# failJSON: { "time": "2005-02-25T04:02:27", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.16.1, AUTH_FAIL_LOG_USER (gh-2757)" } +Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) +# failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } +Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) + +# failJSON: { "time": "2005-05-24T01:58:40", "match": true , "host": "192.0.2.5", "desc": "user not found (gh-3030)" } +May 24 01:58:40 server sm-mta[65696]: 14NNwaRl065696: AUTH failure (DIGEST-MD5): user not found (-20) SASL(-13): user not found: unable to canonify user and get auxprops, user=scanner, relay=[192.0.2.5] +# failJSON: { "time": "2005-05-24T01:59:07", "match": true , "host": "192.0.2.6", "desc": "user not found (gh-3030)" } +May 24 01:59:07 server sm-mta[65815]: 14NNx65Q065815: AUTH failure (CRAM-MD5): user not found (-20) SASL(-13): user not found: user: scan@server.example.com property: userPassword not found in sasldb /usr/local/etc/sasldb2, user=scan, relay=[192.0.2.6] + +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.7", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (login): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=test, relay=host.example.com [192.0.2.7] (may be forged) +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.8", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (PLAIN): authentication failure (-13) SASL(-13): authentication failure: Password verification failed, user=test, relay=host.example.com [192.0.2.8] +# failJSON: { "time": "2005-05-29T23:14:05", "match": true , "host": "192.0.2.9", "desc": "authentication failure, no user part (gh-2757)" } +May 29 23:14:05 server sendmail[25411]: 26H8MRrS025411: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, relay=[192.0.2.9] diff --git a/fail2ban/tests/files/logs/sendmail-reject b/fail2ban/tests/files/logs/sendmail-reject index 44f8eb92..8debe7ca 100644 --- a/fail2ban/tests/files/logs/sendmail-reject +++ b/fail2ban/tests/files/logs/sendmail-reject @@ -40,6 +40,9 @@ Feb 27 15:49:07 batman sm-mta[88390]: ruleset=check_relay, arg1=189-30-205-74.pa # failJSON: { "time": "2005-02-19T18:01:50", "match": true , "host": "196.213.73.146" } Feb 19 18:01:50 batman sm-mta[78152]: ruleset=check_relay, arg1=[196.213.73.146], arg2=196.213.73.146, relay=[196.213.73.146], reject=421 4.3.2 Connection rate limit exceeded. +# failJSON: { "time": "2005-02-19T20:17:12", "match": true , "host": "192.0.2.123" } +Feb 19 20:17:12 server sm-mta[201892]: ruleset=check_relay, arg1=[192.0.2.123], arg2=192.0.2.123, relay=host.example.com [192.0.2.123] (may be forged), reject=421 4.3.2 Connection rate limit exceeded. + # failJSON: { "time": "2005-02-27T10:53:06", "match": true , "host": "209.15.212.253" } Feb 27 10:53:06 batman sm-mta[44307]: s1R9r60D044307: rejecting commands from [209.15.212.253] due to pre-greeting traffic after 0 seconds # failJSON: { "time": "2005-02-27T10:53:07", "match": true , "host": "1.2.3.4" } @@ -69,6 +72,8 @@ Feb 13 01:16:50 batman sm-mta[25815]: s1D0GoSs025815: [217.193.142.180]: vrfy in # failJSON: { "time": "2005-02-22T14:02:44", "match": true , "host": "24.73.201.194" } Feb 22 14:02:44 batman sm-mta[4030]: s1MD2hsd004030: rrcs-24-73-201-194.se.biz.rr.com [24.73.201.194]: VRFY root [rejected] +# failJSON: { "time": "2005-02-22T15:20:27", "match": true , "host": "192.0.2.5", "desc": "Fix reverse DNS for ... (gh-3012)" } +Feb 22 15:20:27 localhost sm-mta[275631]: 13O9Ixhq275631: ruleset=check_rcpt, arg1=<linda@domain.com>, relay=[192.0.2.5], reject=550 5.7.1 <linda@domain.com>... Fix reverse DNS for 192.0.2.5 # failJSON: { "match": false } Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026250: <arhipov@domain.com>... No such user here @@ -95,3 +100,15 @@ Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026255: from=<anton@domain.co Mar 6 16:55:28 s192-168-0-1 sm-mta[20949]: v26LtRA0020949: some-host-24.example.org [192.0.2.194] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA # failJSON: { "time": "2005-03-07T15:04:37", "match": true , "host": "192.0.2.195", "desc": "wrong resp. non RFC compiant (ddos prelude?), MSP-mode, (may be forged)" } Mar 7 15:04:37 s192-168-0-1 sm-mta[18624]: v27K4Vj8018624: some-host-24.example.org [192.0.2.195] (may be forged) did not issue MAIL/EXPN/VRFY/ETRN during connection to MSP-v4 + +# failJSON: { "time": "2005-03-29T22:33:47", "match": true , "host": "104.152.52.29", "desc": "wrong resp. non RFC compiant (ddos prelude?), TLSMTA-mode" } +Mar 29 22:33:47 kismet sm-mta[23221]: x2TMXH7Y023221: internettl.org [104.152.52.29] (may be forged) did not issue MAIL/EXPN/VRFY/ETRN during connection to TLSMTA +# failJSON: { "time": "2005-03-29T22:51:42", "match": true , "host": "104.152.52.29", "desc": "wrong resp. non RFC compiant (ddos prelude?), MSA-mode" } +Mar 29 22:51:42 kismet sm-mta[24202]: x2TMpAlI024202: internettl.org [104.152.52.29] (may be forged) did not issue MAIL/EXPN/VRFY/ETRN during connection to MSA + +# failJSON: { "time": "2005-03-29T22:51:43", "match": true , "host": "192.0.2.2", "desc": "long PID, ID longer as 14 chars (gh-2563)" } +Mar 29 22:51:43 server sendmail[3529565]: xA32R2PQ3529565: [192.0.2.2] did not issue MAIL/EXPN/VRFY/ETRN during connection to MTA +# failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } +Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 +# failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 diff --git a/fail2ban/tests/files/logs/softethervpn b/fail2ban/tests/files/logs/softethervpn new file mode 100644 index 00000000..dd2a798b --- /dev/null +++ b/fail2ban/tests/files/logs/softethervpn @@ -0,0 +1,7 @@ +# Access of unauthorized host in /usr/local/vpnserver/security_log/*/sec.log +# failJSON: { "time": "2020-05-12T10:53:19", "match": true , "host": "80.10.11.12" } +2020-05-12 10:53:19.781 Connection "CID-72": User authentication failed. The user name that has been provided was "bob", from 80.10.11.12. + +# Access of unauthorized host in syslog +# failJSON: { "time": "2020-05-13T10:53:19", "match": true , "host": "80.10.11.13" } +2020-05-13T10:53:19 localhost [myserver.com/VPN/defaultvpn] (2020-05-13 10:53:19.591) <SECURITY_LOG>: Connection "CID-594": User authentication failed. The user name that has been provided was "alice", from 80.10.11.13. diff --git a/fail2ban/tests/files/logs/sogo-auth b/fail2ban/tests/files/logs/sogo-auth index 02a69c6d..8728a76a 100644 --- a/fail2ban/tests/files/logs/sogo-auth +++ b/fail2ban/tests/files/logs/sogo-auth @@ -29,3 +29,6 @@ Mar 24 08:58:59 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user Mar 24 08:59:04 sogod [26818]: <0x0xb8537990[LDAPSource]> <NSException: 0xb87bc088> NAME:LDAPException REASON:operation bind failed: Invalid credentials (0x31) INFO:{login = "uid=admin,ou=users,dc=mail,dc=example,dc=org"; } # failJSON: { "time": "2005-03-24T08:59:04", "match": true , "host": "173.194.44.31" } Mar 24 08:59:04 sogod [26818]: SOGoRootPage Login from '173.194.44.31' for user 'admin' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 +# failJSON: { "time": "2005-03-24T19:29:32", "match": true , "host": "192.0.2.16", "desc": "behind a proxy, gh-2289" } +Mar 24 19:29:32 sogod [1526]: SOGoRootPage Login from '192.0.2.16, 10.0.0.1' for user 'admin' might not have worked - password policy: 65535 grace: -1 expire: -1 bound: 0 + diff --git a/fail2ban/tests/files/logs/sshd b/fail2ban/tests/files/logs/sshd index e2b3d456..99c3756b 100644 --- a/fail2ban/tests/files/logs/sshd +++ b/fail2ban/tests/files/logs/sshd @@ -134,7 +134,7 @@ Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 po # failJSON: { "time": "2004-09-29T17:15:02", "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } Sep 29 17:15:02 spaceman sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-09-29T17:15:03", "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } -Sep 29 17:15:03 spaceman sshd[12946]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +Sep 29 17:15:03 spaceman sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 # failJSON: { "time": "2004-11-11T08:04:51", "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } Nov 11 08:04:51 redbamboo sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 @@ -166,9 +166,11 @@ Nov 28 09:16:03 srv sshd[32307]: Connection closed by 192.0.2.1 Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx # failJSON: { "match": false } Nov 28 09:16:05 srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx -# failJSON: { "match": false } +# failJSON: { "constraint": "name == 'sshd'", "time": "2004-11-28T09:16:05", "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } Nov 28 09:16:05 srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] -# failJSON: { "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "desc": "Should catch failure - no success/no accepted public key" } +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2004-11-28T09:16:05", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } Nov 28 09:16:05 srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] # failJSON: { "match": false } @@ -215,7 +217,7 @@ Apr 27 13:02:04 host sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal S # Match sshd auth errors on OpenSUSE systems (gh-1024) # failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } 2015-04-16T18:02:50.321974+00:00 host sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root -# failJSON: { "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "time": "2015-04-16T20:02:50", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } 2015-04-16T18:02:50.568798+00:00 host sshd[2716]: Connection closed by 192.0.2.112 [preauth] # disable this test-cases block for obsolete multi-line filter (zzz-sshd-obsolete...): @@ -238,7 +240,7 @@ Mar 7 18:53:20 bar sshd[1556]: Connection closed by 192.0.2.113 Mar 7 18:53:22 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 # failJSON: { "time": "2005-03-07T18:53:23", "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:23 bar sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 -# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 2, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +# failJSON: { "time": "2005-03-07T18:53:24", "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } Mar 7 18:53:24 bar sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) @@ -248,11 +250,18 @@ Mar 7 18:53:24 bar sshd[1558]: pam_unix(sshd:session): session opened for user Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 # failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } Mar 7 18:53:32 bar sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 -# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 2, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +# failJSON: { "time": "2005-03-07T18:53:34", "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } Mar 7 18:53:34 bar sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 # failJSON: { "match": false , "desc": "No failure" } Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116 +# failJSON: { "time": "2005-03-19T16:47:48", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } +Mar 19 16:47:48 test sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 +# failJSON: { "time": "2005-03-19T16:47:49", "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +Mar 19 16:47:49 test sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] +# failJSON: { "time": "2005-03-19T16:47:50", "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } +Mar 19 16:47:50 srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] + # filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] # http://forums.powervps.com/showthread.php?t=1667 @@ -260,6 +269,10 @@ Mar 7 18:53:38 bar sshd[1559]: Connection closed by 192.0.2.116 Jun 7 01:10:56 host sshd[5937]: Did not receive identification string from 69.61.56.114 # failJSON: { "time": "2005-06-07T01:11:57", "match": true , "host": "192.0.2.5", "desc": "refactored message (with port now, gh-2062)" } Jun 7 01:11:57 host sshd[8782]: Did not receive identification string from 192.0.2.5 port 35836 +# failJSON: { "time": "2005-06-07T01:11:58", "match": true , "host": "69.61.56.115", "desc": "bad protocol version, gh-2404" } +Jun 7 01:11:58 host sshd[8783]: Bad protocol version identification 'dummy string' from 69.61.56.115 port 31778 +# failJSON: { "time": "2005-06-07T01:11:58", "match": true , "host": "69.61.56.115", "desc": "check inject on ident" } +Jun 7 01:11:58 host sshd[8783]: Bad protocol version identification 'dummy string' from 192.0.2.1' from 69.61.56.115 port 31778 # gh-864(1): # failJSON: { "match": false } @@ -283,6 +296,9 @@ Nov 24 23:46:43 host sshd[32686]: fatal: Read from socket failed: Connection res # failJSON: { "time": "2005-03-15T09:20:57", "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } Mar 15 09:20:57 host sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] +# failJSON: { "time": "2005-03-16T09:29:50", "match": true , "host": "192.0.2.20", "desc": "connection reset by user (gh-2662)" } +Mar 16 09:29:50 host sshd[19131]: Connection reset by authenticating user root 192.0.2.20 port 1558 [preauth] + # failJSON: { "time": "2005-07-17T23:03:05", "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } Jul 17 23:03:05 srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] # failJSON: { "time": "2005-07-17T23:04:00", "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } @@ -292,11 +308,43 @@ Jul 17 23:04:01 srv sshd[1300]: Connection closed by authenticating user test 12 # filterOptions: [{"test.condition":"name=='sshd'", "mode": "ddos"}, {"test.condition":"name=='sshd'", "mode": "aggressive"}] +# failJSON: { "match": false } +Feb 17 17:40:17 sshd[19725]: Connection from 192.0.2.10 port 62004 on 192.0.2.10 port 22 +# failJSON: { "time": "2005-02-17T17:40:17", "match": true , "host": "192.0.2.10", "desc": "ddos: port scanner (invalid protocol identifier)" } +Feb 17 17:40:17 sshd[19725]: error: kex_exchange_identification: client sent invalid protocol identifier "" +# failJSON: { "time": "2005-02-17T17:40:18", "match": true , "host": "192.0.2.10", "desc": "ddos: flood attack vector, gh-2850" } +Feb 17 17:40:18 sshd[19725]: error: kex_exchange_identification: Connection closed by remote host + +# failJSON: { "match": false } +Mar 1 18:59:33 hostname sshd[1189575]: error: kex_exchange_identification: banner line too long +# failJSON: { "time": "2005-03-01T18:59:33", "match": true , "host": "192.0.2.12", "desc": "ddos: port scanner, https payload on ssh port (banner exchange: invalid format, gh-3169)" } +Mar 1 18:59:33 hostname sshd[1189575]: banner exchange: Connection from 192.0.2.12 port 44105: invalid format + # failJSON: { "time": "2005-03-15T09:21:01", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:01 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] # failJSON: { "time": "2005-03-15T09:21:02", "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } Mar 15 09:21:02 host sshd[2717]: Connection closed by 192.0.2.212 [preauth] +# failJSON: { "time": "2005-07-18T17:19:11", "match": true , "host": "192.0.2.4", "desc": "ddos: disconnect on preauth phase, gh-2115" } +Jul 18 17:19:11 srv sshd[2101]: Disconnected from 192.0.2.4 port 36985 [preauth] + +# failJSON: { "time": "2005-06-06T04:17:04", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:04 host sshd[1189074]: Invalid user from 192.0.2.68 port 34916 +# failJSON: { "time": "2005-06-06T04:17:09", "match": true , "host": "192.0.2.68", "dns": null, "user": "", "desc": "empty user, gh-2749" } +Jun 6 04:17:09 host sshd[1189074]: Connection closed by invalid user 192.0.2.68 port 34916 [preauth] + +# failJSON: { "match": false, "desc": "ddos-failure without IP, retarded, must be triggered with next (closed) message, gh-3086"} +Jun 7 04:10:35 host sshd[424228]: error: kex_exchange_identification: Connection closed by remote host +# failJSON: { "time": "2005-06-07T04:10:35", "match": true , "host": "192.0.2.15", "desc": "kex_exchange_identification: Connection closed, gh-3086" } +Jun 7 04:10:35 host sshd[424228]: Connection closed by 192.0.2.15 port 35352 + +# failJSON: { "match": false } +Jun 7 04:29:10 host sshd[649921]: Connection from 192.0.2.16 port 51280 on 192.0.2.16 port 22 rdomain "" +# failJSON: { "time": "2005-06-07T04:29:10", "match": true, "host": "192.0.2.16", "desc": "ddos-failure without IP, must be triggered here because it became known above, gh-3086"} +Jun 7 04:29:10 host sshd[649921]: error: kex_exchange_identification: read: Connection reset by peer +# failJSON: { "match": false, "desc": "Connection reset already triggered above (known IP, no-fail helper unused here)" } +Jun 7 04:29:10 host sshd[649921]: Connection reset by 192.0.2.16 port 51280 + # filterOptions: [{"mode": "extra"}, {"mode": "aggressive"}] # several other cases from gh-864: @@ -306,6 +354,8 @@ Nov 25 01:34:12 srv sshd[123]: Received disconnect from 127.0.0.1: 14: No suppor Nov 25 01:35:13 srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] # failJSON: { "time": "2004-11-25T01:35:14", "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } Nov 25 01:35:14 srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] +# failJSON: { "time": "2004-11-25T01:35:15", "match": true , "host": "192.168.2.93", "desc": "No authentication methods available (supported is optional, gh-2682)" } +Nov 25 01:35:15 srv sshd[3626]: error: Received disconnect from 192.168.2.93 port 1883:14: No authentication methods available [preauth] # gh-1545: # failJSON: { "time": "2004-11-26T13:03:29", "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } @@ -318,7 +368,7 @@ Nov 26 13:03:30 srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 554 Nov 26 15:03:30 host sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 # failJSON: { "time": "2004-11-26T15:03:31", "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } Nov 26 15:03:31 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] -# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "filter": "sshd", "desc": "Second attempt within the same connect" } +# failJSON: { "time": "2004-11-26T15:03:32", "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } Nov 26 15:03:32 host sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] # gh-1943 (previous OpenSSH log-format) @@ -334,3 +384,16 @@ Oct 26 15:30:40 localhost sshd[14737]: Unable to negotiate with 192.0.2.2 port 5 Nov 26 13:03:38 srv sshd[14737]: Unable to negotiate with 192.0.2.4 port 50404: no matching host key type found. Their offer: ssh-dss # failJSON: { "time": "2004-11-26T13:03:39", "match": true , "host": "192.0.2.5", "desc": "No matching everything ... found." } Nov 26 13:03:39 srv sshd[14738]: fatal: Unable to negotiate with 192.0.2.5 port 55555: no matching everything new here found. Their offer: ... + +# failJSON: { "time": "2004-11-26T16:47:51", "match": true , "host": "192.0.2.6", "desc": "Disconnected during preauth phase (in extra/aggressive mode)" } +Nov 26 16:47:51 srv sshd[19320]: Disconnected from authenticating user root 192.0.2.6 port 33553 [preauth] + +# filterOptions: {"test.condition":"name=='sshd'", "logtype": "rfc5424"} + +# failJSON: { "time": "2019-07-08T23:40:16", "match": true , "host": "192.0.2.1", "desc": "RFC 5424 format (gh-2309)" } +<38>1 2019-07-08T17:40:16.954167-04:00 hostname.example.com sshd 57915 - - Failed unknown for invalid user redmond from 192.0.2.1 port 59197 ssh2 + +# failJSON: { "time": "2019-07-08T23:40:17", "match": true , "host": "192.0.2.2", "desc": "RFC 5424 format with extra data and brackets in value (gh-2309)" } +<38>1 2019-07-08T17:40:17.954167-04:00 hostname.example.com sshd 55555 - [timeQuality tzKnown="1" isSynced="0"][xxx@123 test="[brackets]"] Failed unknown for invalid user redmond from 192.0.2.2 port 55555 ssh2 + +# addFILE: "sshd-journal" diff --git a/fail2ban/tests/files/logs/sshd-journal b/fail2ban/tests/files/logs/sshd-journal new file mode 100644 index 00000000..d19889d7 --- /dev/null +++ b/fail2ban/tests/files/logs/sshd-journal @@ -0,0 +1,348 @@ +# Systemd-Journal filter coverage: +# disable this test-file for obsolete multi-line filter (zzz-sshd-obsolete..., it would work, but slow) +# fileOptions: {"logtype": "journal", "test.condition":"name=='sshd'"} + +# filterOptions: [{}, {"mode": "aggressive"}] + +#1 +# failJSON: { "match": true , "host": "192.030.0.6" } +srv sshd[13709]: error: PAM: Authentication failure for myhlj1374 from 192.030.0.6 +# failJSON: { "match": true , "host": "example.com" } +srv sshd[28732]: error: PAM: Authentication failure for stefanor from example.com +# failJSON: { "match": true , "host": "2606:2800:220:1:248:1893:25c8:1946" } +srv sshd[28732]: error: PAM: Authentication failure for test-ipv6 from 2606:2800:220:1:248:1893:25c8:1946 + +#2 +# failJSON: { "match": true , "host": "194.117.26.69" } +srv sshd[31602]: Failed password for invalid user ROOT from 194.117.26.69 port 50273 ssh2 +# failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +srv sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 50273 ssh2 +# failJSON: { "match": true , "host": "194.117.26.70" } +srv sshd[31602]: Failed password for invalid user ROOT from 194.117.26.70 port 12345 +# failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +srv sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 port 12345 +# failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1" } +srv sshd[31603]: Failed password for invalid user ROOT from aaaa:bbbb:cccc:1234::1:1 + +#3 +# failJSON: { "match": true , "host": "1.2.3.4" } +srv sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 +# failJSON: { "match": true , "host": "1.2.3.4" } +srv sshd[1643]: ROOT LOGIN REFUSED FROM 1.2.3.4 port 12345 [preauth] +# failJSON: { "match": true , "host": "1.2.3.4" } +srv sshd[1643]: ROOT LOGIN REFUSED FROM ::ffff:1.2.3.4 + +#4 +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "Invalid user" } +srv sshd[22708]: Invalid user ftp from 192.0.2.1 +# failJSON: { "match": true , "host": "192.0.2.2", "desc": "Invalid user with port" } +srv sshd[22708]: Invalid user ftp from 192.0.2.2 port 37220 + +#5 new filter introduced after looking at 44087D8C.9090407@bluewin.ch +# failJSON: { "match": true , "host": "211.188.220.49" } +srv sshd[31605]: User root from 211.188.220.49 not allowed because not listed in AllowUsers +# failJSON: { "match": true , "host": "example.com" } +srv sshd[31607]: User root from example.com not allowed because not listed in AllowUsers + +#6 ew filter introduced thanks to report Guido Bozzetto <reportbug@G-B.it> +# failJSON: { "match": true , "host": "218.249.210.161" } +srv sshd[5174]: refused connect from _U2FsdGVkX19P3BCJmFBHhjLza8BcMH06WCUVwttMHpE=_@::ffff:218.249.210.161 (::ffff:218.249.210.161) + +#7 added exclamation mark to BREAK-IN +# Now should be a negative since we decided not to catch those +# failJSON: { "match": false } +srv sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT +# failJSON: { "match": false } +srv sshd[7592]: Address 1.2.3.4 maps to 1234.bbbbbb.com, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! + +#8 DenyUsers https://github.com/fail2ban/fail2ban/issues/47 +# failJSON: { "match": true , "host": "46.45.128.3" } +srv sshd[5154]: User root from 46.45.128.3 not allowed because listed in DenyUsers + +#9 systemd with kernel entry: +# failJSON: { "match": true , "host": "205.186.180.55" } +srv sshd[20878]: kernel:[ 970.699396]: Failed keyboard-interactive for <invalid username> from 205.186.180.55 port 42742 ssh2 +# failJSON: { "match": true , "ip4": "192.0.2.10" } +srv sshd[20879]: kernel: [ 970.699397] Failed password for user admin from 192.0.2.10 port 42745 ssh2 +# failJSON: { "match": true , "ip6": "2001:db8::1" } +srv sshd[20880]: kernel:[12970.699398] Failed password for user admin from 2001:db8::1 port 42746 ssh2 + +#10 OSX syslog error +# failJSON: { "match": true , "host": "example.com" } +srv sshd[62312]: error: PAM: authentication error for james from example.com via 192.168.1.201 +# failJSON: { "match": true , "host": "205.186.180.35" } +srv sshd[63814]: Failed keyboard-interactive for <invalid username> from 205.186.180.35 port 42742 ssh2 +# failJSON: { "match": true , "host": "205.186.180.22" } +srv sshd[63814]: Failed keyboard-interactive for james from 205.186.180.22 port 54520 ssh2 +# failJSON: { "match": true , "host": "205.186.180.42" } +srv sshd[63814]: Failed keyboard-interactive for james from 205.186.180.42 port 54520 ssh2 +# failJSON: { "match": true , "host": "205.186.180.44" } +srv sshd[63814]: Failed keyboard-interactive for <invalid username> from 205.186.180.44 port 42742 ssh2 +# failJSON: { "match": true , "host": "205.186.180.77" } +srv sshd[2554]: Failed keyboard-interactive/pam for invalid user jamedds from 205.186.180.77 port 33723 ssh2 +# failJSON: { "match": true , "host": "205.186.180.88" } +srv sshd[47831]: error: PAM: authentication failure for james from 205.186.180.88 via 192.168.1.201 +# failJSON: { "match": true , "host": "205.186.180.99" } +srv sshd[47831]: error: PAM: Authentication failure for james from 205.186.180.99 via 192.168.1.201 +# failJSON: { "match": true , "host": "205.186.180.100" } +srv sshd[47831]: error: PAM: Authentication error for james from 205.186.180.100 via 192.168.1.201 +# failJSON: { "match": true , "host": "205.186.180.101" } +srv sshd[47831]: error: PAM: authentication error for james from 205.186.180.101 via 192.168.1.201 +# failJSON: { "match": true , "host": "205.186.180.102" } +srv sshd[47831]: error: PAM: authentication error for james from 205.186.180.102 +# failJSON: { "match": true , "host": "205.186.180.103" } +srv sshd[47831]: error: PAM: authentication error for james from 205.186.180.103 + +# failJSON: { "match": false } +srv sshd[3719]: User root not allowed because account is locked +# failJSON: { "match": false } +srv sshd[3719]: input_userauth_request: invalid user root [preauth] +# failJSON: { "match": true , "host": "198.51.100.34" } +srv sshd[3719]: error: Received disconnect from 198.51.100.34: 11: Bye Bye [preauth] +# failJSON: { "match": true , "host": "10.215.4.227" } +srv sshd[1328]: error: PAM: User not known to the underlying authentication module for illegal user kernelitshell from 10.215.4.227 +# failJSON: { "match": true , "host": "example.com" } +srv sshd[9739]: User allena from example.com not allowed because not in any group +# failJSON: { "match": true , "host": "192.51.100.54" } +srv sshd[5106]: User root from 192.51.100.54 not allowed because a group is listed in DenyGroups +# failJSON: { "match": true , "host": "10.0.0.40" } +srv sshd[1966]: User root from 10.0.0.40 not allowed because none of user's groups are listed in AllowGroups + +# failJSON: { "match": false } +srv sshd[2364]: User root not allowed because account is locked +# failJSON: { "match": false } +srv sshd[2364]: input_userauth_request: invalid user root [preauth] +# failJSON: { "match": true , "host": "198.51.100.76" } +srv sshd[2364]: Received disconnect from 198.51.100.76 port 58846:11: Bye Bye [preauth] + +# failJSON: { "match": true , "host": "127.0.0.1" } +srv sshd[16699]: Failed password for dan from 127.0.0.1 port 45416 ssh1 + +# failJSON: { "match": false, "desc": "no failure, just cache mlfid (conn-id)" } +srv sshd[16700]: Connection from 192.0.2.5 +# failJSON: { "match": false, "desc": "no failure, just covering mlfid (conn-id) forget" } +srv sshd[16700]: Connection closed by 192.0.2.5 + +# failJSON: { "match": true , "host": "127.0.0.1" } +srv sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: RSA 8c:e3:aa:0f:64:51:02:f7:14:79:89:3f:65:84:7c:30, client user "dan", client host "localhost.localdomain" + +# failJSON: { "match": true , "host": "127.0.0.1" } +srv sshd[12946]: Failed hostbased for dan from 127.0.0.1 port 45785 ssh2: DSA 01:c0:79:41:91:31:9a:7d:95:23:91:ac:b1:6d:59:81, client user "dan", client host "localhost.localdomain" + +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting into rhost for the format of OpenSSH >=6.3" } +srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser from 1.2.3.4 + +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } +srv sshd[12946]: Failed password for user from 127.0.0.1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 +# failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Injecting while exhausting initially present {0,100} match length limits set for ruser etc" } +srv sshd[12947]: Failed password for user from aaaa:bbbb:cccc:1234::1:1 port 20000 ssh1: ruser XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX from 1.2.3.4 + +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Injecting on username ssh 'from 10.10.1.1'@localhost" } +srv sshd[2737]: Failed password for invalid user from 10.10.1.1 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "More complex injecting on username ssh 'test from 10.10.1.2 port 55555 ssh2'@localhost" } +srv sshd[2737]: Failed password for invalid user test from 10.10.1.2 port 55555 ssh2 from 127.0.0.1 port 58946 ssh2 +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "More complex injecting on auth-info ssh test@localhost, auth-info: ' from 10.10.1.2 port 55555 ssh2'" } +srv sshd[2737]: Failed password for invalid user test from 127.0.0.1 port 58946 ssh2: from 10.10.1.2 port 55555 ssh2 + +# Failure on connect of invalid user with public keys: +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Failed publickey for ..." } +srv sshd[4669]: Failed publickey for invalid user graysky from 127.0.0.1 port 37954 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI +# failJSON: { "match": true , "host": "aaaa:bbbb:cccc:1234::1:1", "desc": "Failed publickey for ..." } +srv sshd[4670]: Failed publickey for invalid user graysky from aaaa:bbbb:cccc:1234::1:1 port 37955 ssh2: RSA SHA256:v3dpapGleDaUKf$4V1vKyR9ZyUgjaJAmoCTcb2PLljI + +# Ignore tries of legitimate users with multiple public keys (gh-1263): +# failJSON: { "match": false } +srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: ECDSA 0e:ff:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx +# failJSON: { "match": false } +srv sshd[32307]: Failed publickey for git from 192.0.2.1 port 57904 ssh2: RSA 04:bc:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx +# failJSON: { "match": false } +srv sshd[32307]: Postponed publickey for git from 192.0.2.1 port 57904 ssh2 [preauth] +# failJSON: { "match": false } +srv sshd[32307]: Accepted publickey for git from 192.0.2.1 port 57904 ssh2: DSA 36:48:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx +# failJSON: { "match": false, "desc": "Should be forgotten by success/accepted public key" } +srv sshd[32307]: Connection closed by 192.0.2.1 + +# Failure on connect with valid user-name but wrong public keys (retarded to disconnect/too many errors, because of gh-1263): +# failJSON: { "match": false } +srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: ECDSA 1e:fe:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx +# failJSON: { "match": false } +srv sshd[32310]: Failed publickey for git from 192.0.2.111 port 57910 ssh2: RSA 14:ba:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx +# failJSON: { "match": true , "attempts": 3, "desc": "Should catch failure - no success/no accepted public key" } +srv sshd[32310]: Disconnecting: Too many authentication failures for git [preauth] +# failJSON: { "constraint": "opts.get('mode') != 'aggressive'", "match": false, "desc": "Nofail in normal mode, failure already produced above" } +srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.111", "attempts":1, "desc": "Matches in aggressive mode only" } +srv sshd[32310]: Connection closed by 192.0.2.111 [preauth] + +# failJSON: { "match": false } +srv sshd[8148]: Disconnecting: Too many authentication failures for root [preauth] +# failJSON: { "match": true , "host": "61.0.0.1", "desc": "Multiline match for preauth failures" } +srv sshd[8148]: Connection closed by 61.0.0.1 [preauth] + +# failJSON: { "match": false } +srv sshd[9148]: Disconnecting: Too many authentication failures for root [preauth] +# failJSON: { "match": false , "desc": "Pids don't match" } +srv sshd[7148]: Connection closed by 61.0.0.1 + +# failJSON: { "match": true , "host": "89.24.13.192", "desc": "from gh-289" } +srv sshd[4931]: Received disconnect from 89.24.13.192: 3: com.jcraft.jsch.JSchException: Auth fail +# failJSON: { "match": true , "host": "10.0.0.1", "desc": "space after port is optional (gh-1652)" } +srv sshd[11808]: error: Received disconnect from 10.0.0.1 port 7736:3: com.jcraft.jsch.JSchException: Auth fail [preauth] + +# failJSON: { "match": true , "host": "94.249.236.6", "desc": "newer format per commit 36919d9f" } +srv sshd[24077]: error: Received disconnect from 94.249.236.6: 3: com.jcraft.jsch.JSchException: Auth fail [preauth] + +# failJSON: { "match": true , "host": "94.249.236.6", "desc": "space in disconnect description per commit 36919d9f" } +srv sshd[24077]: error: Received disconnect from 94.249.236.6: 3: Ha ha, suckers!: Auth fail [preauth] + +# failJSON: { "match": false } +srv sshd[26713]: Connection from 115.249.163.77 port 51353 +# failJSON: { "match": true , "host": "115.249.163.77", "desc": "from gh-457" } +srv sshd[26713]: Disconnecting: Too many authentication failures for root [preauth] + +# failJSON: { "match": false } +srv sshd[26713]: Connection from 115.249.163.77 port 51353 on 127.0.0.1 port 22 +# failJSON: { "match": true , "host": "115.249.163.77", "desc": "Multiline match with interface address" } +srv sshd[26713]: Disconnecting: Too many authentication failures [preauth] + +# failJSON: { "match": true , "host": "61.0.0.1", "desc": "New logline format as openssh 6.8 to replace prev multiline version" } +srv sshd[21810]: error: maximum authentication attempts exceeded for root from 61.0.0.1 port 49940 ssh2 [preauth] + +# failJSON: { "match": false } +srv sshd[29116]: User root not allowed because account is locked +# failJSON: { "match": false } +srv sshd[29116]: input_userauth_request: invalid user root [preauth] +# failJSON: { "match": true , "host": "1.2.3.4", "desc": "No Bye-Bye" } +srv sshd[29116]: Received disconnect from 1.2.3.4: 11: Normal Shutdown, Thank you for playing [preauth] + +# Match sshd auth errors on OpenSUSE systems (gh-1024) +# failJSON: { "match": false, "desc": "No failure until closed or another fail (e. g. F-MLFFORGET by success/accepted password can avoid failure, see gh-2070)" } +srv sshd[2716]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.112 user=root +# failJSON: { "constraint": "opts.get('mode') == 'aggressive'", "match": true , "host": "192.0.2.112", "desc": "Should catch failure - no success/no accepted password" } +srv sshd[2716]: Connection closed by 192.0.2.112 [preauth] + +# filterOptions: [{}] + +# 2 methods auth: pam_unix and pam_ldap are used in combination (gh-2070), succeeded after "failure" in first method: +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.0.2.113 user=rda +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1556]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=rda rhost=192.0.2.113 [preauth] +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1556]: Accepted password for rda from 192.0.2.113 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1556]: pam_unix(sshd:session): session opened for user rda by (uid=0) +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1556]: Connection closed by 192.0.2.113 + +# several attempts, intruder tries to "forget" failed attempts by success login (all 3 attempts with different users): +# failJSON: { "match": false , "desc": "Still no failure (first try)" } +srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.114 +# failJSON: { "match": true , "attempts": 2, "users": ["root", "sudoer"], "host": "192.0.2.114", "desc": "Failure: attempt 2nd user" } +srv sshd[1558]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=sudoer rhost=192.0.2.114 +# failJSON: { "match": true , "attempts": 1, "users": ["root", "sudoer", "known"], "host": "192.0.2.114", "desc": "Failure: attempt 3rd user" } +srv sshd[1558]: Accepted password for known from 192.0.2.114 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1558]: pam_unix(sshd:session): session opened for user known by (uid=0) + +# several attempts, intruder tries to "forget" failed attempts by success login (accepted for other user as in first failed attempt): +# failJSON: { "match": false , "desc": "Still no failure (first try)" } +srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 +# failJSON: { "match": false , "desc": "Still no failure (second try, same user)" } +srv sshd[1559]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser=root rhost=192.0.2.116 +# failJSON: { "match": true , "attempts": 3, "users": ["root", "known"], "host": "192.0.2.116", "desc": "Failure: attempt 2nd user" } +srv sshd[1559]: Accepted password for known from 192.0.2.116 port 52100 ssh2 +# failJSON: { "match": false , "desc": "No failure" } +srv sshd[1559]: Connection closed by 192.0.2.116 + +# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt invalid user" } +srv sshd[5672]: Invalid user admin from 192.0.2.117 port 44004 +# failJSON: { "match": true , "attempts": 1, "user": "admin", "host": "192.0.2.117", "desc": "Failure: attempt to change user (disallowed)" } +srv sshd[5672]: Disconnecting invalid user admin 192.0.2.117 port 44004: Change of username or service not allowed: (admin,ssh-connection) -> (user,ssh-connection) [preauth] +# failJSON: { "match": false, "desc": "Disconnected during preauth phase (no failure in normal mode)" } +srv sshd[5672]: Disconnected from authenticating user admin 192.0.2.6 port 33553 [preauth] + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# http://forums.powervps.com/showthread.php?t=1667 +# failJSON: { "match": true , "host": "69.61.56.114" } +srv sshd[5937]: Did not receive identification string from 69.61.56.114 +# failJSON: { "match": true , "host": "192.0.2.5", "desc": "refactored message (with port now, gh-2062)" } +srv sshd[8782]: Did not receive identification string from 192.0.2.5 port 35836 + +# gh-864(1): +# failJSON: { "match": false } +srv sshd[32686]: SSH: Server;Ltype: Version;Remote: 127.0.0.1-1780;Protocol: 2.0;Client: libssh2_1.4.3 +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (1)" } +srv sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# gh-864(2): +# failJSON: { "match": false } +srv sshd[32686]: SSH: Server;Ltype: Kex;Remote: 127.0.0.1-1780;Enc: aes128-ctr;MAC: hmac-sha1;Comp: none [preauth] +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (2)" } +srv sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# gh-864(3): +# failJSON: { "match": false } +srv sshd[32686]: SSH: Server;Ltype: Authname;Remote: 127.0.0.1-1780;Name: root [preauth] +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "Multiline for connection reset by peer (3)" } +srv sshd[32686]: fatal: Read from socket failed: Connection reset by peer [preauth] + +# gh-1719: +# failJSON: { "match": true , "host": "192.0.2.39", "desc": "Singleline for connection reset by" } +srv sshd[28972]: Connection reset by 192.0.2.39 port 14282 [preauth] + +# failJSON: { "match": true , "host": "192.0.2.10", "user": "root", "desc": "user name additionally, gh-2185" } +srv sshd[1296]: Connection closed by authenticating user root 192.0.2.10 port 46038 [preauth] +# failJSON: { "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1", "desc": "check inject on username, gh-2185" } +srv sshd[1300]: Connection closed by authenticating user test 127.0.0.1 192.0.2.11 port 46039 [preauth] +# failJSON: { "match": true , "host": "192.0.2.11", "user": "test 127.0.0.1 port 12345", "desc": "check inject on username, gh-2185" } +srv sshd[1300]: Connection closed by authenticating user test 127.0.0.1 port 12345 192.0.2.11 port 46039 [preauth] + +# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}] + +# failJSON: { "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } +srv sshd[2717]: Connection closed by 192.0.2.212 [preauth] +# failJSON: { "match": true , "host": "192.0.2.212", "desc": "DDOS mode causes failure on close within preauth stage" } +srv sshd[2717]: Connection closed by 192.0.2.212 [preauth] + +# filterOptions: [{"logtype": "journal", "mode": "extra"}, {"logtype": "journal", "mode": "aggressive"}] + +# several other cases from gh-864: +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } +srv sshd[123]: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] +# failJSON: { "match": true , "host": "127.0.0.1", "desc": "No supported authentication methods" } +srv sshd[123]: error: Received disconnect from 127.0.0.1: 14: No supported authentication methods available [preauth] +# failJSON: { "match": true , "host": "192.168.2.92", "desc": "Optional space after port" } +srv sshd[3625]: error: Received disconnect from 192.168.2.92 port 1684:14: No supported authentication methods available [preauth] + +# gh-1545: +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "No matching cipher" } +srv sshd[45]: Unable to negotiate with 192.0.2.1 port 55419: no matching cipher found. Their offer: aes256-cbc,rijndael-cbc@lysator.liu.se,aes192-cbc,aes128-cbc,arcfour128,arcfour,3des-cbc,none [preauth] + +# gh-1117: +# failJSON: { "match": true , "host": "192.0.2.2", "desc": "No matching key exchange method" } +srv sshd[45]: fatal: Unable to negotiate with 192.0.2.2 port 55419: no matching key exchange method found. Their offer: diffie-hellman-group1-sha1 +# failJSON: { "match": false } +srv sshd[22440]: Connection from 192.0.2.3 port 39678 on 192.168.1.9 port 22 +# failJSON: { "match": true , "host": "192.0.2.3", "desc": "Multiline - no matching key exchange method" } +srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] +# failJSON: { "match": true , "host": "192.0.2.3", "constraint": "name == 'sshd'", "desc": "Second attempt within the same connect" } +srv sshd[22440]: fatal: Unable to negotiate a key exchange method [preauth] + +# gh-1943 (previous OpenSSH log-format) +# failJSON: { "match": false } +srv sshd[22477]: Connection from 192.0.2.1 port 31309 on 192.0.2.8 port 22 +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "No matching mac found" } +srv sshd[22477]: fatal: no matching mac found: client hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx,hmac-xxx server hmac-xxx,hmac-xxx,umac-xxx,hmac-xxx,hmac-xxx,umac-xxx [preauth] + +# gh-1944 (newest OpenSSH log-format) +# failJSON: { "match": true , "host": "192.0.2.2", "desc": "No matching MAC found" } +srv sshd[14737]: Unable to negotiate with 192.0.2.2 port 50404: no matching MAC found. Their offer: hmac-sha1,hmac-sha1-96,hmac-md5,hmac-md5-96,hmac-ripemd160,hmac-ripemd160@openssh.com [preauth] +# failJSON: { "match": true , "host": "192.0.2.4", "desc": "No matching everything ... found." } +srv sshd[14737]: Unable to negotiate with 192.0.2.4 port 50404: no matching host key type found. Their offer: ssh-dss +# failJSON: { "match": true , "host": "192.0.2.5", "desc": "No matching everything ... found." } +srv sshd[14738]: fatal: Unable to negotiate with 192.0.2.5 port 55555: no matching everything new here found. Their offer: ... + +# failJSON: { "match": true , "host": "192.0.2.6", "desc": "Disconnected during preauth phase (in extra/aggressive mode)" } +srv sshd[19320]: Disconnected from authenticating user root 192.0.2.6 port 33553 [preauth] diff --git a/fail2ban/tests/files/logs/traefik-auth b/fail2ban/tests/files/logs/traefik-auth new file mode 100644 index 00000000..edfe7306 --- /dev/null +++ b/fail2ban/tests/files/logs/traefik-auth @@ -0,0 +1,23 @@ +# filterOptions: [{"mode": "normal"}]
+
+# failJSON: { "match": false }
+10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms
+
+# filterOptions: [{"mode": "ddos"}]
+
+# failJSON: { "match": false }
+10.0.0.2 - username [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms
+
+# filterOptions: [{"mode": "normal"}, {"mode": "aggressive"}]
+
+# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2" }
+10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms
+# failJSON: { "time": "2018-11-18T22:34:34", "match": true , "host": "10.0.0.2", "desc": "other request method" }
+10.0.0.2 - username [18/Nov/2018:21:34:34 +0000] "TRACE /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms
+# failJSON: { "match": false }
+10.0.0.2 - username [27/Nov/2018:23:33:31 +0000] "GET /dashboard/ HTTP/2.0" 200 716 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 118 "Host-traefik-0" "/dashboard/" 4ms
+
+# filterOptions: [{"mode": "ddos"}, {"mode": "aggressive"}]
+
+# failJSON: { "time": "2018-11-18T22:34:30", "match": true , "host": "10.0.0.2" }
+10.0.0.2 - - [18/Nov/2018:21:34:30 +0000] "GET /dashboard/ HTTP/2.0" 401 17 "-" "Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0" 72 "Auth for frontend-Host-traefik-0" "/dashboard/" 0ms
diff --git a/fail2ban/tests/files/logs/znc-adminlog b/fail2ban/tests/files/logs/znc-adminlog new file mode 100644 index 00000000..143bf95b --- /dev/null +++ b/fail2ban/tests/files/logs/znc-adminlog @@ -0,0 +1,15 @@ +# failJSON: { "time": "2018-10-27T01:40:55", "match": true , "host": "1.2.3.4" } +[2018-10-27 01:40:55] [girst] failed to login from 1.2.3.4 + +# failJSON: { "match": false } +[2018-10-27 01:40:17] [girst] connected to ZNC from 1.2.3.4 +# failJSON: { "match": false } +[2018-10-27 01:40:21] [girst] disconnected from ZNC from 1.2.3.4 + +# failJSON: { "time": "2019-09-08T15:53:19", "match": true , "host": "192.0.2.1", "desc": "port after IP" } +[2019-09-08 15:53:19] [admin] failed to login from 192.0.2.1:65001 + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.2", "desc": "systemd-journal entry, port after IP" } +Test znc[37232]: [admin] failed to login from 192.0.2.2:65009 diff --git a/fail2ban/tests/files/logs/zoneminder b/fail2ban/tests/files/logs/zoneminder index abd49869..f4b6bd3e 100644 --- a/fail2ban/tests/files/logs/zoneminder +++ b/fail2ban/tests/files/logs/zoneminder @@ -1,2 +1,8 @@ # failJSON: { "time": "2016-03-28T16:50:49", "match": true , "host": "10.1.1.1" } [Mon Mar 28 16:50:49.522240 2016] [:error] [pid 1795] [client 10.1.1.1:50700] WAR [Login denied for user "username1"], referer: https://zoneminder/ + +# failJSON: { "time": "2021-03-28T16:53:00", "match": true , "host": "10.1.1.1" } +[Sun Mar 28 16:53:00.472693 2021] [php7:notice] [pid 11328] [client 10.1.1.1:39568] ERR [Could not retrieve user username1 details], referer: https://zm/zm/?view=logout + +# failJSON: { "time": "2021-03-28T16:59:14", "match": true , "host": "10.1.1.1" } +[Sun Mar 28 16:59:14.150625 2021] [php7:notice] [pid 11336] [client 10.1.1.1:39654] ERR [Login denied for user "username1"], referer: https://zm/zm/? diff --git a/fail2ban/tests/files/logs/zzz-generic-example b/fail2ban/tests/files/logs/zzz-generic-example index d0c31740..118c7e12 100644 --- a/fail2ban/tests/files/logs/zzz-generic-example +++ b/fail2ban/tests/files/logs/zzz-generic-example @@ -30,8 +30,8 @@ Jun 21 16:55:02 <auth.info> machine kernel: [ 970.699396] @vserver_demo test- # failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.3" } [Jun 21 16:55:03] <auth.info> machine kernel: [ 970.699396] @vserver_demo test-demo(pam_unix)[13709] [ID 255 test] F2B: failure from 192.0.2.3 -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-21T16:55:03", "match": true , "host": "192.0.2.1" } +# -- wrong time direct in journal-line (used last known date or now, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.1" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.1 # -- wrong time after newline in message (plist without escaped newlines): # failJSON: { "match": false } @@ -42,8 +42,8 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ applicationDate = "0000-12-30 00:00:00 +0000"; # failJSON: { "match": false } } -# -- wrong time direct in journal-line (used last known date): -# failJSON: { "time": "2005-06-22T20:37:04", "match": true , "host": "192.0.2.2" } +# -- wrong time direct in journal-line (used last known date, but null because no checkFindTime in samples test factory): +# failJSON: { "time": null, "match": true , "host": "192.0.2.2" } 0000-12-30 00:00:00 server test-demo[47831]: F2B: failure from 192.0.2.2 # -- test no zone and UTC/GMT named zone "2005-06-21T14:55:10 UTC" == "2005-06-21T16:55:10 CEST" (diff +2h in CEST): @@ -60,3 +60,6 @@ Jun 22 20:37:04 server test-demo[402]: writeToStorage plist={ [Jun 21 16:56:03] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.251 # failJSON: { "match": false, "desc": "test 2nd ignoreregex" } [Jun 21 16:56:04] machine test-demo(pam_unix)[13709] F2B: error from 192.0.2.252 + +# failJSON: { "match": false, "desc": "ignore other daemon" } +[Jun 21 16:56:04] machine captain-nemo(pam_unix)[55555] F2B: error from 192.0.2.2 diff --git a/fail2ban/tests/filtertestcase.py b/fail2ban/tests/filtertestcase.py index 350d46b0..4e308e38 100644 --- a/fail2ban/tests/filtertestcase.py +++ b/fail2ban/tests/filtertestcase.py @@ -40,10 +40,11 @@ from ..server.jail import Jail from ..server.filterpoll import FilterPoll from ..server.filter import FailTicket, Filter, FileFilter, FileContainer from ..server.failmanager import FailManagerEmpty -from ..server.ipdns import DNSUtils, IPAddr +from ..server.ipdns import asip, getfqdn, DNSUtils, IPAddr, IPAddrSet from ..server.mytime import MyTime from ..server.utils import Utils, uni_decode -from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_tmpdir, LogCaptureTestCase, \ +from .databasetestcase import getFail2BanDb +from .utils import setUpMyTime, tearDownMyTime, mtimesleep, with_alt_time, with_tmpdir, LogCaptureTestCase, \ logSys as DefLogSys, CONFIG_DIR as STOCK_CONF_DIR from .dummyjail import DummyJail @@ -62,10 +63,7 @@ def open(*args): if len(args) == 2: # ~50kB buffer should be sufficient for all tests here. args = args + (50000,) - if sys.version_info >= (3,): - return fopen(*args, **{'encoding': 'utf-8', 'errors': 'ignore'}) - else: - return fopen(*args) + return fopen(*args) def _killfile(f, name): @@ -94,7 +92,7 @@ class _tmSerial(): @staticmethod def _tm(time): # ## strftime it too slow for large time serializer : - # return datetime.datetime.fromtimestamp(time).strftime("%Y-%m-%d %H:%M:%S") + # return MyTime.time2str(time) c = _tmSerial sec = (time % 60) if c._last_s == time - sec: @@ -143,7 +141,7 @@ def _ticket_tuple(ticket): """ attempts = ticket.getAttempt() date = ticket.getTime() - ip = ticket.getIP() + ip = ticket.getID() matches = ticket.getMatches() return (ip, attempts, date, matches) @@ -166,23 +164,31 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): # get fail ticket from jail found.append(_ticket_tuple(filter_.getFailTicket())) else: - # when we are testing without jails - # wait for failures (up to max time) - Utils.wait_for( - lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), - _maxWaitTime(10)) - # get fail ticket(s) from filter - while tickcount: - try: - found.append(_ticket_tuple(filter_.failManager.toBan())) - except FailManagerEmpty: - break - tickcount -= 1 + # when we are testing without jails wait for failures (up to max time) + if filter_.jail: + while True: + t = filter_.jail.getFailTicket() + if not t: break + found.append(_ticket_tuple(t)) + if found: + tickcount -= len(found) + if tickcount > 0: + Utils.wait_for( + lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), + _maxWaitTime(10)) + # get fail ticket(s) from filter + while tickcount: + try: + found.append(_ticket_tuple(filter_.failManager.toBan())) + except FailManagerEmpty: + break + tickcount -= 1 if not isinstance(output[0], (tuple,list)): utest.assertEqual(len(found), 1) _assert_equal_entries(utest, found[0], output, count) else: + utest.assertEqual(len(found), len(output)) # sort by string representation of ip (multiple failures with different ips): found = sorted(found, key=lambda x: str(x)) output = sorted(output, key=lambda x: str(x)) @@ -190,7 +196,7 @@ def _assert_correct_last_attempt(utest, filter_, output, count=None): _assert_equal_entries(utest, f, o) -def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line=""): +def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line="", lines=None): """Copy lines from one file to another (which might be already open) Returns open fout @@ -199,7 +205,7 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # polling filter could detect the change mtimesleep() if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Skip @@ -207,9 +213,9 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line fin.readline() # Read i = 0 - lines = [] + if not lines: lines = [] while n is None or i < n: - l = fin.readline() + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break lines.append(l) @@ -217,7 +223,8 @@ def _copy_lines_between_files(in_, fout, n=None, skip=0, mode='a', terminal_line # Write: all at once and flush if isinstance(fout, str): fout = open(fout, mode) - fout.write('\n'.join(lines)) + DefLogSys.debug(' ++ write %d test lines', len(lines)) + fout.write('\n'.join(lines)+'\n') fout.flush() if isinstance(in_, str): # pragma: no branch - only used with str in test cases # Opened earlier, therefore must close it @@ -237,7 +244,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p Returns None """ if isinstance(in_, str): # pragma: no branch - only used with str in test cases - fin = open(in_, 'r') + fin = open(in_, 'rb') else: fin = in_ # Required for filtering @@ -248,7 +255,7 @@ def _copy_lines_to_journal(in_, fields={},n=None, skip=0, terminal_line=""): # p # Read/Write i = 0 while n is None or i < n: - l = fin.readline() + l = fin.readline().decode('UTF-8', 'replace').rstrip('\r\n') if terminal_line is not None and l == terminal_line: break journal.send(MESSAGE=l.strip(), **fields) @@ -266,7 +273,7 @@ class BasicFilter(unittest.TestCase): def setUp(self): super(BasicFilter, self).setUp() - self.filter = Filter('name') + self.filter = Filter(None) def testGetSetUseDNS(self): # default is warn @@ -279,10 +286,10 @@ class BasicFilter(unittest.TestCase): def testGetSetDatePattern(self): self.assertEqual(self.filter.getDatePattern(), (None, "Default Detectors")) - self.filter.setDatePattern("^%Y-%m-%d-%H%M%S.%f %z **") + self.filter.setDatePattern(r"^%Y-%m-%d-%H%M%S\.%f %z **") self.assertEqual(self.filter.getDatePattern(), - ("^%Y-%m-%d-%H%M%S.%f %z **", - "^Year-Month-Day-24hourMinuteSecond.Microseconds Zone offset **")) + (r"^%Y-%m-%d-%H%M%S\.%f %z **", + r"^Year-Month-Day-24hourMinuteSecond\.Microseconds Zone offset **")) def testGetSetLogTimeZone(self): self.assertEqual(self.filter.getLogTimeZone(), None) @@ -306,7 +313,7 @@ class BasicFilter(unittest.TestCase): unittest.F2B.SkipIfFast() ## test function "_tm" works correct (returns the same as slow strftime): for i in xrange(1417512352, (1417512352 // 3600 + 3) * 3600): - tm = datetime.datetime.fromtimestamp(i).strftime("%Y-%m-%d %H:%M:%S") + tm = MyTime.time2str(i) if _tm(i) != tm: # pragma: no cover - never reachable self.assertEqual((_tm(i), i), (tm, i)) @@ -387,18 +394,127 @@ class IgnoreIP(LogCaptureTestCase): def testIgnoreInProcessLine(self): setUpMyTime() - self.filter.addIgnoreIP('192.168.1.0/25') - self.filter.addFailRegex('<HOST>') - self.filter.setDatePattern('{^LN-BEG}EPOCH') - self.filter.processLineAndAdd('1387203300.222 192.168.1.32') - self.assertLogged('Ignore 192.168.1.32') - tearDownMyTime() + try: + self.filter.addIgnoreIP('192.168.1.0/25') + self.filter.addFailRegex('<HOST>') + self.filter.setDatePattern(r'{^LN-BEG}EPOCH') + self.filter.processLineAndAdd('1387203300.222 192.168.1.32') + self.assertLogged('Ignore 192.168.1.32') + finally: + tearDownMyTime() - def testIgnoreAddBannedIP(self): - self.filter.addIgnoreIP('192.168.1.0/25') - self.filter.addBannedIP('192.168.1.32') - self.assertNotLogged('Ignore 192.168.1.32') - self.assertLogged('Requested to manually ban an ignored IP 192.168.1.32. User knows best. Proceeding to ban it.') + def _testTimeJump(self, inOperation=False): + try: + self.filter.addFailRegex('^<HOST>') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setFindTime(10); # max 10 seconds back + self.filter.setMaxRetry(5); # don't ban here + self.filter.inOperation = inOperation + # + self.pruneLog('[phase 1] DST time jump') + # check local time jump (DST hole): + MyTime.setTime(1572137999) + self.filter.processLineAndAdd('2019-10-27 02:59:59 192.0.2.5'); # +1 = 1 + MyTime.setTime(1572138000) + self.filter.processLineAndAdd('2019-10-27 02:00:00 192.0.2.5'); # +1 = 2 + MyTime.setTime(1572138001) + self.filter.processLineAndAdd('2019-10-27 02:00:01 192.0.2.5'); # +1 = 3 + self.assertLogged( + 'Current failures from 1 IPs (IP:count): 192.0.2.5:1', + 'Current failures from 1 IPs (IP:count): 192.0.2.5:2', + 'Current failures from 1 IPs (IP:count): 192.0.2.5:3', + "Total # of detected failures: 3.", all=True, wait=True) + self.assertNotLogged('Ignore line') + # + self.pruneLog('[phase 2] UTC time jump (NTP correction)') + # check time drifting backwards (NTP correction): + MyTime.setTime(1572210000) + self.filter.processLineAndAdd('2019-10-27 22:00:00 CET 192.0.2.6'); # +1 = 1 + MyTime.setTime(1572200000) + self.filter.processLineAndAdd('2019-10-27 22:00:01 CET 192.0.2.6'); # +1 = 2 (logged before correction) + self.filter.processLineAndAdd('2019-10-27 19:13:20 CET 192.0.2.6'); # +1 = 3 (logged after correction) + self.filter.processLineAndAdd('2019-10-27 19:13:21 CET 192.0.2.6'); # +1 = 4 + self.assertLogged( + '192.0.2.6:1', '192.0.2.6:2', '192.0.2.6:3', '192.0.2.6:4', + "Total # of detected failures: 7.", all=True, wait=True) + self.assertNotLogged('Ignore line') + finally: + tearDownMyTime() + def testTimeJump(self): + self._testTimeJump(inOperation=False) + def testTimeJump_InOperation(self): + self._testTimeJump(inOperation=True) + + def testWrongTimeOrTZ(self): + try: + self.filter.addFailRegex('fail from <ADDR>$') + self.filter.setDatePattern(r'{^LN-BEG}%Y-%m-%d %H:%M:%S(?:\s*%Z)?\s') + self.filter.setMaxRetry(50); # don't ban here + self.filter.inOperation = True; # real processing (all messages are new) + # current time is 1h later than log-entries: + MyTime.setTime(1572138000+3600) + # + self.pruneLog("[phase 1] simulate wrong TZ") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); # +3 = 3 + self.assertLogged( + "Detected a log entry 1h before the current time in operation mode. This looks like a timezone problem.", + "Please check a jail for a timing issue.", + "192.0.2.15:1", "192.0.2.15:2", "192.0.2.15:3", + "Total # of detected failures: 3.", all=True, wait=True) + # + setattr(self.filter, "_next_simByTimeWarn", -1) + self.pruneLog("[phase 2] wrong TZ given in log") + for i in (1,2,3): + self.filter.processLineAndAdd('2019-10-27 04:00:00 GMT fail from 192.0.2.16'); # +3 = 6 + self.assertLogged( + "Detected a log entry 2h after the current time in operation mode. This looks like a timezone problem.", + "Please check a jail for a timing issue.", + "192.0.2.16:1", "192.0.2.16:2", "192.0.2.16:3", + "Total # of detected failures: 6.", all=True, wait=True) + self.assertNotLogged("Found a match but no valid date/time found") + # + self.pruneLog("[phase 3] other timestamp (don't match datepattern), regex matches") + for i in range(3): + self.filter.processLineAndAdd('27.10.2019 04:00:00 fail from 192.0.2.17'); # +3 = 9 + self.assertLogged( + "Found a match but no valid date/time found", + "Match without a timestamp:", + "192.0.2.17:1", "192.0.2.17:2", "192.0.2.17:3", + "Total # of detected failures: 9.", all=True, wait=True) + # + phase = 3 + for delta, expect in ( + (-90*60, "timezone"), #90 minutes after + (-60*60, "timezone"), #60 minutes after + (-10*60, "timezone"), #10 minutes after + (-59, None), #59 seconds after + (59, None), #59 seconds before + (61, "latency"), #>1 minute before + (55*60, "latency"), #55 minutes before + (90*60, "timezone") #90 minutes before + ): + phase += 1 + MyTime.setTime(1572138000+delta) + setattr(self.filter, "_next_simByTimeWarn", -1) + self.pruneLog('[phase {phase}] log entries offset by {delta}s'.format(phase=phase, delta=delta)) + self.filter.processLineAndAdd('2019-10-27 02:00:00 fail from 192.0.2.15'); + self.assertLogged("Found 192.0.2.15", wait=True) + if expect: + self.assertLogged(("timezone problem", "latency problem")[int(expect == "latency")], all=True) + self.assertNotLogged(("timezone problem", "latency problem")[int(expect != "latency")], all=True) + else: + self.assertNotLogged("timezone problem", "latency problem", all=True) + finally: + tearDownMyTime() + + def testAddAttempt(self): + self.filter.setMaxRetry(3) + for i in xrange(1, 1+3): + self.filter.addAttempt('192.0.2.1') + self.assertLogged('Attempt 192.0.2.1', '192.0.2.1:%d' % i, all=True, wait=True) + self.jail.actions._Actions__checkBan() + self.assertLogged('Ban 192.0.2.1', wait=True) def testIgnoreCommand(self): self.filter.ignoreCommand = sys.executable + ' ' + os.path.join(TEST_FILES_DIR, "ignorecommand.py <ip>") @@ -518,13 +634,14 @@ class IgnoreIPDNS(LogCaptureTestCase): cmd = os.path.join(STOCK_CONF_DIR, "filter.d/ignorecommands/apache-fakegooglebot") ## below test direct as python module: mod = Utils.load_python_module(cmd) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "128.178.222.69"]))) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "128.178.222.69"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1", 0.1]))) bot_ips = ['66.249.66.1'] for ip in bot_ips: - self.assertTrue(mod.is_googlebot(mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd]))) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"]))) + self.assertTrue(mod.is_googlebot(*mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd]))) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd, "192.0"]))) ## via command: self.filter.ignoreCommand = cmd + " <ip>" for ip in bot_ips: @@ -536,7 +653,7 @@ class IgnoreIPDNS(LogCaptureTestCase): self.pruneLog() self.filter.ignoreCommand = cmd + " bad arguments <ip>" self.assertFalse(self.filter.inIgnoreIPList("192.0")) - self.assertLogged('Please provide a single IP as an argument.') + self.assertLogged('Usage') @@ -554,6 +671,19 @@ class LogFile(LogCaptureTestCase): self.filter = FilterPoll(None) self.assertRaises(IOError, self.filter.addLogPath, LogFile.MISSING) + def testDecodeLineWarn(self): + # incomplete line (missing byte at end), warning is suppressed: + l = u"correct line\n" + r = l.encode('utf-16le') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r), l) + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-16le', r[0:-1]), l[0:-1]) + self.assertNotLogged('Error decoding line') + # complete line (incorrect surrogate in the middle), warning is there: + r = b"incorrect \xc8\x0a line\n" + l = r.decode('utf-8', 'replace') + self.assertEqual(FileContainer.decode_line('TESTFILE', 'utf-8', r), l) + self.assertLogged('Error decoding line') + class LogFileFilterPoll(unittest.TestCase): @@ -578,7 +708,7 @@ class LogFileFilterPoll(unittest.TestCase): def testSeekToTimeSmallFile(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') @@ -664,7 +794,7 @@ class LogFileFilterPoll(unittest.TestCase): def testSeekToTimeLargeFile(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='.log') time = 1417512352 f = open(fname, 'w') @@ -721,7 +851,7 @@ class LogFileMonitor(LogCaptureTestCase): self.filter = FilterPoll(DummyJail()) self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") def tearDown(self): tearDownMyTime() @@ -763,7 +893,7 @@ class LogFileMonitor(LogCaptureTestCase): def testErrorProcessLine(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.filter.sleeptime /= 1000.0 ## produce error with not callable processLine: _org_processLine = self.filter.processLine @@ -827,7 +957,7 @@ class LogFileMonitor(LogCaptureTestCase): def testNewChangeViaGetFailures_simple(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # suck in lines from this sample log file self.filter.getFailures(self.name) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) @@ -838,13 +968,13 @@ class LogFileMonitor(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.filter.getFailures(self.name) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_rewrite(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we rewrite the file at once self.file.close() @@ -857,14 +987,14 @@ class LogFileMonitor(LogCaptureTestCase): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.filter.getFailures(self.name) #self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) def testNewChangeViaGetFailures_move(self): # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # # if we move file into a new location while it has been open already self.file.close() @@ -876,9 +1006,9 @@ class LogFileMonitor(LogCaptureTestCase): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) - _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) + #_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -889,6 +1019,10 @@ class CommonMonitorTestCase(unittest.TestCase): super(CommonMonitorTestCase, self).setUp() self._failTotal = 0 + def tearDown(self): + super(CommonMonitorTestCase, self).tearDown() + self.assertFalse(hasattr(self, "_unexpectedError")) + def waitFailTotal(self, count, delay=1): """Wait up to `delay` sec to assure that expected failure `count` reached """ @@ -914,6 +1048,16 @@ class CommonMonitorTestCase(unittest.TestCase): last_ticks = self.filter.ticks return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay)) + def commonFltError(self, reason="common", exc=None): + """ Mock-up for default common error handler to find catched unhandled exceptions + could occur in filters + """ + self._commonFltError(reason, exc) + if reason == "unhandled": + DefLogSys.critical("Caught unhandled exception in main cycle of %r : %r", self.filter, exc, exc_info=True) + self._unexpectedError = True + # self.assertNotEqual(reason, "unhandled") + def get_monitor_failures_testcase(Filter_): """Generator of TestCase's for different filters/backends @@ -936,11 +1080,13 @@ def get_monitor_failures_testcase(Filter_): self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') self.filter.active = True - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") self.filter.start() # If filter is polling it would sleep a bit to guarantee that # we have initial time-stamp difference to trigger "actions" @@ -986,13 +1132,13 @@ def get_monitor_failures_testcase(Filter_): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=12) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # and our dummy jail is empty as well self.assertFalse(len(self.jail)) # since it should have not been enough - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=5) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) if idle: self.waitForTicks(1) self.assertTrue(self.isEmpty(1)) @@ -1011,7 +1157,7 @@ def get_monitor_failures_testcase(Filter_): #return # just for fun let's copy all of them again and see if that results # in a new ban - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) self.assert_correct_last_attempt(GetFailures.FAILURES_01) def test_rewrite_file(self): @@ -1025,15 +1171,16 @@ def get_monitor_failures_testcase(Filter_): # filter "marked" as the known beginning, otherwise it # would not detect "rotation" self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=3, mode='w') + skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def _wait4failures(self, count=2): + def _wait4failures(self, count=2, waitEmpty=True): # Poll might need more time - self.assertTrue(self.isEmpty(_maxWaitTime(5)), - "Queue must be empty but it is not: %s." - % (', '.join([str(x) for x in self.jail.queue]))) - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + if waitEmpty: + self.assertTrue(self.isEmpty(_maxWaitTime(5)), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), count) @@ -1046,13 +1193,15 @@ def get_monitor_failures_testcase(Filter_): # move aside, but leaving the handle still open... os.rename(self.name, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1, + lines=["Aug 14 11:59:59 [logrotate] rotation 1"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) # now remove the moved file _killfile(None, self.name + '.bak') - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3, + lines=["Aug 14 11:59:59 [logrotate] rotation 2"]).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) @@ -1106,7 +1255,7 @@ def get_monitor_failures_testcase(Filter_): os.rename(tmpsub1, tmpsub2 + 'a') os.mkdir(tmpsub1) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 1"]) self.file.close() self._wait4failures(2) @@ -1117,7 +1266,7 @@ def get_monitor_failures_testcase(Filter_): os.mkdir(tmpsub1) self.waitForTicks(2) self.file = _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - skip=12, n=1, mode='w') + skip=12, n=1, mode='w', lines=["Aug 14 11:59:59 [logrotate] rotation 2"]) self.file.close() self._wait4failures(3) @@ -1128,8 +1277,7 @@ def get_monitor_failures_testcase(Filter_): def _test_move_into_file(self, interim_kill=False): # if we move a new file into the location of an old (monitored) file - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name).close() # make sure that it is monitored first self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -1140,14 +1288,14 @@ def get_monitor_failures_testcase(Filter_): # now create a new one to override old one _copy_lines_between_files(GetFailures.FILENAME_01, self.name + '.new', - n=100).close() + skip=12, n=3).close() os.rename(self.name + '.new', self.name) self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) # and to make sure that it now monitored for changes _copy_lines_between_files(GetFailures.FILENAME_01, self.name, - n=100).close() + skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 9) @@ -1166,7 +1314,7 @@ def get_monitor_failures_testcase(Filter_): # create a bogus file in the same directory and see if that doesn't affect open(self.name + '.bak2', 'w').close() - _copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100).close() + _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=12, n=3).close() self.assert_correct_last_attempt(GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 6) _killfile(None, self.name + '.bak2') @@ -1195,14 +1343,14 @@ def get_monitor_failures_testcase(Filter_): # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) # we should detect the failures - self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above # now copy and get even more - _copy_lines_between_files(GetFailures.FILENAME_01, self.file, n=100) - # check for 3 failures (not 9), because 6 already get above... - self.assert_correct_last_attempt(GetFailures.FAILURES_01) + _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) + # check for 3 failures (not 9), because 6 already get above... + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 12) + self._wait4failures(12, False) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1220,7 +1368,6 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def setUp(self): """Call before every test case.""" super(MonitorJournalFailures, self).setUp() - self._runtimeJournal = None self.test_file = os.path.join(TEST_FILES_DIR, "testcase-journal.log") self.jail = DummyJail() self.filter = None @@ -1234,6 +1381,8 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1242,7 +1391,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=2", "TEST_UUID=%s" % self.test_uuid]) - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>") def tearDown(self): if self.filter and self.filter.active: @@ -1256,21 +1405,26 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover If not found, SkipTest exception will be raised. """ # we can cache it: - if self._runtimeJournal is None: + if not hasattr(MonitorJournalFailures, "_runtimeJournal"): # Depending on the system, it could be found under /run or /var/log (e.g. Debian) # which are pointed by different systemd-path variables. We will # check one at at time until the first hit for systemd_var in 'system-runtime-logs', 'system-state-logs': tmp = Utils.executeCmd( - 'find "$(systemd-path %s)" -name system.journal' % systemd_var, + 'find "$(systemd-path %s)/journal" -name system.journal -readable' % systemd_var, timeout=10, shell=True, output=True ) self.assertTrue(tmp) out = str(tmp[1].decode('utf-8')).split('\n')[0] if out: break - self._runtimeJournal = out - if self._runtimeJournal: - return self._runtimeJournal + # additional check appropriate default settings (if not root/sudoer and not already set): + if os.geteuid() != 0 and os.getenv("F2B_SYSTEMD_DEFAULT_FLAGS", None) is None: + # filter default SYSTEM_ONLY(4) is hardly usable for not root/sudoer tester, + # so back to default LOCAL_ONLY(1): + os.environ["F2B_SYSTEMD_DEFAULT_FLAGS"] = "0"; # or "1", what will be similar to journalflags=0 or ...=1 + MonitorJournalFailures._runtimeJournal = out + if MonitorJournalFailures._runtimeJournal: + return MonitorJournalFailures._runtimeJournal raise unittest.SkipTest('systemd journal seems to be not available (e. g. no rights to read)') def testJournalFilesArg(self): @@ -1278,6 +1432,11 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover jrnlfile = self._getRuntimeJournal() self._initFilter(journalfiles=jrnlfile) + def testJournalFilesAndFlagsArgs(self): + # retrieve current system journal path + jrnlfile = self._getRuntimeJournal() + self._initFilter(journalfiles=jrnlfile, journalflags=0) + def testJournalPathArg(self): # retrieve current system journal path jrnlpath = self._getRuntimeJournal() @@ -1301,7 +1460,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.assertTrue(ticket) attempts = ticket.getAttempt() - ip = ticket.getIP() + ip = ticket.getID() ticket.getMatches() self.assertEqual(ip, test_ip) @@ -1319,7 +1478,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover if idle: self.filter.sleeptime /= 100.0 self.filter.idle = True - self.waitForTicks(1) + self.waitForTicks(1) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) # Now let's feed it with entries from the file @@ -1352,9 +1511,87 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.test_file, self.journal_fields, skip=5, n=4) self.assert_correct_ban("193.168.0.128", 3) + @with_alt_time + def test_grow_file_with_db(self): + + def _gen_falure(ip): + # insert new failures ans check it is monitored: + fields = self.journal_fields + fields.update(TEST_JOURNAL_FIELDS) + journal.send(MESSAGE="error: PAM: Authentication failure for test from "+ip, **fields) + self.waitForTicks(1) + self.assert_correct_ban(ip, 1) + + # coverage for update log: + self.jail.database = getFail2BanDb(':memory:') + self.jail.database.addJail(self.jail) + MyTime.setTime(time.time()) + self._test_grow_file() + # stop: + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 10) + # update log manually (should cause a seek to end of log without wait for next second): + self.jail.database.updateJournal(self.jail, 'systemd-journal', MyTime.time(), 'TEST') + # check seek to last (simulated) position succeeds (without bans of previous copied tickets): + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(2) + # check new IP but no old IPs found: + _gen_falure("192.0.2.5") + self.assertFalse(self.jail.getFailTicket()) + + # now the same with increased time (check now - findtime case): + self.filter.stop() + self.filter.join() + MyTime.setTime(time.time() + 10000) + self._failTotal = 0 + self._initFilter() + self.filter.setMaxRetry(1) + self.filter.start() + self.waitForTicks(2) + MyTime.setTime(time.time() + 20) + # check new IP but no old IPs found: + _gen_falure("192.0.2.6") + self.assertFalse(self.jail.getFailTicket()) + + # now reset DB, so we'd find all messages before filter entering in operation mode: + self.filter.stop() + self.filter.join() + self.jail.database.updateJournal(self.jail, 'systemd-journal', MyTime.time()-10000, 'TEST') + self._initFilter() + self.filter.setMaxRetry(1) + states = [] + def _state(*args): + try: + self.assertNotIn("** in operation", states) + self.assertFalse(self.filter.inOperation) + states.append("** process line: %r" % (args,)) + except Exception as e: + states.append("** failed: %r" % (e,)) + raise + self.filter.processLineAndAdd = _state + def _inoper(): + try: + self.assertNotIn("** in operation", states) + self.assertEqual(len(states), 11) + states.append("** in operation") + self.filter.__class__.inOperationMode(self.filter) + except Exception as e: + states.append("** failed: %r" % (e,)) + raise + self.filter.inOperationMode = _inoper + self.filter.start() + self.waitForTicks(12) + self.assertTrue(Utils.wait_for(lambda: len(states) == 12, _maxWaitTime(10))) + self.assertEqual(states[-1], "** in operation") + def test_delJournalMatch(self): self._initFilter() self.filter.start() + self.waitForTicks(1); # wait for start # Smoke test for removing of match # basic full test @@ -1378,7 +1615,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", "TEST_UUID=%s" % self.test_uuid]) - self.assert_correct_ban("193.168.0.128", 4) + self.assert_correct_ban("193.168.0.128", 3) _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures @@ -1387,12 +1624,13 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover def test_WrongChar(self): self._initFilter() self.filter.start() + self.waitForTicks(1); # wait for start # Now let's feed it with entries from the file _copy_lines_to_journal( self.test_file, self.journal_fields, skip=15, n=4) self.waitForTicks(1) self.assertTrue(self.isFilled(10)) - self.assert_correct_ban("87.142.124.10", 4) + self.assert_correct_ban("87.142.124.10", 3) # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", @@ -1408,7 +1646,7 @@ def get_monitor_failures_journal_testcase(Filter_): # pragma: systemd no cover self.waitForTicks(1) self.waitFailTotal(6, 10) self.assertTrue(Utils.wait_for(lambda: len(self.jail) == 2, 10)) - self.assertSortedEqual([self.jail.getFailTicket().getIP(), self.jail.getFailTicket().getIP()], + self.assertSortedEqual([self.jail.getFailTicket().getID(), self.jail.getFailTicket().getID()], ["192.0.2.1", "192.0.2.2"]) cls = MonitorJournalFailures @@ -1438,7 +1676,7 @@ class GetFailures(LogCaptureTestCase): self.filter = FileFilter(self.jail) self.filter.active = True # speedup search using exact date pattern: - self.filter.setDatePattern('^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') + self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') # TODO Test this #self.filter.setTimeRegex("\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") #self.filter.setTimePattern("%b %d %H:%M:%S") @@ -1483,81 +1721,128 @@ class GetFailures(LogCaptureTestCase): failures = failures or GetFailures.FAILURES_01 self.filter.addLogPath(filename, autoSeek=0) - self.filter.addFailRegex("(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>$") + self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) <HOST>$") self.filter.getFailures(filename) _assert_correct_last_attempt(self, self.filter, failures) def testCRLFFailures01(self): # We first adjust logfile/failures to end with CR+LF fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') - # poor man unix2dos: - fin, fout = open(GetFailures.FILENAME_01), open(fname, 'w') - for l in fin.readlines(): - fout.write('%s\r\n' % l.rstrip('\n')) - fin.close() - fout.close() + try: + # poor man unix2dos: + fin, fout = open(GetFailures.FILENAME_01, 'rb'), open(fname, 'wb') + for l in fin.read().splitlines(): + fout.write(l + b'\r\n') + fin.close() + fout.close() - # now see if we should be getting the "same" failures - self.testGetFailures01(filename=fname) - _killfile(fout, fname) + # now see if we should be getting the "same" failures + self.testGetFailures01(filename=fname) + finally: + _killfile(fout, fname) + + def testNLCharAsPartOfUniChar(self): + fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='uni') + # test two multi-byte encodings (both contains `\x0A` in either \x02\x0A or \x0A\x02): + for enc in ('utf-16be', 'utf-16le'): + self.pruneLog("[test-phase encoding=%s]" % enc) + try: + fout = open(fname, 'wb') + tm = int(time.time()) + # test on unicode string containing \x0A as part of uni-char, + # it must produce exactly 2 lines (both are failures): + for l in ( + u'%s \u20AC Failed auth: invalid user Test\u020A from 192.0.2.1\n' % tm, + u'%s \u20AC Failed auth: invalid user TestI from 192.0.2.2\n' % tm + ): + fout.write(l.encode(enc)) + fout.close() + + self.filter.setLogEncoding(enc) + self.filter.addLogPath(fname, autoSeek=0) + self.filter.setDatePattern((r'^EPOCH',)) + self.filter.addFailRegex(r"Failed .* from <HOST>") + self.filter.getFailures(fname) + self.assertLogged( + "[DummyJail] Found 192.0.2.1", + "[DummyJail] Found 192.0.2.2", all=True, wait=True) + finally: + _killfile(fout, fname) + self.filter.delLogPath(fname) + # must find 4 failures and generate 2 tickets (2 IPs with each 2 failures): + self.assertEqual(self.filter.failManager.getFailCount(), (2, 4)) def testGetFailures02(self): output = ('141.3.81.106', 4, 1124013539.0, [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) + self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) - self.filter.addFailRegex("Failed .* from <HOST>") + self.filter.addFailRegex(r"Failed .* from <HOST>") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03(self): - output = ('203.162.223.135', 7, 1124013544.0) + output = ('203.162.223.135', 6, 1124013600.0) + self.filter.setMaxRetry(6) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) - self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=<HOST>,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_InOperation(self): + output = ('203.162.223.135', 9, 1124013600.0) + + self.filter.setMaxRetry(9) + self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) + self.filter.addFailRegex(r"error,relay=<HOST>,.*550 User unknown") + self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) + _assert_correct_last_attempt(self, self.filter, output) + def testGetFailures03_Seek1(self): # same test as above but with seek to 'Aug 14 11:55:04' - so other output ... - output = ('203.162.223.135', 5, 1124013544.0) + output = ('203.162.223.135', 3, 1124013600.0) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2] - 4*60) - self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=<HOST>,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... - output = ('203.162.223.135', 1, 1124013544.0) - self.filter.setMaxRetry(1) + output = ('203.162.223.135', 2, 1124013600.0) + self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) - self.filter.addFailRegex("error,relay=<HOST>,.*550 User unknown") + self.filter.addFailRegex(r"error,relay=<HOST>,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) _assert_correct_last_attempt(self, self.filter, output) def testGetFailures04(self): # because of not exact time in testcase04.log (no year), we should always use our test time: self.assertEqual(MyTime.time(), 1124013600) - # should find exact 4 failures for *.186 and 2 failures for *.185 - output = (('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 2, 1124013598.0)) - + # should find exact 4 failures for *.186 and 2 failures for *.185, but maxretry is 2, so 3 tickets: + output = ( + ('212.41.96.186', 2, 1124013480.0), + ('212.41.96.186', 2, 1124013600.0), + ('212.41.96.185', 2, 1124013598.0) + ) # speedup search using exact date pattern: - self.filter.setDatePattern(('^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', - '^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', - '^EPOCH' + self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', + r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', + r'^EPOCH' )) self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_04, autoSeek=0) - self.filter.addFailRegex("Invalid user .* <HOST>") + self.filter.addFailRegex(r"Invalid user .* <HOST>") self.filter.getFailures(GetFailures.FILENAME_04) _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresWrongChar(self): + self.filter.checkFindTime = False # write wrong utf-8 char: fname = tempfile.mktemp(prefix='tmp_fail2ban', suffix='crlf') fout = fopen(fname, 'wb') @@ -1572,15 +1857,17 @@ class GetFailures(LogCaptureTestCase): fout.close() # output = ('192.0.2.0', 3, 1421262060.0) - failregex = "^\s*user \"[^\"]*\" from \"<HOST>\"\s*$" + failregex = r"^\s*user \"[^\"]*\" from \"<HOST>\"\s*$" # test encoding auto or direct set of encoding: for enc in (None, 'utf-8', 'ascii'): if enc is not None: self.tearDown();self.setUp(); + if DefLogSys.getEffectiveLevel() > 7: DefLogSys.setLevel(7); # ensure decode_line logs always + self.filter.checkFindTime = False; self.filter.setLogEncoding(enc); # speedup search using exact date pattern: - self.filter.setDatePattern('^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') + self.filter.setDatePattern(r'^%ExY-%Exm-%Exd %ExH:%ExM:%ExS') self.assertNotLogged('Error decoding line'); self.filter.addLogPath(fname) self.filter.addFailRegex(failregex) @@ -1598,9 +1885,11 @@ class GetFailures(LogCaptureTestCase): unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ( - ('93.184.216.34', 2, 1124013539.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ('93.184.216.34', 1, 1124013299.0, + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ), + ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] @@ -1629,23 +1918,26 @@ class GetFailures(LogCaptureTestCase): filter_.failManager.setMaxRetry(1) # we might have just few failures filter_.addLogPath(GetFailures.FILENAME_USEDNS, autoSeek=False) - filter_.addFailRegex("Failed .* from <HOST>") + filter_.addFailRegex(r"Failed .* from <HOST>") filter_.getFailures(GetFailures.FILENAME_USEDNS) _assert_correct_last_attempt(self, filter_, output) def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = [ + ('141.3.81.106', 8, 1124013541.0) + ] + self.filter.setMaxRetry(8) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) - self.filter.addFailRegex("Failed .* from <HOST>") - self.filter.addFailRegex("Accepted .* from <HOST>") + self.filter.addFailRegex(r"Failed .* from <HOST>") + self.filter.addFailRegex(r"Accepted .* from <HOST>") self.filter.getFailures(GetFailures.FILENAME_02) _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresIgnoreRegex(self): self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) - self.filter.addFailRegex("Failed .* from <HOST>") - self.filter.addFailRegex("Accepted .* from <HOST>") + self.filter.addFailRegex(r"Failed .* from <HOST>") + self.filter.addFailRegex(r"Accepted .* from <HOST>") self.filter.addIgnoreRegex("for roehl") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1653,58 +1945,55 @@ class GetFailures(LogCaptureTestCase): self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + + _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.addIgnoreRegex("rsync error: Received SIGINT") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + ("192.0.43.15", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) - self.filter.addFailRegex("^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") - self.filter.addFailRegex("^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") + self.filter.addFailRegex(r"^.*rsyncd\[(?P<pid>\d+)\]: connect from .+ \(<HOST>\)$<SKIPLINES>^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") + self.filter.addFailRegex(r"^.* sendmail\[.*, msgid=<(?P<msgid>[^>]+).*relay=\[<HOST>\].*$<SKIPLINES>^.+ spamd: result: Y \d+ .*,mid=<(?P=msgid)>(,bayes=[.\d]+)?(,autolearn=\S+)?\s*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) class DNSUtilsTests(unittest.TestCase): @@ -1753,6 +2042,44 @@ class DNSUtilsTests(unittest.TestCase): # here the whole cache should be empty: self.assertEqual(len(c), 0) + def testOverflowedIPCache(self): + # test overflow of IP-cache multi-threaded (2 "parasite" threads flooding cache): + from threading import Thread + from random import shuffle + # save original cache and use smaller cache during the test here: + _org_cache = IPAddr.CACHE_OBJ + cache = IPAddr.CACHE_OBJ = Utils.Cache(maxCount=5, maxTime=60) + result = list() + count = 1 if unittest.F2B.fast else 50 + try: + # tester procedure of worker: + def _TestCacheStr2IP(forw=True, result=[], random=False): + try: + c = count + while c: + c -= 1 + s = xrange(0, 256, 1) if forw else xrange(255, -1, -1) + if random: shuffle([i for i in s]) + for i in s: + IPAddr('192.0.2.'+str(i), IPAddr.FAM_IPv4) + IPAddr('2001:db8::'+str(i), IPAddr.FAM_IPv6) + result.append(None) + except Exception as e: + DefLogSys.debug(e, exc_info=True) + result.append(e) + + # 2 workers flooding it forwards and backwards: + th1 = Thread(target=_TestCacheStr2IP, args=(True, result)); th1.start() + th2 = Thread(target=_TestCacheStr2IP, args=(False, result)); th2.start() + # and here we flooding it with random IPs too: + _TestCacheStr2IP(True, result, True) + finally: + # wait for end of threads and restore cache: + th1.join() + th2.join() + IPAddr.CACHE_OBJ = _org_cache + self.assertEqual(result, [None]*3) # no errors + self.assertTrue(len(cache) <= cache.maxCount) class DNSUtilsNetworkTests(unittest.TestCase): @@ -1760,17 +2087,24 @@ class DNSUtilsNetworkTests(unittest.TestCase): def setUp(self): """Call before every test case.""" super(DNSUtilsNetworkTests, self).setUp() - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() def test_IPAddr(self): - self.assertTrue(IPAddr('192.0.2.1').isIPv4) - self.assertTrue(IPAddr('2001:DB8::').isIPv6) + ip4 = IPAddr('192.0.2.1') + ip6 = IPAddr('2001:DB8::') + self.assertTrue(ip4.isIPv4) + self.assertTrue(ip4.isSingle) + self.assertTrue(ip6.isIPv6) + self.assertTrue(ip6.isSingle) + self.assertTrue(asip('192.0.2.1').isIPv4) + self.assertTrue(id(asip(ip4)) == id(ip4)) def test_IPAddr_Raw(self): # raw string: r = IPAddr('xxx', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, 'xxx') self.assertEqual('xxx', str(r)) @@ -1779,6 +2113,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): r = IPAddr('1:2', IPAddr.CIDR_RAW) self.assertFalse(r.isIPv4) self.assertFalse(r.isIPv6) + self.assertFalse(r.isSingle) self.assertTrue(r.isValid) self.assertEqual(r, '1:2') self.assertEqual('1:2', str(r)) @@ -1800,7 +2135,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testUseDns(self): res = DNSUtils.textToIp('www.example.com', 'no') - self.assertEqual(res, []) + self.assertSortedEqual(res, []) + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.textToIp('www.example.com', 'warn') # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) @@ -1809,6 +2145,7 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) def testTextToIp(self): + #unittest.F2B.SkipIfNoNetwork() # Test hostnames hostnames = [ 'www.example.com', @@ -1821,20 +2158,23 @@ class DNSUtilsNetworkTests(unittest.TestCase): # sort ipaddr, IPv4 is always smaller as IPv6 self.assertSortedEqual(res, ['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']) else: - self.assertEqual(res, []) + self.assertSortedEqual(res, []) + + def testIpToIp(self): # pure ips: for s in ('93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946'): ips = DNSUtils.textToIp(s, 'yes') - self.assertEqual(ips, [s]) - self.assertTrue(isinstance(ips[0], IPAddr)) + self.assertSortedEqual(ips, [s]) + for ip in ips: + self.assertTrue(isinstance(ip, IPAddr)) def testIpToName(self): - unittest.F2B.SkipIfNoNetwork() + #unittest.F2B.SkipIfNoNetwork() res = DNSUtils.ipToName('8.8.4.4') - self.assertEqual(res, 'google-public-dns-b.google.com') + self.assertTrue(res.endswith(('.google', '.google.com'))) # same as above, but with IPAddr: res = DNSUtils.ipToName(IPAddr('8.8.4.4')) - self.assertEqual(res, 'google-public-dns-b.google.com') + self.assertTrue(res.endswith(('.google', '.google.com'))) # invalid ip (TEST-NET-1 according to RFC 5737) res = DNSUtils.ipToName('192.0.2.0') self.assertEqual(res, None) @@ -1851,8 +2191,10 @@ class DNSUtilsNetworkTests(unittest.TestCase): self.assertEqual(res.addr, 167772160L) res = IPAddr('10.0.0.1', cidr=32L) self.assertEqual(res.addr, 167772161L) + self.assertTrue(res.isSingle) res = IPAddr('10.0.0.1', cidr=31L) self.assertEqual(res.addr, 167772160L) + self.assertFalse(res.isSingle) self.assertEqual(IPAddr('10.0.0.0').hexdump, '0a000000') self.assertEqual(IPAddr('1::2').hexdump, '00010000000000000000000000000002') @@ -1877,6 +2219,8 @@ class DNSUtilsNetworkTests(unittest.TestCase): def testIPAddr_InInet(self): ip4net = IPAddr('93.184.0.1/24') ip6net = IPAddr('2606:2800:220:1:248:1893:25c8:0/120') + self.assertFalse(ip4net.isSingle) + self.assertFalse(ip6net.isSingle) # ip4: self.assertTrue(IPAddr('93.184.0.1').isInNet(ip4net)) self.assertTrue(IPAddr('93.184.0.255').isInNet(ip4net)) @@ -1972,11 +2316,13 @@ class DNSUtilsNetworkTests(unittest.TestCase): ) def testIPAddr_CompareDNS(self): + #unittest.F2B.SkipIfNoNetwork() ips = IPAddr('example.com') self.assertTrue(IPAddr("93.184.216.34").isInNet(ips)) self.assertTrue(IPAddr("2606:2800:220:1:248:1893:25c8:1946").isInNet(ips)) def testIPAddr_wrongDNS_IP(self): + unittest.F2B.SkipIfNoNetwork() DNSUtils.dnsToIp('`this`.dns-is-wrong.`wrong-nic`-dummy') DNSUtils.ipToName('*') @@ -1987,6 +2333,69 @@ class DNSUtilsNetworkTests(unittest.TestCase): ip1 = IPAddr('93.184.216.34'); ip2 = IPAddr('93.184.216.34'); self.assertEqual(id(ip1), id(ip2)) ip1 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); ip2 = IPAddr('2606:2800:220:1:248:1893:25c8:1946'); self.assertEqual(id(ip1), id(ip2)) + def test_NetworkInterfacesAddrs(self): + for withMask in (False, True): + try: + ips = IPAddrSet([a for ni, a in DNSUtils._NetworkInterfacesAddrs(withMask)]) + ip = IPAddr('127.0.0.1') + self.assertEqual(ip in ips, any(ip in n for n in ips)) + ip = IPAddr('::1') + self.assertEqual(ip in ips, any(ip in n for n in ips)) + except Exception as e: # pragma: no cover + # simply skip if not available, TODO: make coverage platform dependent + raise unittest.SkipTest(e) + + def test_IPAddrSet(self): + ips = IPAddrSet([IPAddr('192.0.2.1/27'), IPAddr('2001:DB8::/32')]) + self.assertTrue(IPAddr('192.0.2.1') in ips) + self.assertTrue(IPAddr('192.0.2.31') in ips) + self.assertFalse(IPAddr('192.0.2.32') in ips) + self.assertTrue(IPAddr('2001:DB8::1') in ips) + self.assertTrue(IPAddr('2001:0DB8:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF') in ips) + self.assertFalse(IPAddr('2001:DB9::') in ips) + # self IPs must be a set too (cover different mechanisms to obtain own IPs): + for cov in ('ni', 'dns', 'last'): + _org_NetworkInterfacesAddrs = None + if cov == 'dns': # mock-up _NetworkInterfacesAddrs like it's not implemented (raises error) + _org_NetworkInterfacesAddrs = DNSUtils._NetworkInterfacesAddrs + def _tmp_NetworkInterfacesAddrs(): + raise NotImplementedError() + DNSUtils._NetworkInterfacesAddrs = staticmethod(_tmp_NetworkInterfacesAddrs) + try: + ips = DNSUtils.getSelfIPs() + # print('*****', ips) + if ips: + ip = IPAddr('127.0.0.1') + self.assertEqual(ip in ips, any(ip in n for n in ips)) + ip = IPAddr('127.0.0.2') + self.assertEqual(ip in ips, any(ip in n for n in ips)) + ip = IPAddr('::1') + self.assertEqual(ip in ips, any(ip in n for n in ips)) + finally: + if _org_NetworkInterfacesAddrs: + DNSUtils._NetworkInterfacesAddrs = staticmethod(_org_NetworkInterfacesAddrs) + if cov != 'last': + DNSUtils.CACHE_nameToIp.unset(DNSUtils._getSelfIPs_key) + DNSUtils.CACHE_nameToIp.unset(DNSUtils._getNetIntrfIPs_key) + + def testFQDN(self): + unittest.F2B.SkipIfNoNetwork() + sname = DNSUtils.getHostname(fqdn=False) + lname = DNSUtils.getHostname(fqdn=True) + # FQDN is not localhost if short hostname is not localhost too (or vice versa): + self.assertEqual(lname != 'localhost', + sname != 'localhost') + # FQDN from short name should be long name: + self.assertEqual(getfqdn(sname), lname) + # FQDN from FQDN is the same: + self.assertEqual(getfqdn(lname), lname) + # coverage (targeting all branches): FQDN from loopback and DNS blackhole is always the same: + self.assertIn(getfqdn('localhost.'), ('localhost', 'localhost.')) + + def testFQDN_DNS(self): + unittest.F2B.SkipIfNoNetwork() + self.assertIn(getfqdn('as112.arpa.'), ('as112.arpa.', 'as112.arpa')) + class JailTests(unittest.TestCase): diff --git a/fail2ban/tests/misctestcase.py b/fail2ban/tests/misctestcase.py index cd27ad92..4b026377 100644 --- a/fail2ban/tests/misctestcase.py +++ b/fail2ban/tests/misctestcase.py @@ -34,7 +34,7 @@ from StringIO import StringIO from utils import LogCaptureTestCase, logSys as DefLogSys from ..helpers import formatExceptionInfo, mbasename, TraceBack, FormatterWithTraceBack, getLogger, \ - splitwords, uni_decode, uni_string + getVerbosityFormat, splitwords, uni_decode, uni_string from ..server.mytime import MyTime @@ -66,18 +66,14 @@ class HelpersTest(unittest.TestCase): self.assertEqual(splitwords(' 1, 2 , '), ['1', '2']) self.assertEqual(splitwords(' 1\n 2'), ['1', '2']) self.assertEqual(splitwords(' 1\n 2, 3'), ['1', '2', '3']) + # string as unicode: + self.assertEqual(splitwords(u' 1\n 2, 3'), ['1', '2', '3']) -if sys.version_info >= (2,7): - def _sh_call(cmd): - import subprocess - ret = subprocess.check_output(cmd, shell=True) - return uni_decode(ret).rstrip() -else: - def _sh_call(cmd): - import subprocess - ret = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read() - return uni_decode(ret).rstrip() +def _sh_call(cmd): + import subprocess + ret = subprocess.check_output(cmd, shell=True) + return uni_decode(ret).rstrip() def _getSysPythonVersion(): return _sh_call("fail2ban-python -c 'import sys; print(tuple(sys.version_info))'") @@ -90,7 +86,7 @@ class SetupTest(unittest.TestCase): unittest.F2B.SkipIfFast() setup = os.path.join(os.path.dirname(__file__), '..', '..', 'setup.py') self.setup = os.path.exists(setup) and setup or None - if not self.setup and sys.version_info >= (2,7): # pragma: no cover - running not out of the source + if not self.setup: # pragma: no cover - running not out of the source raise unittest.SkipTest( "Seems to be running not out of source distribution" " -- cannot locate setup.py") @@ -109,7 +105,7 @@ class SetupTest(unittest.TestCase): supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: # try dry-run: - os.system("%s %s --dry-run install --disable-2to3 --root=%s%s" + os.system("%s %s --dry-run install --root=%s%s" % (sys.executable, self.setup , tmp, supdbgout)) # check nothing was created: self.assertTrue(not os.listdir(tmp)) @@ -125,7 +121,7 @@ class SetupTest(unittest.TestCase): # suppress stdout (and stderr) if not heavydebug supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: - self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s" + self.assertEqual(os.system("%s %s install --root=%s%s" % (sys.executable, self.setup, tmp, supdbgout)), 0) def strippath(l): @@ -199,7 +195,8 @@ class TestsUtilsTest(LogCaptureTestCase): uni_decode((b'test\xcf' if sys.version_info >= (3,) else u'test\xcf')) uni_string(b'test\xcf') uni_string('test\xcf') - uni_string(u'test\xcf') + if sys.version_info < (3,) and 'PyPy' not in sys.version: + uni_string(u'test\xcf') def testSafeLogging(self): # logging should be exception-safe, to avoid possible errors (concat, str. conversion, representation failures, etc) @@ -388,12 +385,28 @@ class TestsUtilsTest(LogCaptureTestCase): self.assertSortedEqual(['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], level=-1) self.assertRaises(AssertionError, lambda: self.assertSortedEqual( - ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'])) + ['Z', {'A': ['B', 'C'], 'B': ['E', 'F']}], [{'B': ['F', 'E'], 'A': ['C', 'B']}, 'Z'], + nestedOnly=True)) + self.assertSortedEqual( + (0, [['A1'], ['A2', 'A1'], []]), + (0, [['A1'], ['A1', 'A2'], []]), + ) + self.assertSortedEqual(list('ABC'), list('CBA')) + self.assertRaises(AssertionError, self.assertSortedEqual, ['ABC'], ['CBA']) + self.assertRaises(AssertionError, self.assertSortedEqual, [['ABC']], [['CBA']]) self._testAssertionErrorRE(r"\['A'\] != \['C', 'B'\]", self.assertSortedEqual, ['A'], ['C', 'B']) self._testAssertionErrorRE(r"\['A', 'B'\] != \['B', 'C'\]", self.assertSortedEqual, ['A', 'B'], ['C', 'B']) + def testVerbosityFormat(self): + self.assertEqual(getVerbosityFormat(1), + '%(asctime)s %(name)-24s[%(process)d]: %(levelname)-7s %(message)s') + self.assertEqual(getVerbosityFormat(1, padding=False), + '%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s') + self.assertEqual(getVerbosityFormat(1, addtime=False, padding=False), + '%(name)s[%(process)d]: %(levelname)s %(message)s') + def testFormatterWithTraceBack(self): strout = StringIO() Formatter = FormatterWithTraceBack @@ -438,3 +451,18 @@ class MyTimeTest(unittest.TestCase): self.assertEqual(float(str2sec("1 month")) / 60 / 60 / 24, 30.4375) self.assertEqual(float(str2sec("1 year")) / 60 / 60 / 24, 365.25) + def testSec2Str(self): + sec2str = lambda s: str(MyTime.seconds2str(s)) + self.assertEqual(sec2str(86400*390), '1y 3w 4d') + self.assertEqual(sec2str(86400*368), '1y 3d') + self.assertEqual(sec2str(86400*365.49), '1y') + self.assertEqual(sec2str(86400*15), '2w 1d') + self.assertEqual(sec2str(86400*14-10), '2w') + self.assertEqual(sec2str(86400*2+3600*7+60*15), '2d 7h 15m') + self.assertEqual(sec2str(86400*2+3599), '2d 1h') + self.assertEqual(sec2str(3600*3.52), '3h 31m') + self.assertEqual(sec2str(3600*2-5), '2h') + self.assertEqual(sec2str(3600-5), '1h') + self.assertEqual(sec2str(3600-10), '59m 50s') + self.assertEqual(sec2str(59), '59s') + self.assertEqual(sec2str(0), '0') diff --git a/fail2ban/tests/observertestcase.py b/fail2ban/tests/observertestcase.py index f0016f1e..9b44c6dd 100644 --- a/fail2ban/tests/observertestcase.py +++ b/fail2ban/tests/observertestcase.py @@ -36,7 +36,6 @@ from ..server.failmanager import FailManager from ..server.observer import Observers, ObserverThread from ..server.utils import Utils from .utils import LogCaptureTestCase -from ..server.filter import Filter from .dummyjail import DummyJail from .databasetestcase import getFail2BanDb, Fail2BanDb @@ -177,13 +176,12 @@ class BanTimeIncr(LogCaptureTestCase): a.setBanTimeExtra('rndtime', None) -class BanTimeIncrDB(unittest.TestCase): -#class BanTimeIncrDB(LogCaptureTestCase): +class BanTimeIncrDB(LogCaptureTestCase): def setUp(self): """Call before every test case.""" super(BanTimeIncrDB, self).setUp() - if Fail2BanDb is None and sys.version_info >= (2,7): # pragma: no cover + if Fail2BanDb is None: # pragma: no cover raise unittest.SkipTest( "Unable to import fail2ban database module as sqlite is not " "available.") @@ -225,7 +223,7 @@ class BanTimeIncrDB(unittest.TestCase): jail.actions.setBanTime(10) jail.setBanTimeExtra('increment', 'true') jail.setBanTimeExtra('multipliers', '1 2 4 8 16 32 64 128 256 512 1024 2048') - ip = "127.0.0.2" + ip = "192.0.2.1" # used as start and fromtime (like now but time independence, cause test case can run slow): stime = int(MyTime.time()) ticket = FailTicket(ip, stime, []) @@ -369,14 +367,14 @@ class BanTimeIncrDB(unittest.TestCase): # this old ticket should be removed now: restored_tickets = self.db.getCurrentBans(fromtime=stime, correctBanTime=False) self.assertEqual(len(restored_tickets), 2) - self.assertEqual(restored_tickets[0].getIP(), ip) + self.assertEqual(restored_tickets[0].getID(), ip) # purge remove 1st ip self.db._purgeAge = -48*60*60 self.db.purge() restored_tickets = self.db.getCurrentBans(fromtime=stime, correctBanTime=False) self.assertEqual(len(restored_tickets), 1) - self.assertEqual(restored_tickets[0].getIP(), ip+'1') + self.assertEqual(restored_tickets[0].getID(), ip+'1') # this should purge all bans, bips and logs - nothing should be found now self.db._purgeAge = -240*60*60 @@ -386,10 +384,12 @@ class BanTimeIncrDB(unittest.TestCase): # two separate jails : jail1 = DummyJail(backend='polling') + jail1.filter.ignoreSelf = False jail1.setBanTimeExtra('increment', 'true') jail1.database = self.db self.db.addJail(jail1) jail2 = DummyJail(name='DummyJail-2', backend='polling') + jail2.filter.ignoreSelf = False jail2.database = self.db self.db.addJail(jail2) ticket1 = FailTicket(ip, stime, []) @@ -450,7 +450,8 @@ class BanTimeIncrDB(unittest.TestCase): def testObserver(self): if Fail2BanDb is None: # pragma: no cover return - jail = self.jail + jail = self.jail = DummyJail(backend='polling') + jail.database = self.db self.db.addJail(jail) # we tests with initial ban time = 10 seconds: jail.actions.setBanTime(10) @@ -465,6 +466,7 @@ class BanTimeIncrDB(unittest.TestCase): # purge database right now, but using timer, to test it also: self.db._purgeAge = -240*60*60 obs.add_named_timer('DB_PURGE', 0.001, 'db_purge') + self.assertLogged("Purge database event occurred", wait=True); # wait for purge timer # wait for timer ready obs.wait_idle(0.025) # wait for ready @@ -477,29 +479,29 @@ class BanTimeIncrDB(unittest.TestCase): self.assertEqual(tickets, []) # add failure: - ip = "127.0.0.2" + ip = "192.0.2.1" ticket = FailTicket(ip, stime-120, []) - failManager = FailManager() + failManager = jail.filter.failManager = FailManager() failManager.setMaxRetry(3) for i in xrange(3): failManager.addFailure(ticket) - obs.add('failureFound', failManager, jail, ticket) + obs.add('failureFound', jail, ticket) obs.wait_empty(5) self.assertEqual(ticket.getBanCount(), 0) # check still not ban : self.assertTrue(not jail.getFailTicket()) # add manually 4th times banned (added to bips - make ip bad): ticket.setBanCount(4) - self.db.addBan(self.jail, ticket) + self.db.addBan(jail, ticket) restored_tickets = self.db.getCurrentBans(jail=jail, fromtime=stime-120, correctBanTime=False) self.assertEqual(len(restored_tickets), 1) # check again, new ticket, new failmanager: ticket = FailTicket(ip, stime, []) - failManager = FailManager() + failManager = jail.filter.failManager = FailManager() failManager.setMaxRetry(3) # add once only - but bad - should be banned: failManager.addFailure(ticket) - obs.add('failureFound', failManager, self.jail, ticket) + obs.add('failureFound', jail, ticket) obs.wait_empty(5) # wait until ticket transfered from failmanager into jail: ticket2 = Utils.wait_for(jail.getFailTicket, 10) diff --git a/fail2ban/tests/samplestestcase.py b/fail2ban/tests/samplestestcase.py index 7f03b96a..b33b46c1 100644 --- a/fail2ban/tests/samplestestcase.py +++ b/fail2ban/tests/samplestestcase.py @@ -23,7 +23,6 @@ __copyright__ = "Copyright (c) 2013 Steven Hiscocks" __license__ = "GPL" import datetime -import fileinput import inspect import json import os @@ -32,9 +31,12 @@ import sys import time import unittest from ..server.failregex import Regex -from ..server.filter import Filter +from ..server.filter import Filter, FileContainer from ..client.filterreader import FilterReader -from .utils import setUpMyTime, tearDownMyTime, CONFIG_DIR +from .utils import setUpMyTime, tearDownMyTime, TEST_NOW, CONFIG_DIR + +# test-time in UTC as string in isoformat (2005-08-14T10:00:00): +TEST_NOW_STR = datetime.datetime.utcfromtimestamp(TEST_NOW).isoformat() TEST_CONFIG_DIR = os.path.join(os.path.dirname(__file__), "config") TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "files") @@ -133,6 +135,10 @@ class FilterSamplesRegex(unittest.TestCase): self._filters[fltName] = flt return flt + @staticmethod + def _filterOptions(opts): + return dict((k, v) for k, v in opts.iteritems() if not k.startswith('test.')) + def testSampleRegexsFactory(name, basedir): def testFilter(self): @@ -144,36 +150,51 @@ def testSampleRegexsFactory(name, basedir): regexsUsedRe = set() # process each test-file (note: array filenames can grow during processing): + commonOpts = {} faildata = {} i = 0 while i < len(filenames): filename = filenames[i]; i += 1; - logFile = fileinput.FileInput(os.path.join(TEST_FILES_DIR, "logs", - filename)) + logFile = FileContainer(os.path.join(TEST_FILES_DIR, "logs", + filename), 'UTF-8', doOpen=True) + # avoid errors if no NL char at end of test log-file: + logFile.waitForLineEnd = False ignoreBlock = False + lnnum = 0 for line in logFile: - jsonREMatch = re.match("^#+ ?(failJSON|filterOptions|addFILE):(.+)$", line) + lnnum += 1 + jsonREMatch = re.match("^#+ ?(failJSON|(?:file|filter)Options|addFILE):(.+)$", line) if jsonREMatch: try: faildata = json.loads(jsonREMatch.group(2)) + # fileOptions - dict in JSON to control common test-file filter options: + if jsonREMatch.group(1) == 'fileOptions': + commonOpts = faildata + continue # filterOptions - dict in JSON to control filter options (e. g. mode, etc.): if jsonREMatch.group(1) == 'filterOptions': # following lines with another filter options: self._filterTests = [] ignoreBlock = False - for opts in (faildata if isinstance(faildata, list) else [faildata]): + for faildata in (faildata if isinstance(faildata, list) else [faildata]): + if commonOpts: # merge with common file options: + opts = commonOpts.copy() + opts.update(faildata) + else: + opts = faildata # unique filter name (using options combination): self.assertTrue(isinstance(opts, dict)) if opts.get('test.condition'): ignoreBlock = not eval(opts.get('test.condition')) - del opts['test.condition'] - fltName = opts.get('filterName') - if not fltName: fltName = str(opts) if opts else '' - fltName = name + fltName - # read it: - flt = self._readFilter(fltName, name, basedir, opts=opts) - self._filterTests.append((fltName, flt)) + if not ignoreBlock: + fltOpts = self._filterOptions(opts) + fltName = opts.get('test.filter-name') + if not fltName: fltName = str(fltOpts) if fltOpts else '' + fltName = name + fltName + # read it: + flt = self._readFilter(fltName, name, basedir, opts=fltOpts) + self._filterTests.append((fltName, flt, opts)) continue # addFILE - filename to "include" test-files should be additionally parsed: if jsonREMatch.group(1) == 'addFILE': @@ -182,7 +203,7 @@ def testSampleRegexsFactory(name, basedir): # failJSON - faildata contains info of the failure to check it. except ValueError as e: # pragma: no cover - we've valid json's raise ValueError("%s: %s:%i" % - (e, logFile.filename(), logFile.filelineno())) + (e, logFile.getFileName(), lnnum)) line = next(logFile) elif ignoreBlock or line.startswith("#") or not line.strip(): continue @@ -194,79 +215,91 @@ def testSampleRegexsFactory(name, basedir): if not self._filterTests: fltName = name flt = self._readFilter(fltName, name, basedir, opts=None) - self._filterTests = [(fltName, flt)] + self._filterTests = [(fltName, flt, {})] + line = line.rstrip('\r\n') # process line using several filter options (if specified in the test-file): - for fltName, flt in self._filterTests: + for fltName, flt, opts in self._filterTests: + # Bypass if constraint (as expression) is not valid: + if faildata.get('constraint') and not eval(faildata['constraint']): + continue flt, regexsUsedIdx = flt regexList = flt.getFailRegex() - failregex = -1 try: fail = {} - ret = flt.processLine(line) + # for logtype "journal" we don't need parse timestamp (simulate real systemd-backend handling): + if opts.get('logtype') != 'journal': + ret = flt.processLine(line) + else: # simulate journal processing, time is known from journal (formatJournalEntry): + if opts.get('test.prefix-line'): # journal backends creates common prefix-line: + line = opts.get('test.prefix-line') + line + ret = flt.processLine(('', TEST_NOW_STR, line), TEST_NOW) + if ret: + # filter matched only (in checkAllRegex mode it could return 'nofail' too): + found = [] + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # bypass pending and nofail: + if fid is None or fail.get('nofail'): + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) + continue + found.append(ret) + ret = found + if not ret: - # Bypass if filter constraint specified: - if faildata.get('filter') and name != faildata.get('filter'): - continue # Check line is flagged as none match - self.assertFalse(faildata.get('match', True), + self.assertFalse(faildata.get('match', False), "Line not matched when should have") continue - failregex, fid, fail2banTime, fail = ret[0] - # Bypass no failure helpers-regexp: - if not faildata.get('match', False) and (fid is None or fail.get('nofail')): - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) - continue - # Check line is flagged to match self.assertTrue(faildata.get('match', False), "Line matched when shouldn't have") self.assertEqual(len(ret), 1, "Multiple regexs matched %r" % (map(lambda x: x[0], ret))) - # Verify match captures (at least fid/host) and timestamp as expected - for k, v in faildata.iteritems(): - if k not in ("time", "match", "desc", "filter"): - fv = fail.get(k, None) - if fv is None: - # Fallback for backwards compatibility (previously no fid, was host only): - if k == "host": - fv = fid - # special case for attempts counter: - if k == "attempts": - fv = len(fail.get('matches', {})) - # compare sorted (if set) - if isinstance(fv, (set, list, dict)): - self.assertSortedEqual(fv, v) - continue - self.assertEqual(fv, v) - - t = faildata.get("time", None) - try: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") - except ValueError: - jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") + for ret in ret: + failregex, fid, fail2banTime, fail = ret + # Verify match captures (at least fid/host) and timestamp as expected + for k, v in faildata.iteritems(): + if k not in ("time", "match", "desc", "constraint"): + fv = fail.get(k, None) + if fv is None: + # Fallback for backwards compatibility (previously no fid, was host only): + if k == "host": + fv = fid + # special case for attempts counter: + if k == "attempts": + fv = len(fail.get('matches', {})) + # compare sorted (if set) + if isinstance(fv, (set, list, dict)): + self.assertSortedEqual(fv, v) + continue + self.assertEqual(fv, v) + + t = faildata.get("time", None) + if t is not None: + try: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S") + except ValueError: + jsonTimeLocal = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%f") + jsonTime = time.mktime(jsonTimeLocal.timetuple()) + jsonTime += jsonTimeLocal.microsecond / 1000000.0 + self.assertEqual(fail2banTime, jsonTime, + "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % + (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), + jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), + fail2banTime - jsonTime) ) - jsonTime = time.mktime(jsonTimeLocal.timetuple()) - - jsonTime += jsonTimeLocal.microsecond / 1000000 - - self.assertEqual(fail2banTime, jsonTime, - "UTC Time mismatch %s (%s) != %s (%s) (diff %.3f seconds)" % - (fail2banTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(fail2banTime)), - jsonTime, time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(jsonTime)), - fail2banTime - jsonTime) ) - - regexsUsedIdx.add(failregex) - regexsUsedRe.add(regexList[failregex]) + regexsUsedIdx.add(failregex) + regexsUsedRe.add(regexList[failregex]) except AssertionError as e: # pragma: no cover import pprint - raise AssertionError("%s: %s on: %s:%i, line:\n %sregex (%s):\n %s\n" + raise AssertionError("%s: %s on: %s:%i, line:\n %s\nregex (%s):\n %s\n" "faildata: %s\nfail: %s" % ( - fltName, e, logFile.filename(), logFile.filelineno(), + fltName, e, logFile.getFileName(), lnnum, line, failregex, regexList[failregex] if failregex != -1 else None, '\n'.join(pprint.pformat(faildata).splitlines()), '\n'.join(pprint.pformat(fail).splitlines()))) diff --git a/fail2ban/tests/servertestcase.py b/fail2ban/tests/servertestcase.py index 8a4ef401..62ae81fd 100644 --- a/fail2ban/tests/servertestcase.py +++ b/fail2ban/tests/servertestcase.py @@ -35,13 +35,13 @@ import platform from ..server.failregex import Regex, FailRegex, RegexException from ..server import actions as _actions from ..server.server import Server -from ..server.ipdns import IPAddr +from ..server.ipdns import DNSUtils, IPAddr from ..server.jail import Jail from ..server.jailthread import JailThread from ..server.ticket import BanTicket from ..server.utils import Utils from .dummyjail import DummyJail -from .utils import LogCaptureTestCase +from .utils import LogCaptureTestCase, with_alt_time, MyTime from ..helpers import getLogger, extractOptions, PREFER_ENC from .. import version @@ -64,11 +64,14 @@ class TestServer(Server): pass -class TransmitterBase(unittest.TestCase): +class TransmitterBase(LogCaptureTestCase): + TEST_SRV_CLASS = TestServer + def setUp(self): """Call before every test case.""" super(TransmitterBase, self).setUp() + self.server = self.TEST_SRV_CLASS() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -157,10 +160,6 @@ class TransmitterBase(unittest.TestCase): class Transmitter(TransmitterBase): - def setUp(self): - self.server = TestServer() - super(Transmitter, self).setUp() - def testServerIsNotStarted(self): # so far isStarted only tested but not used otherwise # and here we don't really .start server @@ -175,6 +174,19 @@ class Transmitter(TransmitterBase): def testVersion(self): self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) + def testSetIPv6(self): + try: + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'yes']), (0, 'yes')) + self.assertTrue(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is on"); self.pruneLog() + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'no']), (0, 'no')) + self.assertFalse(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is off"); self.pruneLog() + finally: + # restore back to auto: + self.assertEqual(self.transm.proceed(["set", "allowipv6", "auto"]), (0, "auto")) + self.assertLogged("IPv6 is auto"); self.pruneLog() + def testSleep(self): if not unittest.F2B.fast: t0 = time.time() @@ -197,6 +209,8 @@ class Transmitter(TransmitterBase): self.setGetTest("dbfile", tmpFilename) # the same file name (again no jails / not changed): self.setGetTest("dbfile", tmpFilename) + self.setGetTest("dbmaxmatches", "100", 100) + self.setGetTestNOK("dbmaxmatches", "LIZARD") self.setGetTest("dbpurgeage", "600", 600) self.setGetTestNOK("dbpurgeage", "LIZARD") # the same file name (again with jails / not changed): @@ -212,6 +226,12 @@ class Transmitter(TransmitterBase): ["get", "dbfile"]), (0, None)) self.assertEqual(self.transm.proceed( + ["set", "dbmaxmatches", "100"]), + (0, None)) + self.assertEqual(self.transm.proceed( + ["get", "dbmaxmatches"]), + (0, None)) + self.assertEqual(self.transm.proceed( ["set", "dbpurgeage", "500"]), (0, None)) self.assertEqual(self.transm.proceed( @@ -330,22 +350,99 @@ class Transmitter(TransmitterBase): self.server.startJail(self.jailName) # Jail must be started self.assertEqual( - self.transm.proceed(["set", self.jailName, "banip", "127.0.0.1"]), - (0, "127.0.0.1")) - time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban + self.transm.proceed(["set", self.jailName, "banip", "192.0.2.1", "192.0.2.1", "192.0.2.2"]), + (0, 2)) + self.assertLogged("Ban 192.0.2.1", "Ban 192.0.2.2", all=True, wait=True) # Give chance to ban self.assertEqual( self.transm.proceed(["set", self.jailName, "banip", "Badger"]), - (0, "Badger")) #NOTE: Is IP address validated? Is DNS Lookup done? - time.sleep(Utils.DEFAULT_SLEEP_TIME) # Give chance to ban - # Unban IP + (0, 1)) #NOTE: Is IP address validated? Is DNS Lookup done? + self.assertLogged("Ban Badger", wait=True) # Give chance to ban + # Unban IP (first/last are not banned, so checking unban of both other succeeds): + self.assertEqual( + self.transm.proceed( + ["set", self.jailName, "unbanip", "192.0.2.255", "192.0.2.1", "192.0.2.2", "192.0.2.254"]), + (0, 2)) + self.assertLogged("Unban 192.0.2.1", "Unban 192.0.2.2", all=True, wait=True) + self.assertLogged("192.0.2.255 is not banned", "192.0.2.254 is not banned", all=True, wait=True) + self.pruneLog() + # Unban IP which isn't banned (error): self.assertEqual( self.transm.proceed( - ["set", self.jailName, "unbanip", "127.0.0.1"]), - (0, "127.0.0.1")) - # Unban IP which isn't banned + ["set", self.jailName, "unbanip", "--report-absent", "192.0.2.255"])[0],1) + # ... (no error, IPs logged only): self.assertEqual( self.transm.proceed( - ["set", self.jailName, "unbanip", "192.168.1.1"])[0],1) + ["set", self.jailName, "unbanip", "192.0.2.255", "192.0.2.254"]),(0, 0)) + self.assertLogged("192.0.2.255 is not banned", "192.0.2.254 is not banned", all=True, wait=True) + + def testJailAttemptIP(self): + self.server.startJail(self.jailName) # Jail must be started + + def attempt(ip, matches): + return self.transm.proceed(["set", self.jailName, "attempt", ip] + matches) + + self.setGetTest("maxretry", "5", 5, jail=self.jailName) + # produce 2 single attempts per IP: + for i in (1, 2): + for ip in ("192.0.2.1", "192.0.2.2"): + self.assertEqual(attempt(ip, ["test failure %d" % i]), (0, 1)) + self.assertLogged("192.0.2.1:2", "192.0.2.2:2", all=True, wait=True) + # this 3 attempts at once should cause a ban: + self.assertEqual(attempt(ip, ["test failure %d" % i for i in (3,4,5)]), (0, 1)) + self.assertLogged("192.0.2.2:5", wait=True) + # resulted to ban for "192.0.2.2" but not for "192.0.2.1": + self.assertLogged("Ban 192.0.2.2", wait=True) + self.assertNotLogged("Ban 192.0.2.1") + + @with_alt_time + def testJailBanList(self): + jail = "TestJailBanList" + self.server.addJail(jail, FAST_BACKEND) + self.server.startJail(jail) + + # Helper to process set banip/set unbanip commands and compare the list of + # banned IP addresses with outList. + def _getBanListTest(jail, banip=None, unbanip=None, args=(), outList=[]): + # Ban IP address + if banip is not None: + self.assertEqual( + self.transm.proceed(["set", jail, "banip", banip]), + (0, 1)) + self.assertLogged("Ban %s" % banip, wait=True) # Give chance to ban + # Unban IP address + if unbanip is not None: + self.assertEqual( + self.transm.proceed(["set", jail, "unbanip", unbanip]), + (0, 1)) + self.assertLogged("Unban %s" % unbanip, wait=True) # Give chance to unban + # Compare the list of banned IP addresses with outList + self.assertSortedEqual( + self.transm.proceed(["get", jail, "banip"]+list(args)), + (0, outList), nestedOnly=False) + MyTime.setTime(MyTime.time() + 1) + + _getBanListTest(jail, + outList=[]) + _getBanListTest(jail, banip="127.0.0.1", args=('--with-time',), + outList=["127.0.0.1 \t2005-08-14 12:00:01 + 600 = 2005-08-14 12:10:01"]) + _getBanListTest(jail, banip="192.168.0.1", args=('--with-time',), + outList=[ + "127.0.0.1 \t2005-08-14 12:00:01 + 600 = 2005-08-14 12:10:01", + "192.168.0.1 \t2005-08-14 12:00:02 + 600 = 2005-08-14 12:10:02"]) + _getBanListTest(jail, banip="192.168.1.10", + outList=["127.0.0.1", "192.168.0.1", "192.168.1.10"]) + _getBanListTest(jail, unbanip="127.0.0.1", + outList=["192.168.0.1", "192.168.1.10"]) + _getBanListTest(jail, unbanip="192.168.1.10", + outList=["192.168.0.1"]) + _getBanListTest(jail, unbanip="192.168.0.1", + outList=[]) + + def testJailMaxMatches(self): + self.setGetTest("maxmatches", "5", 5, jail=self.jailName) + self.setGetTest("maxmatches", "2", 2, jail=self.jailName) + self.setGetTest("maxmatches", "-2", -2, jail=self.jailName) + self.setGetTestNOK("maxmatches", "Duck", jail=self.jailName) def testJailMaxRetry(self): self.setGetTest("maxretry", "5", 5, jail=self.jailName) @@ -472,6 +569,9 @@ class Transmitter(TransmitterBase): jail=self.jailName) self.setGetTest("ignorecache", '', None, jail=self.jailName) + def testJailPrefRegex(self): + self.setGetTest("prefregex", "^Test", jail=self.jailName) + def testJailRegex(self): self.jailAddDelRegexTest("failregex", [ @@ -661,11 +761,11 @@ class Transmitter(TransmitterBase): self.assertEqual( self.transm.proceed( ["set", self.jailName, "action", action, "timeout", "10"]), - (0, "10")) + (0, 10)) self.assertEqual( self.transm.proceed( ["get", self.jailName, "action", action, "timeout"]), - (0, "10")) + (0, 10)) self.assertEqual( self.transm.proceed(["set", self.jailName, "delaction", action]), (0, None)) @@ -675,27 +775,11 @@ class Transmitter(TransmitterBase): def testPythonActionMethodsAndProperties(self): action = "TestCaseAction" - try: - out = self.transm.proceed( - ["set", self.jailName, "addaction", action, - os.path.join(TEST_FILES_DIR, "action.d", "action.py"), - '{"opt1": "value"}']) - self.assertEqual(out, (0, action)) - except AssertionError: - if ((2, 6) <= sys.version_info < (2, 6, 5)) \ - and '__init__() keywords must be strings' in out[1]: - # known issue http://bugs.python.org/issue2646 in 2.6 series - # since general Fail2Ban warnings are suppressed in normal - # operation -- let's issue Python's native warning here - import warnings - warnings.warn( - "Your version of Python %s seems to experience a known " - "issue forbidding correct operation of Fail2Ban: " - "http://bugs.python.org/issue2646 Upgrade your Python and " - "meanwhile other intestPythonActionMethodsAndProperties will " - "be skipped" % (sys.version)) - return - raise + out = self.transm.proceed( + ["set", self.jailName, "addaction", action, + os.path.join(TEST_FILES_DIR, "action.d", "action.py"), + '{"opt1": "value"}']) + self.assertEqual(out, (0, action)) self.assertSortedEqual( self.transm.proceed(["get", self.jailName, "actionproperties", action])[1], @@ -711,7 +795,7 @@ class Transmitter(TransmitterBase): self.assertSortedEqual( self.transm.proceed(["get", self.jailName, "actionmethods", action])[1], - ['ban', 'start', 'stop', 'testmethod', 'unban']) + ['ban', 'reban', 'start', 'stop', 'testmethod', 'unban']) self.assertEqual( self.transm.proceed(["set", self.jailName, "action", action, "testmethod", '{"text": "world!"}']), @@ -836,8 +920,9 @@ class Transmitter(TransmitterBase): class TransmitterLogging(TransmitterBase): + TEST_SRV_CLASS = Server + def setUp(self): - self.server = Server() super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") @@ -985,28 +1070,28 @@ class RegexTests(unittest.TestCase): def testHost(self): self.assertRaises(RegexException, FailRegex, '') self.assertRaises(RegexException, FailRegex, '^test no group$') - self.assertTrue(FailRegex('^test <HOST> group$')) - self.assertTrue(FailRegex('^test <IP4> group$')) - self.assertTrue(FailRegex('^test <IP6> group$')) - self.assertTrue(FailRegex('^test <DNS> group$')) - self.assertTrue(FailRegex('^test id group: ip:port = <F-ID><IP4>(?::<F-PORT/>)?</F-ID>$')) - self.assertTrue(FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$')) - self.assertTrue(FailRegex('^test id group: anything = <F-ID/>$')) + self.assertTrue(FailRegex(r'^test <HOST> group$')) + self.assertTrue(FailRegex(r'^test <IP4> group$')) + self.assertTrue(FailRegex(r'^test <IP6> group$')) + self.assertTrue(FailRegex(r'^test <DNS> group$')) + self.assertTrue(FailRegex(r'^test id group: ip:port = <F-ID><IP4>(?::<F-PORT/>)?</F-ID>$')) + self.assertTrue(FailRegex(r'^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$')) + self.assertTrue(FailRegex(r'^test id group: anything = <F-ID/>$')) # Testing obscure case when host group might be missing in the matched pattern, # e.g. if we made it optional. - fr = FailRegex('%%<HOST>?') + fr = FailRegex(r'%%<HOST>?') self.assertFalse(fr.hasMatched()) fr.search([('%%',"","")]) self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) # The same as above but using separated IPv4/IPv6 expressions - fr = FailRegex('%%inet(?:=<F-IP4/>|inet6=<F-IP6/>)?') + fr = FailRegex(r'%%inet(?:=<F-IP4/>|inet6=<F-IP6/>)?') self.assertFalse(fr.hasMatched()) fr.search([('%%inet=test',"","")]) self.assertTrue(fr.hasMatched()) self.assertRaises(RegexException, fr.getHost) # Success case: using separated IPv4/IPv6 expressions (no HOST) - fr = FailRegex('%%(?:inet(?:=<IP4>|6=<IP6>)?|dns=<DNS>?)') + fr = FailRegex(r'%%(?:inet(?:=<IP4>|6=<IP6>)?|dns=<DNS>?)') self.assertFalse(fr.hasMatched()) fr.search([('%%inet=192.0.2.1',"","")]) self.assertTrue(fr.hasMatched()) @@ -1018,11 +1103,39 @@ class RegexTests(unittest.TestCase): self.assertTrue(fr.hasMatched()) self.assertEqual(fr.getHost(), 'example.com') # Success case: using user as failure-id - fr = FailRegex('^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$') + fr = FailRegex(r'^test id group: user:\(<F-ID>[^\)]+</F-ID>\)$') self.assertFalse(fr.hasMatched()) fr.search([('test id group: user:(test login name)',"","")]) self.assertTrue(fr.hasMatched()) self.assertEqual(fr.getFailID(), 'test login name') + # Success case: subnet with IPAddr (IP and subnet) conversion: + fr = FailRegex(r'%%net=<SUBNET>') + fr.search([('%%net=192.0.2.1',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('192.0.2.1', 'inet4')) + fr.search([('%%net=192.0.2.1/24',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('192.0.2.0/24', 'inet4')) + fr.search([('%%net=2001:DB8:FF:FF::1',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('2001:db8:ff:ff::1', 'inet6')) + fr.search([('%%net=2001:DB8:FF:FF::1/60',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('2001:db8:ff:f0::/60', 'inet6')) + # CIDR: + fr = FailRegex(r'%%ip="<ADDR>", mask="<CIDR>?"') + fr.search([('%%ip="192.0.2.2", mask=""',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('192.0.2.2', 'inet4')) + fr.search([('%%ip="192.0.2.2", mask="24"',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('192.0.2.0/24', 'inet4')) + fr.search([('%%ip="2001:DB8:2FF:FF::1", mask=""',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('2001:db8:2ff:ff::1', 'inet6')) + fr.search([('%%ip="2001:DB8:2FF:FF::1", mask="60"',"","")]) + ip = fr.getIP() + self.assertEqual((ip, ip.familyStr), ('2001:db8:2ff:f0::/60', 'inet6')) class _BadThread(JailThread): @@ -1236,10 +1349,106 @@ class ServerConfigReaderTests(LogCaptureTestCase): # 'start', 'stop' - should be found (logged) on action start/stop, # etc. testJailsActions = ( + # nftables-multiport -- + ('j-w-nft-mp', 'nftables-multiport[name=%(__name__)s, port="http,https", protocol="tcp,udp,sctp"]', { + 'ip4': ('ip ', 'ipv4_addr', 'addr-'), 'ip6': ('ip6 ', 'ipv6_addr', 'addr6-'), + '*-start': ( + r"`nft add table inet f2b-table`", + r"`nft -- add chain inet f2b-table f2b-chain \{ type filter hook input priority -1 \; \}`", + # iterator over protocol is same for both families: + r"`for proto in $(echo 'tcp,udp,sctp' | sed 's/,/ /g'); do`", + r"`done`", + ), + 'ip4-start': ( + r"`nft add set inet f2b-table addr-set-j-w-nft-mp \{ type ipv4_addr\; \}`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip saddr @addr-set-j-w-nft-mp reject`", + ), + 'ip6-start': ( + r"`nft add set inet f2b-table addr6-set-j-w-nft-mp \{ type ipv6_addr\; \}`", + r"`nft add rule inet f2b-table f2b-chain $proto dport \{ $(echo 'http,https' | sed s/:/-/g) \} ip6 saddr @addr6-set-j-w-nft-mp reject`", + ), + 'flush': ( + "`{ nft flush set inet f2b-table addr-set-j-w-nft-mp 2> /dev/null; } || ", + "`{ nft flush set inet f2b-table addr6-set-j-w-nft-mp 2> /dev/null; } || ", + ), + 'stop': ( + "`{ nft -a list chain inet f2b-table f2b-chain | grep -oP '@addr-set-j-w-nft-mp\s+.*\s+\Khandle\s+(\d+)$'; } | while read -r hdl; do`", + "`nft delete rule inet f2b-table f2b-chain $hdl; done`", + "`nft delete set inet f2b-table addr-set-j-w-nft-mp`", + "`{ nft -a list chain inet f2b-table f2b-chain | grep -oP '@addr6-set-j-w-nft-mp\s+.*\s+\Khandle\s+(\d+)$'; } | while read -r hdl; do`", + "`nft delete rule inet f2b-table f2b-chain $hdl; done`", + "`nft delete set inet f2b-table addr6-set-j-w-nft-mp`", + ), + 'ip4-check': ( + r"`nft list chain inet f2b-table f2b-chain | grep -q '@addr-set-j-w-nft-mp[ \t]'`", + ), + 'ip6-check': ( + r"`nft list chain inet f2b-table f2b-chain | grep -q '@addr6-set-j-w-nft-mp[ \t]'`", + ), + 'ip4-ban': ( + r"`nft add element inet f2b-table addr-set-j-w-nft-mp \{ 192.0.2.1 \}`", + ), + 'ip4-unban': ( + r"`nft delete element inet f2b-table addr-set-j-w-nft-mp \{ 192.0.2.1 \}`", + ), + 'ip6-ban': ( + r"`nft add element inet f2b-table addr6-set-j-w-nft-mp \{ 2001:db8:: \}`", + ), + 'ip6-unban': ( + r"`nft delete element inet f2b-table addr6-set-j-w-nft-mp \{ 2001:db8:: \}`", + ), + }), + # nft-allports -- + ('j-w-nft-ap', 'nftables-allports[name=%(__name__)s, protocol="tcp,udp"]', { + 'ip4': ('ip ', 'ipv4_addr', 'addr-'), 'ip6': ('ip6 ', 'ipv6_addr', 'addr6-'), + '*-start': ( + r"`nft add table inet f2b-table`", + r"`nft -- add chain inet f2b-table f2b-chain \{ type filter hook input priority -1 \; \}`", + ), + 'ip4-start': ( + r"`nft add set inet f2b-table addr-set-j-w-nft-ap \{ type ipv4_addr\; \}`", + r"`nft add rule inet f2b-table f2b-chain meta l4proto \{ tcp,udp \} ip saddr @addr-set-j-w-nft-ap reject`", + ), + 'ip6-start': ( + r"`nft add set inet f2b-table addr6-set-j-w-nft-ap \{ type ipv6_addr\; \}`", + r"`nft add rule inet f2b-table f2b-chain meta l4proto \{ tcp,udp \} ip6 saddr @addr6-set-j-w-nft-ap reject`", + ), + 'flush': ( + "`{ nft flush set inet f2b-table addr-set-j-w-nft-ap 2> /dev/null; } || ", + "`{ nft flush set inet f2b-table addr6-set-j-w-nft-ap 2> /dev/null; } || ", + ), + 'stop': ( + "`{ nft -a list chain inet f2b-table f2b-chain | grep -oP '@addr-set-j-w-nft-ap\s+.*\s+\Khandle\s+(\d+)$'; } | while read -r hdl; do`", + "`nft delete rule inet f2b-table f2b-chain $hdl; done`", + "`nft delete set inet f2b-table addr-set-j-w-nft-ap`", + "`{ nft -a list chain inet f2b-table f2b-chain | grep -oP '@addr6-set-j-w-nft-ap\s+.*\s+\Khandle\s+(\d+)$'; } | while read -r hdl; do`", + "`nft delete rule inet f2b-table f2b-chain $hdl; done`", + "`nft delete set inet f2b-table addr6-set-j-w-nft-ap`", + ), + 'ip4-check': ( + r"""`nft list chain inet f2b-table f2b-chain | grep -q '@addr-set-j-w-nft-ap[ \t]'`""", + ), + 'ip6-check': ( + r"""`nft list chain inet f2b-table f2b-chain | grep -q '@addr6-set-j-w-nft-ap[ \t]'`""", + ), + 'ip4-ban': ( + r"`nft add element inet f2b-table addr-set-j-w-nft-ap \{ 192.0.2.1 \}`", + ), + 'ip4-unban': ( + r"`nft delete element inet f2b-table addr-set-j-w-nft-ap \{ 192.0.2.1 \}`", + ), + 'ip6-ban': ( + r"`nft add element inet f2b-table addr6-set-j-w-nft-ap \{ 2001:db8:: \}`", + ), + 'ip6-unban': ( + r"`nft delete element inet f2b-table addr6-set-j-w-nft-ap \{ 2001:db8:: \}`", + ), + }), # dummy -- - ('j-dummy', 'dummy[name=%(__name__)s, init="==", target="/tmp/fail2ban.dummy"]', { + ('j-dummy', '''dummy[name=%(__name__)s, init="=='<family>/<ip>'==bt:<bantime>==bc:<bancount>==", target="/tmp/fail2ban.dummy"]''', { 'ip4': ('family: inet4',), 'ip6': ('family: inet6',), 'start': ( + '''`printf %b "=='/'==bt:600==bc:0==\\n"''', ## empty family (independent in this action, same for both), no ip on start, initial bantime and bancount '`echo "[j-dummy] dummy /tmp/fail2ban.dummy -- started"`', ), 'flush': ( @@ -1278,35 +1487,42 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-multiport -- - ('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp", chain="<known/chain>"]', { + ('j-w-iptables-mp', 'iptables-multiport[name=%(__name__)s, bantime="10m", port="http,https", protocol="tcp,udp,sctp", chain="<known/chain>"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + r"`for proto in $(echo 'tcp,udp,sctp' | sed 's/,/ /g'); do`", + r"`done`", + ), 'ip4-start': ( - "`iptables -w -N f2b-j-w-iptables-mp`", - "`iptables -w -A f2b-j-w-iptables-mp -j RETURN`", - "`iptables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", + "`{ iptables -w -C f2b-j-w-iptables-mp -j RETURN >/dev/null 2>&1; } || " + "{ iptables -w -N f2b-j-w-iptables-mp || true; iptables -w -A f2b-j-w-iptables-mp -j RETURN; }`", + "`{ iptables -w -C INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp >/dev/null 2>&1; } || " + "{ iptables -w -I INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp; }`", ), 'ip6-start': ( - "`ip6tables -w -N f2b-j-w-iptables-mp`", - "`ip6tables -w -A f2b-j-w-iptables-mp -j RETURN`", - "`ip6tables -w -I INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", + "`{ ip6tables -w -C f2b-j-w-iptables-mp -j RETURN >/dev/null 2>&1; } || " + "{ ip6tables -w -N f2b-j-w-iptables-mp || true; ip6tables -w -A f2b-j-w-iptables-mp -j RETURN; }`", + "`{ ip6tables -w -C INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp >/dev/null 2>&1; } || ", + "{ ip6tables -w -I INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp; }`", ), 'flush': ( "`iptables -w -F f2b-j-w-iptables-mp`", "`ip6tables -w -F f2b-j-w-iptables-mp`", ), 'stop': ( - "`iptables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", + "`iptables -w -D INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp`", "`iptables -w -F f2b-j-w-iptables-mp`", "`iptables -w -X f2b-j-w-iptables-mp`", - "`ip6tables -w -D INPUT -p tcp -m multiport --dports http,https -j f2b-j-w-iptables-mp`", + "`ip6tables -w -D INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp`", "`ip6tables -w -F f2b-j-w-iptables-mp`", "`ip6tables -w -X f2b-j-w-iptables-mp`", ), 'ip4-check': ( - r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""", + r"""`iptables -w -C INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp`""", ), 'ip6-check': ( - r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-mp[ \t]'`""", + r"""`ip6tables -w -C INPUT -p $proto -m multiport --dports http,https -j f2b-j-w-iptables-mp`""", ), 'ip4-ban': ( r"`iptables -w -I f2b-j-w-iptables-mp 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`", @@ -1322,35 +1538,42 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-allports -- - ('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp", chain="<known/chain>"]', { + ('j-w-iptables-ap', 'iptables-allports[name=%(__name__)s, bantime="10m", protocol="tcp,udp,sctp", chain="<known/chain>"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + r"`for proto in $(echo 'tcp,udp,sctp' | sed 's/,/ /g'); do`", + r"`done`", + ), 'ip4-start': ( - "`iptables -w -N f2b-j-w-iptables-ap`", - "`iptables -w -A f2b-j-w-iptables-ap -j RETURN`", - "`iptables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`", + "`{ iptables -w -C f2b-j-w-iptables-ap -j RETURN >/dev/null 2>&1; } || " + "{ iptables -w -N f2b-j-w-iptables-ap || true; iptables -w -A f2b-j-w-iptables-ap -j RETURN; }`", + "`{ iptables -w -C INPUT -p $proto -j f2b-j-w-iptables-ap >/dev/null 2>&1; } || ", + "{ iptables -w -I INPUT -p $proto -j f2b-j-w-iptables-ap; }`", ), 'ip6-start': ( - "`ip6tables -w -N f2b-j-w-iptables-ap`", - "`ip6tables -w -A f2b-j-w-iptables-ap -j RETURN`", - "`ip6tables -w -I INPUT -p tcp -j f2b-j-w-iptables-ap`", + "`{ ip6tables -w -C f2b-j-w-iptables-ap -j RETURN >/dev/null 2>&1; } || " + "{ ip6tables -w -N f2b-j-w-iptables-ap || true; ip6tables -w -A f2b-j-w-iptables-ap -j RETURN; }`", + "`{ ip6tables -w -C INPUT -p $proto -j f2b-j-w-iptables-ap >/dev/null 2>&1; } || ", + "{ ip6tables -w -I INPUT -p $proto -j f2b-j-w-iptables-ap; }`", ), 'flush': ( "`iptables -w -F f2b-j-w-iptables-ap`", "`ip6tables -w -F f2b-j-w-iptables-ap`", ), 'stop': ( - "`iptables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`", + "`iptables -w -D INPUT -p $proto -j f2b-j-w-iptables-ap`", "`iptables -w -F f2b-j-w-iptables-ap`", "`iptables -w -X f2b-j-w-iptables-ap`", - "`ip6tables -w -D INPUT -p tcp -j f2b-j-w-iptables-ap`", + "`ip6tables -w -D INPUT -p $proto -j f2b-j-w-iptables-ap`", "`ip6tables -w -F f2b-j-w-iptables-ap`", "`ip6tables -w -X f2b-j-w-iptables-ap`", ), 'ip4-check': ( - r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""", + r"""`iptables -w -C INPUT -p $proto -j f2b-j-w-iptables-ap`""", ), 'ip6-check': ( - r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-ap[ \t]'`""", + r"""`ip6tables -w -C INPUT -p $proto -j f2b-j-w-iptables-ap`""", ), 'ip4-ban': ( r"`iptables -w -I f2b-j-w-iptables-ap 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`", @@ -1366,107 +1589,140 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # iptables-ipset-proto6 -- - ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain="<known/chain>"]', { + ('j-w-iptables-ipset', 'iptables-ipset-proto6[name=%(__name__)s, port="http", protocol="tcp", chain="<known/chain>"]', { 'ip4': (' f2b-j-w-iptables-ipset ',), 'ip6': (' f2b-j-w-iptables-ipset6 ',), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + "`for proto in $(echo 'tcp' | sed 's/,/ /g'); do`", + "`done`", + ), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset hash:ip timeout 0`", - "`iptables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", + "`ipset -exist create f2b-j-w-iptables-ipset hash:ip timeout 0 `", + "`{ iptables -w -C INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable >/dev/null 2>&1; } || " + "{ iptables -w -I INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable; }`", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset6 hash:ip timeout 0 family inet6`", - "`ip6tables -w -I INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + "`ipset -exist create f2b-j-w-iptables-ipset6 hash:ip timeout 0 family inet6`", + "`{ ip6tables -w -C INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable >/dev/null 2>&1; } || " + "{ ip6tables -w -I INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable; }`", ), 'flush': ( "`ipset flush f2b-j-w-iptables-ipset`", "`ipset flush f2b-j-w-iptables-ipset6`", ), 'stop': ( - "`iptables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", + "`iptables -w -D INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset`", "`ipset destroy f2b-j-w-iptables-ipset`", - "`ip6tables -w -D INPUT -p tcp -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", + "`ip6tables -w -D INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset6`", "`ipset destroy f2b-j-w-iptables-ipset6`", ), + 'ip4-check': ( + r"""`iptables -w -C INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset src -j REJECT --reject-with icmp-port-unreachable`""", + ), + 'ip6-check': ( + r"""`ip6tables -w -C INPUT -p $proto -m multiport --dports http -m set --match-set f2b-j-w-iptables-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`""", + ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-iptables-ipset 192.0.2.1 timeout 0`", ), 'ip4-unban': ( - r"`ipset del f2b-j-w-iptables-ipset 192.0.2.1 -exist`", + r"`ipset -exist del f2b-j-w-iptables-ipset 192.0.2.1`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-iptables-ipset6 2001:db8:: timeout 0`", ), 'ip6-unban': ( - r"`ipset del f2b-j-w-iptables-ipset6 2001:db8:: -exist`", + r"`ipset -exist del f2b-j-w-iptables-ipset6 2001:db8::`", ), }), # iptables-ipset-proto6-allports -- - ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, bantime="10m", default-timeout=0, chain="<known/chain>"]', { + ('j-w-iptables-ipset-ap', 'iptables-ipset-proto6-allports[name=%(__name__)s, chain="<known/chain>"]', { 'ip4': (' f2b-j-w-iptables-ipset-ap ',), 'ip6': (' f2b-j-w-iptables-ipset-ap6 ',), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + "`for proto in $(echo 'tcp' | sed 's/,/ /g'); do`", + "`done`", + ), 'ip4-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap hash:ip timeout 0`", - "`iptables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", + "`ipset -exist create f2b-j-w-iptables-ipset-ap hash:ip timeout 0 `", + "`{ iptables -w -C INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable >/dev/null 2>&1; } || " + "{ iptables -w -I INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable; }", ), 'ip6-start': ( - "`ipset create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 0 family inet6`", - "`ip6tables -w -I INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", + "`ipset -exist create f2b-j-w-iptables-ipset-ap6 hash:ip timeout 0 family inet6`", + "`{ ip6tables -w -C INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable >/dev/null 2>&1; } || " + "{ ip6tables -w -I INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable; }", ), 'flush': ( "`ipset flush f2b-j-w-iptables-ipset-ap`", "`ipset flush f2b-j-w-iptables-ipset-ap6`", ), 'stop': ( - "`iptables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", + "`iptables -w -D INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset-ap`", "`ipset destroy f2b-j-w-iptables-ipset-ap`", - "`ip6tables -w -D INPUT -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", + "`ip6tables -w -D INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", "`ipset flush f2b-j-w-iptables-ipset-ap6`", "`ipset destroy f2b-j-w-iptables-ipset-ap6`", ), + 'ip4-check': ( + r"""`iptables -w -C INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`""", + ), + 'ip6-check': ( + r"""`ip6tables -w -C INPUT -p $proto -m set --match-set f2b-j-w-iptables-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`""", + ), 'ip4-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-iptables-ipset-ap 192.0.2.1 timeout 0`", ), 'ip4-unban': ( - r"`ipset del f2b-j-w-iptables-ipset-ap 192.0.2.1 -exist`", + r"`ipset -exist del f2b-j-w-iptables-ipset-ap 192.0.2.1`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-iptables-ipset-ap6 2001:db8:: timeout 0`", ), 'ip6-unban': ( - r"`ipset del f2b-j-w-iptables-ipset-ap6 2001:db8:: -exist`", + r"`ipset -exist del f2b-j-w-iptables-ipset-ap6 2001:db8::`", ), }), - # iptables -- + # iptables (oneport) -- ('j-w-iptables', 'iptables[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + "`for proto in $(echo 'tcp' | sed 's/,/ /g'); do`", + "`done`", + ), 'ip4-start': ( - "`iptables -w -N f2b-j-w-iptables`", - "`iptables -w -A f2b-j-w-iptables -j RETURN`", - "`iptables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`", + "`{ iptables -w -C f2b-j-w-iptables -j RETURN >/dev/null 2>&1; } || " + "{ iptables -w -N f2b-j-w-iptables || true; iptables -w -A f2b-j-w-iptables -j RETURN; }", + "`{ iptables -w -C INPUT -p $proto --dport http -j f2b-j-w-iptables >/dev/null 2>&1; } || " + "{ iptables -w -I INPUT -p $proto --dport http -j f2b-j-w-iptables; }`", ), 'ip6-start': ( - "`ip6tables -w -N f2b-j-w-iptables`", - "`ip6tables -w -A f2b-j-w-iptables -j RETURN`", - "`ip6tables -w -I INPUT -p tcp --dport http -j f2b-j-w-iptables`", + "`{ ip6tables -w -C f2b-j-w-iptables -j RETURN >/dev/null 2>&1; } || " + "{ ip6tables -w -N f2b-j-w-iptables || true; ip6tables -w -A f2b-j-w-iptables -j RETURN; }", + "`{ ip6tables -w -C INPUT -p $proto --dport http -j f2b-j-w-iptables >/dev/null 2>&1; } || " + "{ ip6tables -w -I INPUT -p $proto --dport http -j f2b-j-w-iptables; }`", ), 'flush': ( "`iptables -w -F f2b-j-w-iptables`", "`ip6tables -w -F f2b-j-w-iptables`", ), 'stop': ( - "`iptables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`", + "`iptables -w -D INPUT -p $proto --dport http -j f2b-j-w-iptables`", "`iptables -w -F f2b-j-w-iptables`", "`iptables -w -X f2b-j-w-iptables`", - "`ip6tables -w -D INPUT -p tcp --dport http -j f2b-j-w-iptables`", + "`ip6tables -w -D INPUT -p $proto --dport http -j f2b-j-w-iptables`", "`ip6tables -w -F f2b-j-w-iptables`", "`ip6tables -w -X f2b-j-w-iptables`", ), 'ip4-check': ( - r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""", + r"""`iptables -w -C INPUT -p $proto --dport http -j f2b-j-w-iptables`""", ), 'ip6-check': ( - r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables[ \t]'`""", + r"""`ip6tables -w -C INPUT -p $proto --dport http -j f2b-j-w-iptables`""", ), 'ip4-ban': ( r"`iptables -w -I f2b-j-w-iptables 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`", @@ -1484,33 +1740,40 @@ class ServerConfigReaderTests(LogCaptureTestCase): # iptables-new -- ('j-w-iptables-new', 'iptables-new[name=%(__name__)s, bantime="10m", port="http", protocol="tcp", chain="<known/chain>"]', { 'ip4': ('`iptables ', 'icmp-port-unreachable'), 'ip6': ('`ip6tables ', 'icmp6-port-unreachable'), + '*-start-stop-check': ( + # iterator over protocol is same for both families: + "`for proto in $(echo 'tcp' | sed 's/,/ /g'); do`", + "`done`", + ), 'ip4-start': ( - "`iptables -w -N f2b-j-w-iptables-new`", - "`iptables -w -A f2b-j-w-iptables-new -j RETURN`", - "`iptables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", + "`{ iptables -w -C f2b-j-w-iptables-new -j RETURN >/dev/null 2>&1; } || " + "{ iptables -w -N f2b-j-w-iptables-new || true; iptables -w -A f2b-j-w-iptables-new -j RETURN; }`", + "`{ iptables -w -C INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new >/dev/null 2>&1; } || " + "{ iptables -w -I INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new; }`", ), 'ip6-start': ( - "`ip6tables -w -N f2b-j-w-iptables-new`", - "`ip6tables -w -A f2b-j-w-iptables-new -j RETURN`", - "`ip6tables -w -I INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", + "`{ ip6tables -w -C f2b-j-w-iptables-new -j RETURN >/dev/null 2>&1; } || " + "{ ip6tables -w -N f2b-j-w-iptables-new || true; ip6tables -w -A f2b-j-w-iptables-new -j RETURN; }`", + "`{ ip6tables -w -C INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new >/dev/null 2>&1; } || " + "{ ip6tables -w -I INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new; }`", ), 'flush': ( "`iptables -w -F f2b-j-w-iptables-new`", "`ip6tables -w -F f2b-j-w-iptables-new`", ), 'stop': ( - "`iptables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", + "`iptables -w -D INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new`", "`iptables -w -F f2b-j-w-iptables-new`", "`iptables -w -X f2b-j-w-iptables-new`", - "`ip6tables -w -D INPUT -m state --state NEW -p tcp --dport http -j f2b-j-w-iptables-new`", + "`ip6tables -w -D INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new`", "`ip6tables -w -F f2b-j-w-iptables-new`", "`ip6tables -w -X f2b-j-w-iptables-new`", ), 'ip4-check': ( - r"""`iptables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""", + r"""`iptables -w -C INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new`""", ), 'ip6-check': ( - r"""`ip6tables -w -n -L INPUT | grep -q 'f2b-j-w-iptables-new[ \t]'`""", + r"""`ip6tables -w -C INPUT -m state --state NEW -p $proto --dport http -j f2b-j-w-iptables-new`""", ), 'ip4-ban': ( r"`iptables -w -I f2b-j-w-iptables-new 1 -s 192.0.2.1 -j REJECT --reject-with icmp-port-unreachable`", @@ -1529,22 +1792,26 @@ class ServerConfigReaderTests(LogCaptureTestCase): ('j-w-iptables-xtre', 'iptables-xt_recent-echo[name=%(__name__)s, bantime="10m", chain="<known/chain>"]', { 'ip4': ('`iptables ', '/f2b-j-w-iptables-xtre`'), 'ip6': ('`ip6tables ', '/f2b-j-w-iptables-xtre6`'), 'ip4-start': ( - "`if [ `id -u` -eq 0 ];then iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`", + "`{ iptables -w -C INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable >/dev/null 2>&1; } || { iptables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable; }`", ), 'ip6-start': ( - "`if [ `id -u` -eq 0 ];then ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`", + "`{ ip6tables -w -C INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable >/dev/null 2>&1; } || { ip6tables -w -I INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable; }`", ), 'stop': ( "`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre`", - "`if [ `id -u` -eq 0 ];then iptables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;fi`", + "`if [ `id -u` -eq 0 ];then`", + "`iptables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable;`", + "`fi`", "`echo / > /proc/net/xt_recent/f2b-j-w-iptables-xtre6`", - "`if [ `id -u` -eq 0 ];then ip6tables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;fi`", + "`if [ `id -u` -eq 0 ];then`", + "`ip6tables -w -D INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable;`", + "`fi`", ), 'ip4-check': ( - r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre`", + r"`{ iptables -w -C INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre -j REJECT --reject-with icmp-port-unreachable; } && test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre`", ), 'ip6-check': ( - r"`test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre6`", + r"`{ ip6tables -w -C INPUT -m recent --update --seconds 3600 --name f2b-j-w-iptables-xtre6 -j REJECT --reject-with icmp6-port-unreachable; } && test -e /proc/net/xt_recent/f2b-j-w-iptables-xtre6`", ), 'ip4-ban': ( r"`echo +192.0.2.1 > /proc/net/xt_recent/f2b-j-w-iptables-xtre`", @@ -1709,14 +1976,14 @@ class ServerConfigReaderTests(LogCaptureTestCase): ), }), # firewallcmd-ipset (multiport) -- - ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", default-timeout=0, port="http", protocol="tcp", chain="<known/chain>"]', { + ('j-w-fwcmd-ipset', 'firewallcmd-ipset[name=%(__name__)s, port="http", protocol="tcp", chain="<known/chain>"]', { 'ip4': (' f2b-j-w-fwcmd-ipset ',), 'ip6': (' f2b-j-w-fwcmd-ipset6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset hash:ip timeout 0`", + "`ipset -exist create f2b-j-w-fwcmd-ipset hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", + "`ipset -exist create f2b-j-w-fwcmd-ipset6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m multiport --dports http -m set --match-set f2b-j-w-fwcmd-ipset6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1732,27 +1999,27 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-fwcmd-ipset 192.0.2.1 timeout 0`", ), 'ip4-unban': ( - r"`ipset del f2b-j-w-fwcmd-ipset 192.0.2.1 -exist`", + r"`ipset -exist del f2b-j-w-fwcmd-ipset 192.0.2.1`", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-fwcmd-ipset6 2001:db8:: timeout 0`", ), 'ip6-unban': ( - r"`ipset del f2b-j-w-fwcmd-ipset6 2001:db8:: -exist`", + r"`ipset -exist del f2b-j-w-fwcmd-ipset6 2001:db8::`", ), }), # firewallcmd-ipset (allports) -- - ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, bantime="10m", actiontype=<allports>, protocol="tcp", chain="<known/chain>"]', { + ('j-w-fwcmd-ipset-ap', 'firewallcmd-ipset[name=%(__name__)s, actiontype=<allports>, protocol="tcp", chain="<known/chain>"]', { 'ip4': (' f2b-j-w-fwcmd-ipset-ap ',), 'ip6': (' f2b-j-w-fwcmd-ipset-ap6 ',), 'ip4-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 600`", + "`ipset -exist create f2b-j-w-fwcmd-ipset-ap hash:ip timeout 0 `", "`firewall-cmd --direct --add-rule ipv4 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap src -j REJECT --reject-with icmp-port-unreachable`", ), 'ip6-start': ( - "`ipset create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 600 family inet6`", + "`ipset -exist create f2b-j-w-fwcmd-ipset-ap6 hash:ip timeout 0 family inet6`", "`firewall-cmd --direct --add-rule ipv6 filter INPUT_direct 0 -p tcp -m set --match-set f2b-j-w-fwcmd-ipset-ap6 src -j REJECT --reject-with icmp6-port-unreachable`", ), 'flush': ( @@ -1768,16 +2035,48 @@ class ServerConfigReaderTests(LogCaptureTestCase): "`ipset destroy f2b-j-w-fwcmd-ipset-ap6`", ), 'ip4-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 600 -exist`", + r"`ipset -exist add f2b-j-w-fwcmd-ipset-ap 192.0.2.1 timeout 0`", + ), + 'ip4-unban': ( + r"`ipset -exist del f2b-j-w-fwcmd-ipset-ap 192.0.2.1`", + ), + 'ip6-ban': ( + r"`ipset -exist add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 0`", + ), + 'ip6-unban': ( + r"`ipset -exist del f2b-j-w-fwcmd-ipset-ap6 2001:db8::`", + ), + }), + # firewallcmd-rich-rules -- + ('j-fwcmd-rr', 'firewallcmd-rich-rules[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", ), 'ip4-unban': ( - r"`ipset del f2b-j-w-fwcmd-ipset-ap 192.0.2.1 -exist`", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' reject type='icmp-port-unreachable'"; done`""", ), 'ip6-ban': ( - r"`ipset add f2b-j-w-fwcmd-ipset-ap6 2001:db8:: timeout 600 -exist`", + """ `ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", ), 'ip6-unban': ( - r"`ipset del f2b-j-w-fwcmd-ipset-ap6 2001:db8:: -exist`", + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' reject type='icmp6-port-unreachable'"; done`""", + ), + }), + # firewallcmd-rich-logging -- + ('j-fwcmd-rl', 'firewallcmd-rich-logging[port="22:24", protocol="tcp"]', { + 'ip4': ("family='ipv4'", "icmp-port-unreachable",), 'ip6': ("family='ipv6'", 'icmp6-port-unreachable',), + 'ip4-ban': ( + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip4-unban': ( + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv4' source address='192.0.2.1' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp-port-unreachable'"; done`""", + ), + 'ip6-ban': ( + """ `ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --add-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", + ), + 'ip6-unban': ( + """`ports="22:24"; for p in $(echo $ports | tr ", " " "); do firewall-cmd --remove-rich-rule="rule family='ipv6' source address='2001:db8::' port port='$p' protocol='tcp' log prefix='f2b-j-fwcmd-rl' level='info' limit value='1/m' reject type='icmp6-port-unreachable'"; done`""", ), }), ) @@ -1820,27 +2119,40 @@ class ServerConfigReaderTests(LogCaptureTestCase): # test ban ip4 : self.pruneLog('# === ban-ipv4 ===') action.ban(aInfos['ipv4']) - if tests.get('ip4-start'): self.assertLogged(*tests['ip4-start'], all=True) + if tests.get('ip4-start'): self.assertLogged(*tests.get('*-start', tests.get('*-start-stop-check', ()))+tests['ip4-start'], all=True) if tests.get('ip6-start'): self.assertNotLogged(*tests['ip6-start'], all=True) - self.assertLogged(*tests.get('ip4-check',())+tests['ip4-ban'], all=True) + self.assertLogged(*tests['ip4-ban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test unban ip4 : self.pruneLog('# === unban ipv4 ===') action.unban(aInfos['ipv4']) - self.assertLogged(*tests.get('ip4-check',())+tests['ip4-unban'], all=True) + self.assertLogged(*tests['ip4-unban'], all=True) self.assertNotLogged(*tests['ip6'], all=True) # test ban ip6 : self.pruneLog('# === ban ipv6 ===') action.ban(aInfos['ipv6']) - if tests.get('ip6-start'): self.assertLogged(*tests['ip6-start'], all=True) + if tests.get('ip6-start'): self.assertLogged(*tests.get('*-start', tests.get('*-start-stop-check', ()))+tests['ip6-start'], all=True) if tests.get('ip4-start'): self.assertNotLogged(*tests['ip4-start'], all=True) - self.assertLogged(*tests.get('ip6-check',())+tests['ip6-ban'], all=True) + self.assertLogged(*tests['ip6-ban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) # test unban ip6 : self.pruneLog('# === unban ipv6 ===') action.unban(aInfos['ipv6']) - self.assertLogged(*tests.get('ip6-check',())+tests['ip6-unban'], all=True) + self.assertLogged(*tests['ip6-unban'], all=True) self.assertNotLogged(*tests['ip4'], all=True) + # test invariant check (normally on demand in error case only): + if tests.get('ip4-check'): + self.pruneLog('# === check ipv4 ===') + action._invariantCheck(aInfos['ipv4']['family']) + self.assertLogged(*tests.get('*-check', tests.get('*-start-stop-check', ()))+tests['ip4-check'], all=True) + if tests.get('ip6-check') and tests['ip6-check'] != tests['ip4-check']: + self.assertNotLogged(*tests['ip6-check'], all=True) + if tests.get('ip6-check'): + self.pruneLog('# === check ipv6 ===') + action._invariantCheck(aInfos['ipv6']['family']) + self.assertLogged(*tests.get('*-check', tests.get('*-start-stop-check', ()))+tests['ip6-check'], all=True) + if tests.get('ip4-check') and tests['ip4-check'] != tests['ip6-check']: + self.assertNotLogged(*tests['ip4-check'], all=True) # test flush for actions should supported this: if tests.get('flush'): self.pruneLog('# === flush ===') @@ -1849,16 +2161,23 @@ class ServerConfigReaderTests(LogCaptureTestCase): # test stop : self.pruneLog('# === stop ===') action.stop() - if tests.get('stop'): self.assertLogged(*tests['stop'], all=True) + if tests.get('stop'): self.assertLogged(*tests.get('*-start-stop-check', ())+tests['stop'], all=True) def _executeMailCmd(self, realCmd, timeout=60): # replace pipe to mail with pipe to cat: - realCmd = re.sub(r'\)\s*\|\s*mail\b([^\n]*)', - r') | cat; printf "\\n... | "; echo mail \1', realCmd) + cmd = realCmd + if isinstance(realCmd, list): + cmd = realCmd[0] + cmd = re.sub(r'\)\s*\|\s*(\S*mail\b[^\n]*)', + r') | cat; printf "\\n... | "; echo \1', cmd) # replace abuse retrieving (possible no-network), just replace first occurrence of 'dig...': - realCmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+', + cmd = re.sub(r'\bADDRESSES=\$\(dig\s[^\n]+', lambda m: 'ADDRESSES="abuse-1@abuse-test-server, abuse-2@abuse-test-server"', - realCmd, 1) + cmd, 1) + if isinstance(realCmd, list): + realCmd[0] = cmd + else: + realCmd = cmd # execute action: return _actions.CommandAction.executeCmd(realCmd, timeout=timeout) @@ -1885,6 +2204,26 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', ), }), + # sendmail-whois-lines -- + ('j-sendmail-whois-lines', + 'sendmail-whois-lines[' + '''name=%(__name__)s, grepopts="-m 1", grepmax=2, mailcmd='testmail -f "<sender>" "<dest>"', ''' + + # 2 logs to test grep from multiple logs: + 'logpath="' + os.path.join(TEST_FILES_DIR, "testcase01.log") + '\n' + + ' ' + os.path.join(TEST_FILES_DIR, "testcase01a.log") + '", ' + '_whois_command="echo \'-- information about <ip> --\'"' + ']', + { + 'ip4-ban': ( + 'The IP 87.142.124.10 has just been banned by Fail2Ban after', + '100 attempts against j-sendmail-whois-lines.', + 'Here is more information about 87.142.124.10 :', + '-- information about 87.142.124.10 --', + 'Lines containing failures of 87.142.124.10 (max 2)', + 'testcase01.log:Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'testcase01a.log:Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + ), + }), # complain -- ('j-complain-abuse', 'complain[' @@ -1913,6 +2252,31 @@ class ServerConfigReaderTests(LogCaptureTestCase): 'mail -s Hostname: test-host, family: inet6 - Abuse from 2001:db8::1 abuse-1@abuse-test-server abuse-2@abuse-test-server', ), }), + # xarf-login-attack -- + ('j-xarf-abuse', + 'xarf-login-attack[' + 'name=%(__name__)s, mailcmd="mail", mailargs="",' + + # test reverse ip: + 'debug=1' + + ']', + { + 'ip4-ban': ( + # test reverse ip: + 'try to resolve 10.124.142.87.abuse-contacts.abusix.org', + 'We have detected abuse from the IP address 87.142.124.10', + 'Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10', + # both abuse mails should be separated with space: + 'mail abuse-1@abuse-test-server abuse-2@abuse-test-server', + ), + 'ip6-ban': ( + # test reverse ip: + 'try to resolve 1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.abuse-contacts.abusix.org', + 'We have detected abuse from the IP address 2001:db8::1', + # both abuse mails should be separated with space: + 'mail abuse-1@abuse-test-server abuse-2@abuse-test-server', + ), + }), ) server = TestServer() transm = server._Server__transm @@ -1950,6 +2314,10 @@ class ServerConfigReaderTests(LogCaptureTestCase): self.pruneLog('# === %s ===' % test) ticket = BanTicket(ip) ticket.setAttempt(100) + ticket.setMatches([ + 'Dec 31 11:59:59 [sshd] error: PAM: Authentication failure for kevin from 87.142.124.10', + 'Dec 31 11:55:01 [sshd] error: PAM: Authentication failure for test from 87.142.124.10' + ]) ticket = _actions.Actions.ActionInfo(ticket, dmyjail) action.ban(ticket) self.assertLogged(*tests[test], all=True) diff --git a/fail2ban/tests/sockettestcase.py b/fail2ban/tests/sockettestcase.py index 69bf8d8b..e3a07998 100644 --- a/fail2ban/tests/sockettestcase.py +++ b/fail2ban/tests/sockettestcase.py @@ -87,7 +87,7 @@ class Socket(LogCaptureTestCase): def _stopServerThread(self): serverThread = self.serverThread # wait for end of thread : - Utils.wait_for(lambda: not serverThread.isAlive() + Utils.wait_for(lambda: not serverThread.is_alive() or serverThread.join(Utils.DEFAULT_SLEEP_TIME), unittest.F2B.maxWaitTime(10)) self.serverThread = None @@ -98,7 +98,7 @@ class Socket(LogCaptureTestCase): self.server.close() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) # clean : self.server.stop() self.assertFalse(self.server.isActive()) @@ -139,7 +139,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) @@ -153,7 +153,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: self.close() - self.assertRaisesRegexp(RuntimeError, r"socket connection broken", + self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", lambda: client.send(testMessage, timeout=unittest.F2B.maxWaitTime(10))) finally: RequestHandler.found_terminator = org_handler @@ -169,7 +169,7 @@ class Socket(LogCaptureTestCase): org_handler = RequestHandler.found_terminator try: RequestHandler.found_terminator = lambda self: TestMsgError() - #self.assertRaisesRegexp(RuntimeError, r"socket connection broken", client.send, testMessage) + #self.assertRaisesRegexp(Exception, r"reset by peer|Broken pipe", client.send, testMessage) self.assertEqual(client.send(testMessage), 'ERROR: test unpickle error') finally: RequestHandler.found_terminator = org_handler @@ -180,7 +180,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) def testLoopErrors(self): # replace poll handler to produce error in loop-cycle: @@ -216,7 +216,7 @@ class Socket(LogCaptureTestCase): self.server.stop() # wait for end of thread : self._stopServerThread() - self.assertFalse(serverThread.isAlive()) + self.assertFalse(serverThread.is_alive()) self.assertFalse(self.server.isActive()) self.assertFalse(os.path.exists(self.sock_name)) diff --git a/fail2ban/tests/tickettestcase.py b/fail2ban/tests/tickettestcase.py index 277c2f28..771d2b50 100644 --- a/fail2ban/tests/tickettestcase.py +++ b/fail2ban/tests/tickettestcase.py @@ -39,6 +39,7 @@ class TicketTests(unittest.TestCase): # Ticket t = Ticket('193.168.0.128', tm, matches) + self.assertEqual(t.getID(), '193.168.0.128') self.assertEqual(t.getIP(), '193.168.0.128') self.assertEqual(t.getTime(), tm) self.assertEqual(t.getMatches(), matches2) @@ -65,14 +66,15 @@ class TicketTests(unittest.TestCase): matches = ['first', 'second'] ft = FailTicket('193.168.0.128', tm, matches) ft.setBanTime(60*60) + self.assertEqual(ft.getID(), '193.168.0.128') self.assertEqual(ft.getIP(), '193.168.0.128') self.assertEqual(ft.getTime(), tm) self.assertEqual(ft.getMatches(), matches2) ft.setAttempt(2) - self.assertEqual(ft.getAttempt(), 2) - # retry is max of set retry and failures: - self.assertEqual(ft.getRetry(), 2) ft.setRetry(1) + self.assertEqual(ft.getAttempt(), 2) + self.assertEqual(ft.getRetry(), 1) + ft.setRetry(2) self.assertEqual(ft.getRetry(), 2) ft.setRetry(3) self.assertEqual(ft.getRetry(), 3) @@ -86,13 +88,21 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft.getRetry(), 14) self.assertEqual(ft.getMatches(), matches3) # last time (ignore if smaller as time): - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm-60) self.assertEqual(ft.getTime(), tm) - self.assertEqual(ft.getLastTime(), tm) - ft.setLastTime(tm+60) + ft.adjustTime(tm-60, 3600) + self.assertEqual(ft.getTime(), tm) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+60, 3600) self.assertEqual(ft.getTime(), tm+60) - self.assertEqual(ft.getLastTime(), tm+60) + self.assertEqual(ft.getRetry(), 14) + ft.adjustTime(tm+3600, 3600) + self.assertEqual(ft.getTime(), tm+3600) + self.assertEqual(ft.getRetry(), 14) + # adjust time so interval is larger than find time (3600), so reset retry count: + ft.adjustTime(tm+7200, 3600) + self.assertEqual(ft.getTime(), tm+7200) + self.assertEqual(ft.getRetry(), 7); # estimated attempts count + self.assertEqual(ft.getAttempt(), 4); # real known failure count ft.setData('country', 'DE') self.assertEqual(ft.getData(), {'matches': ['first', 'second', 'third'], 'failures': 4, 'country': 'DE'}) @@ -102,12 +112,23 @@ class TicketTests(unittest.TestCase): self.assertEqual(ft, ft2) self.assertEqual(ft.getData(), ft2.getData()) self.assertEqual(ft2.getAttempt(), 4) - self.assertEqual(ft2.getRetry(), 14) + self.assertEqual(ft2.getRetry(), 7) self.assertEqual(ft2.getMatches(), matches3) self.assertEqual(ft2.getTime(), ft.getTime()) - self.assertEqual(ft2.getLastTime(), ft.getLastTime()) + self.assertEqual(ft2.getTime(), ft.getTime()) self.assertEqual(ft2.getBanTime(), ft.getBanTime()) + def testDiffIDAndIPTicket(self): + tm = MyTime.time() + # different ID (string) and IP: + t = Ticket('123-456-678', tm, data={'ip':'192.0.2.1'}) + self.assertEqual(t.getID(), '123-456-678') + self.assertEqual(t.getIP(), '192.0.2.1') + # different ID (tuple) and IP: + t = Ticket(('192.0.2.1', '5000'), tm, data={'ip':'192.0.2.1'}) + self.assertEqual(t.getID(), ('192.0.2.1', '5000')) + self.assertEqual(t.getIP(), '192.0.2.1') + def testTicketFlags(self): flags = ('restored', 'banned') ticket = Ticket('test', 0) diff --git a/fail2ban/tests/utils.py b/fail2ban/tests/utils.py index d6a5e678..8bcc1431 100644 --- a/fail2ban/tests/utils.py +++ b/fail2ban/tests/utils.py @@ -22,6 +22,7 @@ __author__ = "Yaroslav Halchenko" __copyright__ = "Copyright (c) 2013 Yaroslav Halchenko" __license__ = "GPL" +import fileinput import itertools import logging import optparse @@ -38,7 +39,7 @@ from cStringIO import StringIO from functools import wraps from ..helpers import getLogger, str2LogLevel, getVerbosityFormat, uni_decode -from ..server.ipdns import DNSUtils +from ..server.ipdns import IPAddr, DNSUtils from ..server.mytime import MyTime from ..server.utils import Utils # for action_d.test_smtp : @@ -46,7 +47,7 @@ from ..server import asyncserver from ..version import version -logSys = getLogger(__name__) +logSys = getLogger("fail2ban") TEST_NOW = 1124013600 @@ -125,9 +126,6 @@ def getOptParser(doc=""): def initProcess(opts): # Logger: - global logSys - logSys = getLogger("fail2ban") - llev = None if opts.log_level is not None: # pragma: no cover # so we had explicit settings @@ -245,23 +243,17 @@ def with_tmpdir(f): shutil.rmtree(tmp) return wrapper +def with_alt_time(f): + """Helper decorator to execute test in alternate (fixed) test time.""" + @wraps(f) + def wrapper(self, *args, **kwargs): + setUpMyTime() + try: + return f(self, *args, **kwargs) + finally: + tearDownMyTime() + return wrapper -# backwards compatibility to python 2.6: -if not hasattr(unittest, 'SkipTest'): # pragma: no cover - class SkipTest(Exception): - pass - unittest.SkipTest = SkipTest - _org_AddError = unittest._TextTestResult.addError - def addError(self, test, err): - if err[0] is SkipTest: - if self.showAll: - self.stream.writeln(str(err[1])) - elif self.dots: - self.stream.write('s') - self.stream.flush() - return - _org_AddError(self, test, err) - unittest._TextTestResult.addError = addError def initTests(opts): ## if running from installer (setup.py): @@ -280,15 +272,15 @@ def initTests(opts): unittest.F2B.SkipIfFast = F2B_SkipIfFast else: # smaller inertance inside test-cases (litle speedup): - Utils.DEFAULT_SLEEP_TIME = 0.25 - Utils.DEFAULT_SLEEP_INTERVAL = 0.025 + Utils.DEFAULT_SLEEP_TIME = 0.025 + Utils.DEFAULT_SLEEP_INTERVAL = 0.005 Utils.DEFAULT_SHORT_INTERVAL = 0.0005 # sleep intervals are large - use replacement for sleep to check time to sleep: _org_sleep = time.sleep def _new_sleep(v): - if v > max(1, Utils.DEFAULT_SLEEP_TIME): # pragma: no cover + if v > 0.25: # pragma: no cover raise ValueError('[BAD-CODE] To long sleep interval: %s, try to use conditional Utils.wait_for instead' % v) - _org_sleep(min(v, Utils.DEFAULT_SLEEP_TIME)) + _org_sleep(v) time.sleep = _new_sleep # --no-network : if unittest.F2B.no_network: # pragma: no cover @@ -308,6 +300,7 @@ def initTests(opts): # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName + c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) for i in xrange(256): @@ -319,15 +312,28 @@ def initTests(opts): c.set('2001:db8::ffff', 'test-other') c.set('87.142.124.10', 'test-host') if unittest.F2B.no_network: # pragma: no cover - # precache all wrong dns to ip's used in test cases: + # precache all ip to dns used in test cases: + c.set('192.0.2.888', None) + c.set('8.8.4.4', 'dns.google') + c.set('8.8.4.4', 'dns.google') + # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp + c.clear = lambda: logSys.warn('clear CACHE_nameToIp is disabled in test suite') for i in ( - ('999.999.999.999', []), - ('abcdef.abcdef', []), - ('192.168.0.', []), - ('failed.dns.ch', []), + ('999.999.999.999', set()), + ('abcdef.abcdef', set()), + ('192.168.0.', set()), + ('failed.dns.ch', set()), + ('doh1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('1.2.3.4.buga.xxxxx.yyy.invalid', set()), + ('example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), + ('www.example.com', set([IPAddr('2606:2800:220:1:248:1893:25c8:1946'), IPAddr('93.184.216.34')])), ): c.set(*i) + # if fast - precache all host names as localhost addresses (speed-up getSelfIPs/ignoreself): + if unittest.F2B.fast: # pragma: no cover + for i in DNSUtils.getSelfNames(): + c.set(i, DNSUtils.dnsToIp('localhost')) def mtimesleep(): @@ -536,7 +542,7 @@ if not hasattr(unittest.TestCase, 'assertDictEqual'): self.fail(msg) unittest.TestCase.assertDictEqual = assertDictEqual -def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): +def assertSortedEqual(self, a, b, level=1, nestedOnly=False, key=repr, msg=None): """Compare complex elements (like dict, list or tuple) in sorted order until level 0 not reached (initial level = -1 meant all levels), or if nestedOnly set to True and some of the objects still contains nested lists or dicts. @@ -546,6 +552,13 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): if isinstance(v, dict): return any(isinstance(v, (dict, list, tuple)) for v in v.itervalues()) return any(isinstance(v, (dict, list, tuple)) for v in v) + if nestedOnly: + _nest_sorted = sorted + else: + def _nest_sorted(v, key=key): + if isinstance(v, (set, list, tuple)): + return sorted(list(_nest_sorted(v, key) for v in v), key=key) + return v # level comparison routine: def _assertSortedEqual(a, b, level, nestedOnly, key): # first the lengths: @@ -564,8 +577,8 @@ def assertSortedEqual(self, a, b, level=1, nestedOnly=True, key=repr, msg=None): elif v1 != v2: raise ValueError('%r != %r' % (a, b)) else: # list, tuple, something iterable: - a = sorted(a, key=key) - b = sorted(b, key=key) + a = _nest_sorted(a, key=key) + b = _nest_sorted(b, key=key) for v1, v2 in zip(a, b): if isinstance(v1, (dict, list, tuple)) and isinstance(v2, (dict, list, tuple)): _assertSortedEqual(v1, v2, level-1 if level != 0 else 0, nestedOnly, key) @@ -730,30 +743,28 @@ class LogCaptureTestCase(unittest.TestCase): self._dirty |= 2 # records changed def setUp(self): - # For extended testing of what gets output into logging # system, we will redirect it to a string - logSys = getLogger("fail2ban") - # Keep old settings self._old_level = logSys.level self._old_handlers = logSys.handlers # Let's log everything into a string self._log = LogCaptureTestCase._MemHandler(unittest.F2B.log_lazy) logSys.handlers = [self._log] - if self._old_level <= logging.DEBUG: + # lowest log level to capture messages (expected in tests) is Lev.9 + if self._old_level <= logging.DEBUG: # pragma: no cover logSys.handlers += self._old_handlers - else: # lowest log level to capture messages - logSys.setLevel(logging.DEBUG) + if self._old_level > logging.DEBUG-1: + logSys.setLevel(logging.DEBUG-1) super(LogCaptureTestCase, self).setUp() def tearDown(self): """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() self.pruneLog() - logSys = getLogger("fail2ban") + self._log.close() logSys.handlers = self._old_handlers - logSys.level = self._old_level + logSys.setLevel(self._old_level) super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): @@ -834,5 +845,15 @@ class LogCaptureTestCase(unittest.TestCase): def getLog(self): return self._log.getvalue() + @staticmethod + def dumpFile(fn, handle=logSys.debug): + """Helper which outputs content of the file at HEAVYDEBUG loglevels""" + if (handle != logSys.debug or logSys.getEffectiveLevel() <= logging.DEBUG): + handle('---- ' + fn + ' ----') + for line in fileinput.input(fn): + line = line.rstrip('\n') + handle(line) + handle('-'*30) + pid_exists = Utils.pid_exists diff --git a/fail2ban/version.py b/fail2ban/version.py index 573c3e57..96a0760a 100644 --- a/fail2ban/version.py +++ b/fail2ban/version.py @@ -24,7 +24,7 @@ __author__ = "Cyril Jaquier, Yaroslav Halchenko, Steven Hiscocks, Daniel Black" __copyright__ = "Copyright (c) 2004 Cyril Jaquier, 2005-2016 Yaroslav Halchenko, 2013-2014 Steven Hiscocks, Daniel Black" __license__ = "GPL-v2+" -version = "0.11.0.dev2" +version = "1.0.3.dev1" def normVersion(): """ Returns fail2ban version in normalized machine-readable format""" diff --git a/files/debian-initd b/files/debian-initd index 3b1745c1..a9cc584f 100755 --- a/files/debian-initd +++ b/files/debian-initd @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/sh ### BEGIN INIT INFO # Provides: fail2ban # Required-Start: $local_fs $remote_fs @@ -22,28 +22,28 @@ # rename this file: (sudo) mv /etc/init.d/fail2ban.init /etc/init.d/fail2ban # same with the logrotate file: (sudo) mv /etc/logrotate.d/fail2ban.logrotate /etc/logrotate.d/fail2ban # -PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin -DESC="authentication failure monitor" -NAME=fail2ban +PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin" +DESC="Authentication failure monitor" +NAME="fail2ban" # fail2ban-client is not a daemon itself but starts a daemon and # loads its with configuration -DAEMON=/usr/local/bin/$NAME-client -SCRIPTNAME=/etc/init.d/$NAME +DAEMON="/usr/local/bin/$NAME-client" +SCRIPTNAME="/etc/init.d/$NAME" # Ad-hoc way to parse out socket file name -SOCKFILE=`grep -h '^[^#]*socket *=' /etc/$NAME/$NAME.conf /etc/$NAME/$NAME.local 2>/dev/null \ - | tail -n 1 | sed -e 's/.*socket *= *//g' -e 's/ *$//g'` -[ -z "$SOCKFILE" ] && SOCKFILE='/var/run/fail2ban.sock' +SOCKFILE="$(grep -h '^[^#]*socket *=' "/etc/$NAME/$NAME.conf" "/etc/$NAME/$NAME.local" 2>/dev/null \ + | tail -n 1 | sed -e 's/.*socket *= *//g' -e 's/ *$//g')" +[ -z "$SOCKFILE" ] && SOCKFILE="/var/run/fail2ban.sock" # Exit if the package is not installed [ -x "$DAEMON" ] || exit 0 # Run as root by default. -FAIL2BAN_USER=root +FAIL2BAN_USER="root" # Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME +[ -r "/etc/default/$NAME" ] && . "/etc/default/$NAME" DAEMON_ARGS="$FAIL2BAN_OPTS" # Load the VERBOSE setting and other rcS variables @@ -51,7 +51,8 @@ DAEMON_ARGS="$FAIL2BAN_OPTS" # Predefine what can be missing from lsb source later on -- necessary to run # on sarge. Just present it in a bit more compact way from what was shipped -log_daemon_msg () { +log_daemon_msg() +{ [ -z "$1" ] && return 1 echo -n "$1:" [ -z "$2" ] || echo -n " $2" @@ -68,7 +69,7 @@ log_daemon_msg () { # report_bug() { - echo $* + echo "$*" echo "Please submit a bug report to Debian BTS (reportbug fail2ban)" exit 1 } @@ -80,10 +81,10 @@ report_bug() check_socket() { # Return - # 0 if socket is present and readable - # 1 if socket file is not present - # 2 if socket file is present but not readable - # 3 if socket file is present but is not a socket + # 0 if socket is present and readable + # 1 if socket file is not present + # 2 if socket file is present but not readable + # 3 if socket file is present but is not a socket [ -e "$SOCKFILE" ] || return 1 [ -r "$SOCKFILE" ] || return 2 [ -S "$SOCKFILE" ] || return 3 @@ -96,14 +97,14 @@ check_socket() do_start() { # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started + # 0 if daemon has been started + # 1 if daemon was already running + # 2 if daemon could not be started do_status && return 1 if [ -e "$SOCKFILE" ]; then log_failure_msg "Socket file $SOCKFILE is present" - [ "$1" = "force-start" ] \ + [ "$1" = force-start ] \ && log_success_msg "Starting anyway as requested" \ || return 2 DAEMON_ARGS="$DAEMON_ARGS -x" @@ -112,18 +113,20 @@ do_start() # Assure that /var/run/fail2ban exists [ -d /var/run/fail2ban ] || mkdir -p /var/run/fail2ban - if [ "$FAIL2BAN_USER" != "root" ]; then + if [ "$FAIL2BAN_USER" != root ]; then # Make the socket directory, IP lists and fail2ban log # files writable by fail2ban chown "$FAIL2BAN_USER" /var/run/fail2ban # Create the logfile if it doesn't exist touch /var/log/fail2ban.log chown "$FAIL2BAN_USER" /var/log/fail2ban.log - find /proc/net/xt_recent -name 'fail2ban-*' -exec chown "$FAIL2BAN_USER" {} \; + find /proc/net/xt_recent -name "fail2ban-*" -exec chown "$FAIL2BAN_USER" "{}" ";" fi - start-stop-daemon --start --quiet --chuid "$FAIL2BAN_USER" --exec $DAEMON -- \ - $DAEMON_ARGS start > /dev/null\ + # $DAEMON_ARGS need to be expanded possibly with multiple or no options + # shellcheck disable=SC2086 + start-stop-daemon --start --quiet --chuid "$FAIL2BAN_USER" --exec "$DAEMON" -- \ + $DAEMON_ARGS start >/dev/null \ || return 2 return 0 @@ -136,8 +139,8 @@ do_start() # do_status() { - $DAEMON ping > /dev/null 2>&1 - return $? + $DAEMON ping >/dev/null 2>&1 + return "$?" } # @@ -146,22 +149,22 @@ do_status() do_stop() { # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - $DAEMON status > /dev/null 2>&1 || return 1 - $DAEMON stop > /dev/null || return 2 + # 0 if daemon has been stopped + # 1 if daemon was already stopped + # 2 if daemon could not be stopped + # other if a failure occurred + $DAEMON status >/dev/null 2>&1 || return 1 + $DAEMON stop >/dev/null || return 2 # now we need actually to wait a bit since it might take time # for server to react on client's stop request. Especially # important for restart command on slow boxes count=1 - while do_status && [ $count -lt 60 ]; do + while do_status && [ "$count" -lt 60 ]; do sleep 1 - count=$(($count+1)) + count="$((count + 1))" done - [ $count -lt 60 ] || return 3 # failed to stop + [ "$count" -lt 60 ] || return 3 # failed to stop return 0 } @@ -169,8 +172,9 @@ do_stop() # # Function to reload configuration # -do_reload() { - $DAEMON reload > /dev/null && return 0 || return 1 +do_reload() +{ + "$DAEMON" reload >/dev/null && return 0 || return 1 return 0 } @@ -180,16 +184,16 @@ do_reload() { # log_end_msg_wrapper() { - if [ $1 != 0 ] && [ $1 != $2 ]; then - value=1 + if [ "$1" != 0 ] && [ "$1" != "$2" ]; then + value="1" else - value=0 + value="0" fi - if [ "$3" != "no" ]; then - log_end_msg $value + if [ "$3" != no ]; then + log_end_msg "$value" fi - if [ $value != "0" ]; then - exit $1 + if [ "$value" != 0 ]; then + exit "$1" fi } @@ -198,13 +202,13 @@ case "$command" in start|force-start) [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" do_start "$command" - log_end_msg_wrapper $? 255 "$VERBOSE" + log_end_msg_wrapper "$?" 255 "$VERBOSE" ;; stop) [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" do_stop - log_end_msg_wrapper $? 255 "$VERBOSE" + log_end_msg_wrapper "$?" 255 "$VERBOSE" ;; restart|force-reload) @@ -213,41 +217,55 @@ case "$command" in case "$?" in 0|1) do_start - log_end_msg_wrapper $? 0 "always" + log_end_msg_wrapper "$?" 0 always ;; *) # Failed to stop log_end_msg 1 ;; - esac + esac ;; - reload|force-reload) - log_daemon_msg "Reloading $DESC" "$NAME" - do_reload - log_end_msg $? - ;; + reload) + log_daemon_msg "Reloading $DESC" "$NAME" + do_reload + log_end_msg "$?" + ;; status) log_daemon_msg "Status of $DESC" do_status - case $? in - 0) log_success_msg " $NAME is running" ;; + case "$?" in + 0) + log_success_msg " $NAME is running" + ;; 255) check_socket - case $? in - 1) log_failure_msg " $NAME is not running" && exit 3 ;; - 0) log_failure_msg " $NAME is not running but $SOCKFILE exists" && exit 3 ;; - 2) log_failure_msg " $SOCKFILE not readable, status of $NAME is unknown" && exit 3 ;; - 3) log_failure_msg " $SOCKFILE exists but not a socket, status of $NAME is unknown" && exit 3 ;; - *) report_bug "Unknown return code from $NAME:check_socket." && exit 4 ;; + case "$?" in + 1) + log_failure_msg " $NAME is not running" && exit 3 + ;; + 0) + log_failure_msg " $NAME is not running but $SOCKFILE exists" && exit 3 + ;; + 2) + log_failure_msg " $SOCKFILE not readable, status of $NAME is unknown" && exit 3 + ;; + 3) + log_failure_msg " $SOCKFILE exists but not a socket, status of $NAME is unknown" && exit 3 + ;; + *) + report_bug "Unknown return code from $NAME:check_socket." && exit 4 + ;; esac ;; - *) report_bug "Unknown $NAME status code" && exit 4 + *) + report_bug "Unknown $NAME status code" && exit 4 + ;; esac ;; *) - echo "Usage: $SCRIPTNAME {start|force-start|stop|restart|force-reload|status}" >&2 + echo "Usage: $SCRIPTNAME {start|force-start|stop|restart|force-reload|status}" 1>&2 exit 3 ;; esac diff --git a/files/fail2ban-openrc.conf b/files/fail2ban-openrc.conf new file mode 100644 index 00000000..9454ef68 --- /dev/null +++ b/files/fail2ban-openrc.conf @@ -0,0 +1,2 @@ +# For available options, plase run "fail2ban-server --help". +#FAIL2BAN_OPTIONS="-x" diff --git a/files/fail2ban-openrc.init.in b/files/fail2ban-openrc.init.in new file mode 100755 index 00000000..2c56ee3a --- /dev/null +++ b/files/fail2ban-openrc.init.in @@ -0,0 +1,86 @@ +#!/sbin/openrc-run +# This file is part of Fail2Ban. +# +# Fail2Ban is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Fail2Ban is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Fail2Ban; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Author: Sireyessire, Cyril Jaquier +# + +description="Ban hosts that cause multiple authentication errors" +description_reload="reload configuration without dropping bans" +extra_started_commands="reload" + +# Can't (and shouldn't) be changed by the end-user. +# +# Note that @BINDIR@ is already supplied by the build system. Some +# day, it might be nice to have @RUNDIR@ supplied by the build system +# as well, so that we don't have to hard-code /run here. +FAIL2BAN_RUNDIR="/run/${RC_SVCNAME}" +FAIL2BAN_SOCKET="${FAIL2BAN_RUNDIR}/${RC_SVCNAME}.sock" + +# The fail2ban-client program is also capable of starting and stopping +# the server, but things are simpler if we let start-stop-daemon do it. +command="@BINDIR@/fail2ban-server" +pidfile="${FAIL2BAN_RUNDIR}/${RC_SVCNAME}.pid" + +# We force the pidfile/socket location in this service script because +# we're taking responsibility for ensuring that their parent directory +# exists and has the correct permissions (which we can't do if the +# user is allowed to change them). +command_args="${FAIL2BAN_OPTIONS} -p ${pidfile} -s ${FAIL2BAN_SOCKET}" +retry="30" + +depend() { + use logger + after iptables +} + +checkconfig() { + "${command}" ${command_args} --test +} + +start_pre() { + # If this isn't a restart, make sure that the user's config isn't + # busted before we try to start the daemon (this will produce + # better error messages than if we just try to start it blindly). + # + # If, on the other hand, this *is* a restart, then the stop_pre + # action will have ensured that the config is usable and we don't + # need to do that again. + if [ "${RC_CMD}" != "restart" ] ; then + checkconfig || return $? + fi + checkpath -d "${FAIL2BAN_RUNDIR}" +} + +stop_pre() { + # If this is a restart, check to make sure the user's config + # isn't busted before we stop the running daemon. + if [ "${RC_CMD}" = "restart" ] ; then + checkconfig || return $? + fi +} + +reload() { + # The fail2ban-client uses an undocumented protocol to tell + # the server to reload(), so we have to use it here rather + # than e.g. sending a signal to the server daemon. Note that + # the reload will fail (on the server side) if the new config + # is invalid; we therefore don't need to test it ourselves + # with checkconfig() before initiating the reload. + ebegin "Reloading ${RC_SVCNAME}" + "@BINDIR@/fail2ban-client" ${command_args} reload + eend $? "Failed to reload ${RC_SVCNAME}" +} diff --git a/files/fail2ban-tmpfiles.conf b/files/fail2ban-tmpfiles.conf index 3fd783f3..68f8e345 100644 --- a/files/fail2ban-tmpfiles.conf +++ b/files/fail2ban-tmpfiles.conf @@ -1 +1 @@ -D /var/run/fail2ban 0755 root root -
\ No newline at end of file +D /run/fail2ban 0755 root root -
\ No newline at end of file diff --git a/files/fail2ban.service.in b/files/fail2ban.service.in index 24dcb51e..9a245c61 100644 --- a/files/fail2ban.service.in +++ b/files/fail2ban.service.in @@ -1,18 +1,19 @@ [Unit] Description=Fail2Ban Service Documentation=man:fail2ban(1) -After=network.target iptables.service firewalld.service ip6tables.service ipset.service -PartOf=iptables.service firewalld.service ip6tables.service ipset.service +After=network.target iptables.service firewalld.service ip6tables.service ipset.service nftables.service +PartOf=iptables.service firewalld.service ip6tables.service ipset.service nftables.service [Service] Type=simple -ExecStartPre=/bin/mkdir -p /var/run/fail2ban +Environment="PYTHONNOUSERSITE=1" +ExecStartPre=/bin/mkdir -p /run/fail2ban ExecStart=@BINDIR@/fail2ban-server -xf start # if should be logged in systemd journal, use following line or set logtarget to sysout in fail2ban.local # ExecStart=@BINDIR@/fail2ban-server -xf --logtarget=sysout start ExecStop=@BINDIR@/fail2ban-client stop ExecReload=@BINDIR@/fail2ban-client reload -PIDFile=/var/run/fail2ban/fail2ban.pid +PIDFile=/run/fail2ban/fail2ban.pid Restart=on-failure RestartPreventExitStatus=0 255 diff --git a/files/gentoo-confd b/files/gentoo-confd deleted file mode 100644 index 00d19f8b..00000000 --- a/files/gentoo-confd +++ /dev/null @@ -1,8 +0,0 @@ -# Config file for /etc/init.d/fail2ban -# -# For information on options, see "/usr/bin/fail2ban-client -h". - -FAIL2BAN_OPTIONS="" - -# Force execution of the server even if the socket already exists: -#FAIL2BAN_OPTIONS="-x" diff --git a/files/gentoo-initd b/files/gentoo-initd deleted file mode 100755 index 0fb157cd..00000000 --- a/files/gentoo-initd +++ /dev/null @@ -1,60 +0,0 @@ -#!/sbin/openrc-run -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# Author: Sireyessire, Cyril Jaquier -# - -description="Daemon to ban hosts that cause multiple authentication errors" -description_reload="reload configuration" -description_showlog="show fail2ban logs" -extra_started_commands="reload showlog" - -FAIL2BAN="/usr/bin/fail2ban-client ${FAIL2BAN_OPTIONS}" - -depend() { - need net - need logger - after iptables -} - -start() { - ebegin "Starting fail2ban" - mkdir -p /var/run/fail2ban || return 1 - # remove stalled sock file after system crash - # bug 347477 - rm -f /var/run/fail2ban/fail2ban.sock || return 1 - start-stop-daemon --start --pidfile /var/run/fail2ban/fail2ban.pid \ - -- ${FAIL2BAN} start - eend $? "Failed to start fail2ban" -} - -stop() { - ebegin "Stopping fail2ban" - start-stop-daemon --stop --pidfile /var/run/fail2ban/fail2ban.pid --retry 30 \ - -- ${FAIL2BAN} stop - eend $? "Failed to stop fail2ban" -} - -reload() { - ebegin "Reloading fail2ban" - ${FAIL2BAN} reload - eend $? "Failed to reload fail2ban" -} - -showlog(){ - less /var/log/fail2ban.log -} diff --git a/man/fail2ban-client.1 b/man/fail2ban-client.1 index 07d9e650..9fa0b803 100644 --- a/man/fail2ban-client.1 +++ b/man/fail2ban-client.1 @@ -1,24 +1,27 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-CLIENT "1" "April 2018" "fail2ban-client v0.11.0.dev2" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.48.1. +.TH FAIL2BAN-CLIENT "1" "November 2022" "Fail2Ban v1.0.3.dev1" "User Commands" .SH NAME fail2ban-client \- configure and control the server .SH SYNOPSIS .B fail2ban-client [\fI\,OPTIONS\/\fR] \fI\,<COMMAND>\/\fR .SH DESCRIPTION -Fail2Ban v0.11.0.dev2 reads log file that contains password failure report +Fail2Ban v1.0.3.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR <DIR> +\fB\-c\fR, \fB\-\-conf\fR <DIR> configuration directory .TP -\fB\-s\fR <FILE> +\fB\-s\fR, \fB\-\-socket\fR <FILE> socket path .TP -\fB\-p\fR <FILE> +\fB\-p\fR, \fB\-\-pidfile\fR <FILE> pidfile path .TP +\fB\-\-pname\fR <NAME> +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR <LEVEL> logging level .TP @@ -67,7 +70,7 @@ convert time abbreviation format to seconds display this help message .TP \fB\-V\fR, \fB\-\-version\fR -print the version +print the version (\fB\-V\fR returns machine\-readable short format) .SH COMMAND .IP BASIC @@ -108,6 +111,14 @@ jails and database) unbans <IP> (in all jails and database) .TP +\fBbanned\fR +return jails with banned IPs as +dictionary +.TP +\fBbanned <IP> ... <IP>]\fR +return list(s) of jails where +given IP(s) are banned +.TP \fBstatus\fR gets the current status of the server @@ -139,8 +150,8 @@ gets the logging level .TP \fBset logtarget <TARGET>\fR sets logging target to <TARGET>. -Can be STDOUT, STDERR, SYSLOG or a -file +Can be STDOUT, STDERR, SYSLOG, +SYSTEMD\-JOURNAL or a file .TP \fBget logtarget\fR gets logging target @@ -168,6 +179,14 @@ persistent datastore. Set to get the location of fail2ban persistent datastore .TP +\fBset dbmaxmatches <INT>\fR +sets the max number of matches +stored in database per ticket +.TP +\fBget dbmaxmatches\fR +gets the max number of matches +stored in database per ticket +.TP \fBset dbpurgeage <SECONDS>\fR sets the max age in <SECONDS> that history of bans will be kept @@ -210,6 +229,12 @@ adds <IP> to the ignore list of removes <IP> from the ignore list of <JAIL> .TP +\fBset <JAIL> ignorecommand <VALUE>\fR +sets ignorecommand of <JAIL> +.TP +\fBset <JAIL> ignorecache <VALUE>\fR +sets ignorecache of <JAIL> +.TP \fBset <JAIL> addlogpath <FILE> ['tail']\fR adds <FILE> to the monitoring list of <JAIL>, optionally starting at @@ -241,9 +266,6 @@ for <JAIL> removes the regular expression at <INDEX> for failregex .TP -\fBset <JAIL> ignorecommand <VALUE>\fR -sets ignorecommand of <JAIL> -.TP \fBset <JAIL> addignoreregex <REGEX>\fR adds the regular expression <REGEX> which should match pattern @@ -269,10 +291,13 @@ date/times for <JAIL> \fBset <JAIL> usedns <VALUE>\fR sets the usedns mode for <JAIL> .TP -\fBset <JAIL> banip <IP>\fR +\fBset <JAIL> attempt <IP> [<failure1> ... <failureN>]\fR +manually notify about <IP> failure +.TP +\fBset <JAIL> banip <IP> ... <IP>\fR manually Ban <IP> for <JAIL> .TP -\fBset <JAIL> unbanip <IP>\fR +\fBset <JAIL> unbanip [\-\-report\-absent] <IP> ... <IP>\fR manually Unban <IP> in <JAIL> .TP \fBset <JAIL> maxretry <RETRY>\fR @@ -280,6 +305,11 @@ sets the number of failures <RETRY> before banning the host for <JAIL> .TP +\fBset <JAIL> maxmatches <INT>\fR +sets the max number of matches +stored in memory per ticket in +<JAIL> +.TP \fBset <JAIL> maxlines <LINES>\fR sets the number of <LINES> to buffer for regex search for <JAIL> @@ -334,6 +364,14 @@ for <JAIL> .IP JAIL INFORMATION .TP +\fBget <JAIL> banned\fR +return banned IPs of <JAIL> +.TP +\fBget <JAIL> banned <IP> ... <IP>]\fR +return 1 if IP is banned in <JAIL> +otherwise 0, or a list of 1/0 for +multiple IPs +.TP \fBget <JAIL> logpath\fR gets the list of the monitored files for <JAIL> @@ -377,16 +415,30 @@ gets the time a host is banned for <JAIL> .TP \fBget <JAIL> datepattern\fR -gets the patern used to match +gets the pattern used to match date/times for <JAIL> .TP \fBget <JAIL> usedns\fR gets the usedns setting for <JAIL> .TP +\fBget <JAIL> banip [<SEP>|\-\-with\-time]\fR +gets the list of of banned IP +addresses for <JAIL>. Optionally +the separator character ('<SEP>', +default is space) or the option +\&'\-\-with\-time' (printing the times +of ban) may be specified. The IPs +are ordered by end of ban. +.TP \fBget <JAIL> maxretry\fR gets the number of failures allowed for <JAIL> .TP +\fBget <JAIL> maxmatches\fR +gets the max number of matches +stored in memory per ticket in +<JAIL> +.TP \fBget <JAIL> maxlines\fR gets the number of lines to buffer for <JAIL> @@ -438,11 +490,6 @@ the action <ACT> for <JAIL> \fI/etc/fail2ban/*\fR .SH "REPORTING BUGS" Report bugs to https://github.com/fail2ban/fail2ban/issues -.SH COPYRIGHT -Copyright \(co 2004\-2008 Cyril Jaquier, 2008\- Fail2Ban Contributors -.br -Copyright of modifications held by their respective authors. -Licensed under the GNU General Public License v2 (GPL). .SH "SEE ALSO" .br fail2ban-server(1) diff --git a/man/fail2ban-python.1 b/man/fail2ban-python.1 index 9880726e..f40c160f 100644 --- a/man/fail2ban-python.1 +++ b/man/fail2ban-python.1 @@ -1,10 +1,14 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-PYTHON "1" "April 2018" "fail2ban-python f2bversion" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.48.1. +.TH FAIL2BAN-PYTHON "1" "November 2022" "fail2ban-python 1.0.3.1" "User Commands" .SH NAME fail2ban-python \- a helper for Fail2Ban to assure that the same Python is used .SH DESCRIPTION -usage: ../bin/fail2ban\-python [option] ... [\-c cmd | \fB\-m\fR mod | file | \fB\-]\fR [arg] ... +usage: fail2ban\-python [option] ... [\-c cmd | \fB\-m\fR mod | file | \fB\-]\fR [arg] ... Options and arguments (and corresponding environment variables): +\fB\-b\fR : issue warnings about comparing bytearray with unicode +.IP +(\fB\-bb\fR: issue errors) +.PP \fB\-B\fR : don't write .py[co] files on import; also PYTHONDONTWRITEBYTECODE=x \fB\-c\fR cmd : program passed in as string (terminates option list) \fB\-d\fR : debug output from parser; also PYTHONDEBUG=x diff --git a/man/fail2ban-regex.1 b/man/fail2ban-regex.1 index 307cc0be..40e8a1e8 100644 --- a/man/fail2ban-regex.1 +++ b/man/fail2ban-regex.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-REGEX "1" "April 2018" "fail2ban-regex 0.11.0.dev2" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.48.1. +.TH FAIL2BAN-REGEX "1" "November 2022" "fail2ban-regex 1.0.3.dev1" "User Commands" .SH NAME fail2ban-regex \- test Fail2ban "failregex" option .SH SYNOPSIS @@ -18,13 +18,18 @@ a string representing a log line filename path to a log file (\fI\,/var/log/auth.log\/\fP) .TP -"systemd\-journal" -search systemd journal (systemd\-python required) +systemd\-journal +search systemd journal (systemd\-python required), +optionally with backend parameters, see `man jail.conf` +for usage and examples (systemd\-journal[journalflags=1]). .SS "REGEX:" .TP string a string representing a 'failregex' .TP +filter +name of filter, optionally with options (sshd[mode=aggressive]) +.TP filename path to a filter file (filter.d/sshd.conf) .SS "IGNOREREGEX:" @@ -72,6 +77,9 @@ journalctl style matches overriding filter file. \fB\-l\fR LOG_LEVEL, \fB\-\-log\-level\fR=\fI\,LOG_LEVEL\/\fR Log level for the Fail2Ban logger to use .TP +\fB\-V\fR +get version in machine\-readable short format +.TP \fB\-v\fR, \fB\-\-verbose\fR Increase verbosity .TP @@ -84,6 +92,13 @@ Verbose date patterns/regex in output \fB\-D\fR, \fB\-\-debuggex\fR Produce debuggex.com urls for debugging there .TP +\fB\-\-no\-check\-all\fR +Disable check for all regex's +.TP +\fB\-o\fR OUT, \fB\-\-out\fR=\fI\,OUT\/\fR +Set token to print failure information only (row, id, +ip, msg, host, ip4, ip6, dns, matches, ...) +.TP \fB\-\-print\-no\-missed\fR Do not print any missed lines .TP @@ -107,7 +122,7 @@ Either to make the tracebacks full, not compressed (as by default) .SH AUTHOR Written by Cyril Jaquier <cyril.jaquier@fail2ban.org>. -Many contributions by Yaroslav O. Halchenko and Steven Hiscocks. +Many contributions by Yaroslav O. Halchenko, Steven Hiscocks, Sergey G. Brester (sebres). .SH "REPORTING BUGS" Report bugs to https://github.com/fail2ban/fail2ban/issues .SH COPYRIGHT @@ -119,3 +134,4 @@ Licensed under the GNU General Public License v2 (GPL). .br fail2ban-client(1) fail2ban-server(1) +jail.conf(5) diff --git a/man/fail2ban-regex.h2m b/man/fail2ban-regex.h2m index dfbf9330..91992994 100644 --- a/man/fail2ban-regex.h2m +++ b/man/fail2ban-regex.h2m @@ -8,3 +8,4 @@ fail2ban-regex \- test Fail2ban "failregex" option .br fail2ban-client(1) fail2ban-server(1) +jail.conf(5) diff --git a/man/fail2ban-server.1 b/man/fail2ban-server.1 index 72bf3522..eb404dc9 100644 --- a/man/fail2ban-server.1 +++ b/man/fail2ban-server.1 @@ -1,24 +1,27 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-SERVER "1" "April 2018" "fail2ban-server v0.11.0.dev2" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.48.1. +.TH FAIL2BAN-SERVER "1" "November 2022" "Fail2Ban v1.0.3.dev1" "User Commands" .SH NAME fail2ban-server \- start the server .SH SYNOPSIS .B fail2ban-server [\fI\,OPTIONS\/\fR] .SH DESCRIPTION -Fail2Ban v0.11.0.dev2 reads log file that contains password failure report +Fail2Ban v1.0.3.dev1 reads log file that contains password failure report and bans the corresponding IP addresses using firewall rules. .SH OPTIONS .TP -\fB\-c\fR <DIR> +\fB\-c\fR, \fB\-\-conf\fR <DIR> configuration directory .TP -\fB\-s\fR <FILE> +\fB\-s\fR, \fB\-\-socket\fR <FILE> socket path .TP -\fB\-p\fR <FILE> +\fB\-p\fR, \fB\-\-pidfile\fR <FILE> pidfile path .TP +\fB\-\-pname\fR <NAME> +name of the process (main thread) to identify instance (default fail2ban\-server) +.TP \fB\-\-loglevel\fR <LEVEL> logging level .TP @@ -67,14 +70,9 @@ convert time abbreviation format to seconds display this help message .TP \fB\-V\fR, \fB\-\-version\fR -print the version +print the version (\fB\-V\fR returns machine\-readable short format) .SH "REPORTING BUGS" Report bugs to https://github.com/fail2ban/fail2ban/issues -.SH COPYRIGHT -Copyright \(co 2004\-2008 Cyril Jaquier, 2008\- Fail2Ban Contributors -.br -Copyright of modifications held by their respective authors. -Licensed under the GNU General Public License v2 (GPL). .SH "SEE ALSO" .br fail2ban-client(1) diff --git a/man/fail2ban-testcases.1 b/man/fail2ban-testcases.1 index f77855dc..236e7bd8 100644 --- a/man/fail2ban-testcases.1 +++ b/man/fail2ban-testcases.1 @@ -1,5 +1,5 @@ -.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.4. -.TH FAIL2BAN-TESTCASES "1" "April 2018" "fail2ban-testcases 0.11.0.dev2" "User Commands" +.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.48.1. +.TH FAIL2BAN-TESTCASES "1" "November 2022" "fail2ban-testcases 1.0.3.dev1" "User Commands" .SH NAME fail2ban-testcases \- run Fail2Ban unit-tests .SH SYNOPSIS diff --git a/man/generate-man b/man/generate-man index f7a7836c..538588a9 100755 --- a/man/generate-man +++ b/man/generate-man @@ -3,12 +3,14 @@ set -eu export PYTHONPATH=.. +export PATH="../bin:$PATH" -f2bversion=$(../bin/fail2ban-client --version | head -n1 | sed -e 's,.* v,,g') +f2bversion=$(fail2ban-client -V) +echo "Generating man for $f2bversion ..." # fail2ban-client echo -n "Generating fail2ban-client " -help2man --section=1 --no-info --include=fail2ban-client.h2m --output fail2ban-client.1 ../bin/fail2ban-client +help2man --section=1 --no-info --include=fail2ban-client.h2m --output fail2ban-client.1 fail2ban-client echo "[done]" echo -n "Patching fail2ban-client " # Changes the title. @@ -39,22 +41,22 @@ echo "[done]" # fail2ban-python echo -n "Generating fail2ban-python " -help2man --version-string=f2bversion --section=1 --no-info --include=fail2ban-python.h2m --output fail2ban-python.1 ../bin/fail2ban-python +help2man --version-string=$f2bversion --section=1 --no-info --include=fail2ban-python.h2m --output fail2ban-python.1 fail2ban-python echo "[done]" # fail2ban-server echo -n "Generating fail2ban-server " -help2man --section=1 --no-info --include=fail2ban-server.h2m --output fail2ban-server.1 ../bin/fail2ban-server +help2man --section=1 --no-info --include=fail2ban-server.h2m --output fail2ban-server.1 fail2ban-server echo "[done]" # fail2ban-testcases echo -n "Generating fail2ban-testcases " -help2man --section=1 --no-info --include=fail2ban-testcases.h2m --output fail2ban-testcases.1 ../bin/fail2ban-testcases +help2man --section=1 --no-info --include=fail2ban-testcases.h2m --output fail2ban-testcases.1 fail2ban-testcases echo "[done]" # fail2ban-regex echo -n "Generating fail2ban-regex " -help2man --section=1 --no-info --include=fail2ban-regex.h2m --output fail2ban-regex.1 ../bin/fail2ban-regex +help2man --section=1 --no-info --include=fail2ban-regex.h2m --output fail2ban-regex.1 fail2ban-regex echo "[done]" echo -n "Patching fail2ban-regex " # Changes the title. diff --git a/man/jail.conf.5 b/man/jail.conf.5 index 25b3dd70..052fce80 100644 --- a/man/jail.conf.5 +++ b/man/jail.conf.5 @@ -127,12 +127,10 @@ Comments: use '#' for comment lines and '; ' (space is important) for inline com .SH "FAIL2BAN CONFIGURATION FILE(S) (\fIfail2ban.conf\fB)" -These files have one section, [Definition]. - -The items that can be set are: +The items that can be set in section [Definition] are: .TP .B loglevel -verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: ERROR (equal 40) +verbosity level of log output: CRITICAL, ERROR, WARNING, NOTICE, INFO, DEBUG, TRACEDEBUG, HEAVYDEBUG or corresponding numeric value (50-5). Default: INFO (equal 20) .TP .B logtarget log target: filename, SYSLOG, STDERR or STDOUT. Default: STDOUT if not set in fail2ban.conf/fail2ban.local @@ -153,16 +151,35 @@ PID filename. Default: /var/run/fail2ban/fail2ban.pid .br This is used to store the process ID of the fail2ban server. .TP +.B allowipv6 +option to allow IPv6 interface - auto, yes (on, true, 1) or no (off, false, 0). Default: auto +.br +This value can be used to declare fail2ban whether IPv6 is allowed or not. +.TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 .br This defines where the persistent data for fail2ban is stored. This persistent data allows bans to be reinstated and continue reading log files from the last read position when fail2ban is restarted. A value of \fINone\fR disables this feature. .TP +.B dbmaxmatches +Max number of matches stored in database per ticket. Default: 10 +.br +This option sets the max number of matched log-lines could be stored per ticket in the database. This also affects values resolvable via tags \fB<ipmatches>\fR and \fB<ipjailmatches>\fR in actions. +.TP .B dbpurgeage Database purge age in seconds. Default: 86400 (24hours) .br This sets the age at which bans should be purged from the database. +.RE +The config parameters of section [Thread] are: + +.TP +.B stacksize +Stack size of each thread in fail2ban. Default: 0 (platform or configured default) +.br +This specifies the stack size (in KiB) to be used for subsequently created threads, and must be 0 or a positive integer value of at least 32. + .SH "JAIL CONFIGURATION FILE(S) (\fIjail.conf\fB)" The following options are applicable to any jail. They appear in a section specifying the jail name or in the \fI[DEFAULT]\fR section which defines default values to be used if not specified in the individual section. .TP @@ -254,7 +271,7 @@ effective ban duration (in seconds or time abbreviation format). time interval (in seconds or time abbreviation format) before the current time where failures will count towards a ban. .TP .B maxretry -number of failures that have to occur in the last \fBfindtime\fR seconds to ban then IP. +number of failures that have to occur in the last \fBfindtime\fR seconds to ban the IP. .TP .B backend backend to be used to detect changes in the logpath. @@ -264,11 +281,17 @@ It defaults to "auto" which will try "pyinotify", "gamin", "systemd" before "pol .B usedns use DNS to resolve HOST names that appear in the logs. By default it is "warn" which will resolve hostnames to IPs however it will also log a warning. If you are using DNS here you could be blocking the wrong IPs due to the asymmetric nature of reverse DNS (that the application used to write the domain name to log) compared to forward DNS that fail2ban uses to resolve this back to an IP (but not necessarily the same one). Ideally you should configure your applications to log a real IP. This can be set to "yes" to prevent warnings in the log or "no" to disable DNS resolution altogether (thus ignoring entries where hostname, not an IP is logged).. .TP +.B prefregex +regex (Python \fBreg\fRular \fBex\fRpression) to parse a common part containing in every message (see \fBprefregex\fR in section FILTER FILES for details). +.TP .B failregex -regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes. If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below). +regex (Python \fBreg\fRular \fBex\fRpression) to be added to the filter's failregexes (see \fBfailregex\fR in section FILTER FILES for details). If this is useful for others using your application please share you regular expression with the fail2ban developers by reporting an issue (see REPORTING BUGS below). .TP .B ignoreregex regex which, if the log line matches, would cause Fail2Ban not consider that line. This line will be ignored even if it matches a failregex of the jail or any of its filters. +.TP +.B maxmatches +max number of matched log-lines the jail would hold in memory per ticket. By default it is the same value as \fBmaxretry\fR of jail (or default). This option also affects values resolvable via tag \fB<matches>\fR in actions. .SS Backends Available options are listed below. @@ -283,7 +306,14 @@ requires Gamin (a file alteration monitor) to be installed. If Gamin is not inst uses a polling algorithm which does not require external libraries. .TP .B systemd -uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. +uses systemd python library to access the systemd journal. Specifying \fBlogpath\fR is not valid for this backend and instead utilises \fBjournalmatch\fR from the jails associated filter config. Multiple systemd-specific flags can be passed to the backend, including \fBjournalpath\fR and \fBjournalfiles\fR, to explicitly set the path to a directory or set of files. \fBjournalflags\fR, which by default is 4 and excludes user session files, can be set to include them with \fBjournalflags=1\fR, see the python-systemd documentation for other settings and further details. Examples: +.PP +.RS +.nf +backend = systemd[journalpath=/run/log/journal/machine-1] +backend = systemd[journalfiles="/path/to/system.journal, /path/to/user.journal"] +backend = systemd[journalflags=1] +.fi .SS Actions Each jail can be configured with only a single filter, but may have multiple actions. By default, the name of a action is the action filename, and in the case of Python actions, the ".py" file extension is stripped. Where multiple of the same action are to be used, the \fBactname\fR option can be assigned to the action to avoid duplication e.g.: @@ -410,35 +440,115 @@ These are used to identify failed authentication attempts in log files and to ex Like action files, filter files are ini files. The main section is the [Definition] section. -There are two filter definitions used in the [Definition] section: +There are several standard filter definitions used in the [Definition] section: +.TP +.B prefregex +is the regex (\fBreg\fRular \fBex\fRpression) to parse a common part containing in every message, which is applied after \fBdatepattern\fR found a match, before the search for any \fBfailregex\fR or \fBignoreregex\fR would start. +.br +If this regex doesn't match the process is starting immediately with next message and search for any \fBfailregex\fR does not occur. +.br +If \fBprefregex\fR contains \fI<F-CONTENT>...</F-CONTENT>\fR, the part of message enclosed between this tags will be extracted and herafter used as whole message for search with \fBfailregex\fR or \fBignoreregex\fR. +.IP +For example: +.nf + prefregex = ^%(__prefix_line)s (?:ERROR|FAILURE) <F-CONTENT>.+</F-CONTENT>$ + failregex = ^user not found + ^authentication failed + ^unknown authentication method +.fi +.IP +You can use \fBprefregex\fR in order to: +.RS +.IP +- specify 1 common regex to match some common part present in every messages (do avoid unneeded match in every \fBfailregex\fR if you have more as one); +.IP +- to cut some interesting part of message only (to simplify \fBfailregex\fR) enclosed between tags \fI<F-CONTENT>\fI and \fI</F-CONTENT>\fR; +.IP +- to gather some failure identifier (e. g. some prefix matched by \fI<F-MLFID>...<F-MLFID/>\fR tag) to identify several messages belonging to same session, where a connect message containing IP followed by failure message(s) that are not contain IP; +this provides a new multi-line parsing method as replacement for old (slow an ugly) multi-line parsing using buffering window (\fImaxlines\fR > 1 and \fI<SKIPLINES>\fR); +.IP +- to ignore some wrong, too long or even unneeded messages (a.k.a. parasite log traffic) which can be also present in journal, before \fBfailregex\fR search would take place. +.RE + .TP .B failregex -is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The tag \fI<HOST>\fR is used as part of the regex and is itself a regex -for IPv4 addresses (and hostnames if \fBusedns\fR). Fail2Ban will work out which one of these it actually is. -For multiline regexs the tag \fI<SKIPLINES>\fR should be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. +is the regex (\fBreg\fRular \fBex\fRpression) that will match failed attempts. The standard replacement tags can be used as part of the regex: +.RS +.IP +\fI<HOST>\fR - common regex for IP addresses and hostnames (if \fBusedns\fR is enabled). Fail2Ban will work out which one of these it actually is. +.IP +\fI<ADDR>\fR - regex for IP addresses (both families). +.IP +\fI<IP4>\fR - regex for IPv4 addresses. +.IP +\fI<IP6>\fR - regex for IPv6 addresses. +.IP +\fI<DNS>\fR - regex to match hostnames. +.IP +\fI<CIDR>\fR - helper regex to match CIDR (simple integer form of net-mask). +.IP +\fI<SUBNET>\fR - regex to match sub-net addresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +.IP +\fI<F-ID>...</F-ID>\fR - free regex capturing group targeting identifier used for ban (instead of IP address or hostname). +.IP +\fI<F-*>...</F-*>\fR - free regex capturing named group stored in ticket, which can be used in action. +.nf +For example \fI<F-USER>[^@]+</F-USER>\fR matches and stores a user name, that can be used in action with interpolation tag \fI<F-USER>\fR. +.IP +\fI<F-ALT_*n>...</F-ALT_*n>\fR - free regex capturing alternative named group stored in ticket. +.nf +For example first found matched value defined in regex as \fI<F-ALT_USER>\fR, \fI<F-ALT_USER1>\fR or \fI<F-ALT_USER2>\fR would be stored as <F-USER> (if direct match is not found or empty). +.PP +Every of abovementioned tags can be specified in \fBprefregex\fR and in \fBfailregex\fR, thereby if specified in both, the value matched in \fBfailregex\fR overwrites a value matched in \fBprefregex\fR. +.TQ +All standard tags like IP4 or IP6 can be also specified with custom regex using \fI<F-*>...</F-*>\fR syntax, for example \fI(?:ip4:<F-IP4>\\S+</F-IP4>|ip6:<F-IP6>\\S+</F-IP6>)\fR. +.TQ +Tags \fI<ADDR>\fR, \fI<HOST>\fR and \fI<SUBNET>\fR would also match the IP address enclosed in square brackets. +.PP +\fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message). +.PP +For multiline regexs (parsing with \fImaxlines\fR greater that 1) the tag \fI<SKIPLINES>\fR can be used to separate lines. This allows lines between the matched lines to continue to be searched for other failures. The tag can be used multiple times. +.br +This is an obsolete handling and if the lines contain some common identifier, better would be to use new handling (with tags \fI<F-MLFID>...<F-MLFID/>\fR). +.RE .TP .B ignoreregex is the regex to identify log entries that should be ignored by Fail2Ban, even if they match failregex. - -.PP -Similar to actions, filters have an [Init] section which can be overridden in \fIjail.conf/jail.local\fR. Besides the filter-specific settings, the filter [Init] section can be used to set following standard options: .TP \fBmaxlines\fR specifies the maximum number of lines to buffer to match multi-line regexs. For some log formats this will not required to be changed. Other logs may require to increase this value if a particular log file is frequently written to. .TP \fBdatepattern\fR -specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %Y-%m-%d %H:%M(?::%S)?. For a list of valid format directives, see Python library documentation for strptime behaviour. -.br -Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used. +specifies a custom date pattern/regex as an alternative to the default date detectors e.g. %%Y-%%m-%%d %%H:%%M(?::%%S)?. +For a list of valid format directives, see Python library documentation for strptime behaviour. .br \fBNOTE:\fR due to config file string substitution, that %'s must be escaped by an % in config files. +.br +Also, special values of \fIEpoch\fR (UNIX Timestamp), \fITAI64N\fR and \fIISO8601\fR can be used as datepattern. +.br +Normally the regexp generated for datepattern additionally gets word-start and word-end boundaries to avoid accidental match inside of some word in a message. +There are several prefixes and words with special meaning that could be specified with custom datepattern to control resulting regex: +.RS +.IP +\fI{DEFAULT}\fR - can be used to add default date patterns of fail2ban. +.IP +\fI{DATE}\fR - can be used as part of regex that will be replaced with default date patterns. +.IP +\fI{^LN-BEG}\fR - prefix (similar to \fI^\fR) changing word-start boundary to line-start boundary (ignoring up to 2 characters). If used as value (not as a prefix), it will also set all default date patterns (similar to \fI{DEFAULT}\fR), but anchored at begin of message line. +.IP +\fI{UNB}\fR - prefix to disable automatic word boundaries in regex. +.IP +\fI{NONE}\fR - value would allow one to find failures totally without date-time in log message. Filter will use now as a timestamp (or last known timestamp from previous line with timestamp). +.RE .TP \fBjournalmatch\fR specifies the systemd journal match used to filter the journal entries. See \fBjournalctl(1)\fR and \fBsystemd.journal-fields(7)\fR for matches syntax and more details on special journal fields. This option is only valid for the \fIsystemd\fR backend. .PP -Similar to actions [Init] section enables filter-specific settings. All parameters specified in [Init] section can be redefined or extended in \fIjail.conf/jail.local\fR. +Similar to actions, filters may have an [Init] section also (optional since v.0.10). All parameters of both sections [Definition] and [Init] can be overridden (redefined or extended) in \fIjail.conf\fR or \fIjail.local\fR (or in related \fIfilter.d/filter-name.local\fR). +Every option supplied in the jail to the filter overwrites the value specified in [Init] section, which in turm would overwrite the value in [Definition] section. +Besides the standard settings of filter both sections can be used to initialize filter-specific options. Filters can also have a section called [INCLUDES]. This is used to read other configuration files. @@ -39,14 +39,6 @@ from distutils.command.build_scripts import build_scripts if setuptools is None: from distutils.command.install import install from distutils.command.install_scripts import install_scripts -try: - # python 3.x - from distutils.command.build_py import build_py_2to3 - from distutils.command.build_scripts import build_scripts_2to3 - _2to3 = True -except ImportError: - # python 2.x - _2to3 = False import os from os.path import isfile, join, isdir, realpath @@ -56,13 +48,15 @@ import warnings from glob import glob from fail2ban.setup import updatePyExec - +from fail2ban.version import version source_dir = os.path.realpath(os.path.dirname( # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): sys.argv[0] if os.path.basename(sys.argv[0]) == 'setup.py' else __file__ )) +with_tests = True + # Wrapper to install python binding (to current python version): class install_scripts_f2b(install_scripts): @@ -95,43 +89,45 @@ class install_scripts_f2b(install_scripts): if install_dir.startswith(root): install_dir = install_dir[len(root):] except: # pragma: no cover - print('WARNING: Cannot find root-base option, check the bin-path to fail2ban-scripts in "fail2ban.service".') - print('Creating %s/fail2ban.service (from fail2ban.service.in): @BINDIR@ -> %s' % (buildroot, install_dir)) - with open(os.path.join(source_dir, 'files/fail2ban.service.in'), 'r') as fn: - lines = fn.readlines() - fn = None - if not dry_run: - fn = open(os.path.join(buildroot, 'fail2ban.service'), 'w') - try: - for ln in lines: - ln = re.sub(r'@BINDIR@', lambda v: install_dir, ln) - if dry_run: - sys.stdout.write(' | ' + ln) - continue - fn.write(ln) - finally: - if fn: fn.close() - if dry_run: - print(' `') + print('WARNING: Cannot find root-base option, check the bin-path to fail2ban-scripts in "fail2ban.service" and "fail2ban-openrc.init".') + + scripts = ['fail2ban.service', 'fail2ban-openrc.init'] + for script in scripts: + print('Creating %s/%s (from %s.in): @BINDIR@ -> %s' % (buildroot, script, script, install_dir)) + with open(os.path.join(source_dir, 'files/%s.in' % script), 'r') as fn: + lines = fn.readlines() + fn = None + if not dry_run: + fn = open(os.path.join(buildroot, script), 'w') + try: + for ln in lines: + ln = re.sub(r'@BINDIR@', lambda v: install_dir, ln) + if dry_run: + sys.stdout.write(' | ' + ln) + continue + fn.write(ln) + finally: + if fn: fn.close() + if dry_run: + print(' `') # Wrapper to specify fail2ban own options: class install_command_f2b(install): user_options = install.user_options + [ - ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), + ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): - self.disable_2to3 = None + self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): - global _2to3 - ## in the test cases 2to3 should be already done (fail2ban-2to3): - if self.disable_2to3: - _2to3 = False - if _2to3: - cmdclass = self.distribution.cmdclass - cmdclass['build_py'] = build_py_2to3 - cmdclass['build_scripts'] = build_scripts_2to3 + if self.without_tests: + self.distribution.scripts.remove('bin/fail2ban-testcases') + + self.distribution.packages.remove('fail2ban.tests') + self.distribution.packages.remove('fail2ban.tests.action_d') + + del self.distribution.package_data['fail2ban.tests'] install.finalize_options(self) def run(self): install.run(self) @@ -159,6 +155,12 @@ elif "test" in sys.argv: print("python distribute required to execute fail2ban tests") print("") +# if build without tests: +if "build" in sys.argv: + if "--without-tests" in sys.argv: + with_tests = False + sys.argv.remove("--without-tests") + longdesc = ''' Fail2Ban scans log files like /var/log/pwdfail or /var/log/apache/error_log and bans IP that makes @@ -169,7 +171,6 @@ commands.''' if setuptools: setup_extra = { 'test_suite': "fail2ban.tests.utils.gatherTests", - 'use_2to3': True, } else: setup_extra = {} @@ -193,9 +194,6 @@ if platform_system in ('linux', 'solaris', 'sunos') or platform_system.startswit ('/usr/share/doc/fail2ban', doc_files) ) -# Get version number, avoiding importing fail2ban. -# This is due to tests not functioning for python3 as 2to3 takes place later -exec(open(join("fail2ban", "version.py")).read()) setup( name = "fail2ban", @@ -208,23 +206,25 @@ setup( license = "GPL", platforms = "Posix", cmdclass = { - 'build_py': build_py, 'build_scripts': build_scripts, + 'build_py': build_py, 'build_scripts': build_scripts, 'install_scripts': install_scripts_f2b, 'install': install_command_f2b }, scripts = [ 'bin/fail2ban-client', 'bin/fail2ban-server', 'bin/fail2ban-regex', - 'bin/fail2ban-testcases', # 'bin/fail2ban-python', -- link (binary), will be installed via install_scripts_f2b wrapper - ], + ] + [ + 'bin/fail2ban-testcases', + ] if with_tests else [], packages = [ 'fail2ban', 'fail2ban.client', 'fail2ban.server', + ] + [ 'fail2ban.tests', 'fail2ban.tests.action_d', - ], + ] if with_tests else [], package_data = { 'fail2ban.tests': [ join(w[0], f).replace("fail2ban/tests/", "", 1) @@ -236,7 +236,7 @@ setup( [ join(w[0], f).replace("fail2ban/tests/", "", 1) for w in os.walk('fail2ban/tests/action_d') for f in w[2]] - }, + } if with_tests else {}, data_files = [ ('/etc/fail2ban', glob("config/*.conf") |