From 73f570ba08756c322c0b6fa070d89aa3c64b31c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Mon, 18 Mar 2002 10:43:18 +0000 Subject: Correctly set default entry in all cases. --- Lib/robotparser.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'Lib/robotparser.py') diff --git a/Lib/robotparser.py b/Lib/robotparser.py index aace3a43f4..5b759d4968 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -58,6 +58,13 @@ class RobotFileParser: _debug("parse lines") self.parse(lines) + def _add_entry(self, entry): + if "*" in entry.useragents: + # the default entry is considered last + self.default_entry = entry + else: + self.entries.append(entry) + def parse(self, lines): """parse the input lines from a robot.txt file. We allow that a user-agent: line is not preceded by @@ -76,11 +83,7 @@ class RobotFileParser: entry = Entry() state = 0 elif state==2: - if "*" in entry.useragents: - # the default entry is considered last - self.default_entry = entry - else: - self.entries.append(entry) + self._add_entry(entry) entry = Entry() state = 0 # remove optional comment and strip line @@ -99,7 +102,7 @@ class RobotFileParser: _debug("line %d: warning: you should insert a blank" " line before any user-agent" " directive" % linenumber) - self.entries.append(entry) + self._add_entry(entry) entry = Entry() entry.useragents.append(line[1]) state = 1 -- cgit v1.2.1