diff options
Diffstat (limited to 'Lib/robotparser.py')
| -rw-r--r-- | Lib/robotparser.py | 7 |
1 files changed, 6 insertions, 1 deletions
diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 726854b49f..1722863d14 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -133,7 +133,12 @@ class RobotFileParser: return True # search for given user agent matches # the first match counts - url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/" + parsed_url = urlparse.urlparse(urllib.unquote(url)) + url = urlparse.urlunparse(('', '', parsed_url.path, + parsed_url.params, parsed_url.query, parsed_url.fragment)) + url = urllib.quote(url) + if not url: + url = "/" for entry in self.entries: if entry.applies_to(useragent): return entry.allowance(url) |