1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python
# Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017)
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Checks for broken links in file names specified as command line parameters.
There are a ton of a solutions available for validating URLs in string using
regex, but less for searching, of which very few are accurate.
This snippet is intended to just do the required work, and avoid complexities.
Django Validator has pretty good regex for validation, but we have to find
urls instead of validating them. (REFERENCES [7])
There's always room for improvement.
Method:
* Match URLs using regex (REFERENCES [1]])
* Some URLs need to be fixed, as they have < (or) > due to inefficient regex.
* Remove duplicates (because regex is not 100% efficient as of now).
* Check validity of URL, using HEAD request. (HEAD to save bandwidth)
Uses requests module for others are painful to use. REFERENCES[9]
Handles redirects, http, https, ftp as well.
REFERENCES:
Using [1] with some modificatons for including ftp
[1] http://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python
[2] http://stackoverflow.com/a/31952097/5163807
[3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls
[4] https://mathiasbynens.be/demo/url-regex
[5] https://github.com/django/django/blob/master/django/core/validators.py
[6] https://data.iana.org/TLD/tlds-alpha-by-domain.txt
[7] https://codereview.stackexchange.com/questions/19663/http-url-validating
[8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
[9] http://docs.python-requests.org/
"""
from __future__ import print_function
import os
import re
import sys
import requests
HERE = os.path.abspath(os.path.dirname(__file__))
URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
def get_urls(filename):
"""Extracts all URLs available in specified filename
"""
# fname = os.path.abspath(os.path.join(HERE, filename))
# expecting absolute path
fname = os.path.abspath(filename)
print (fname)
text = ''
with open(fname) as f:
text = f.read()
urls = re.findall(URL_REGEX, text)
# remove duplicates, list for sets are not iterable
urls = list(set(urls))
# correct urls which are between < and/or >
i = 0
while i < len(urls):
urls[i] = re.sub("[<>]", '', urls[i])
i += 1
return urls
def validate_url(url):
"""Validate the URL by attempting an HTTP connection.
Makes an HTTP-HEAD request for each URL.
Uses requests module.
"""
try:
res = requests.head(url)
return res.ok
except Exception as e:
return False
def main():
"""Main function
"""
files = sys.argv[1:]
fails = []
for fname in files:
urls = get_urls(fname)
i = 0
last = len(urls)
for url in urls:
i += 1
if not validate_url(url):
fails.append((url, fname))
sys.stdout.write("\r " + fname + " : " + str(i) + " / " + str(last))
sys.stdout.flush()
print()
if len(fails) == 0:
print("All URLs are valid. Cheers!")
else:
print ("Total :", len(fails), "fails!")
print ("Writing failed urls to fails.txt")
with open("../../fails.txt", 'w') as f:
for fail in fails:
f.write(fail[1] + ' : ' + fail[0] + os.linesep)
f.write('-' * 20)
f.write(os.linesep*2)
if __name__ == '__main__':
main()
|
