Add pyparsing_common.url to urlExtractorNew.py

1 files changed, 49 insertions, 7 deletions

diff --git a/examples/urlExtractorNew.py b/examples/urlExtractorNew.py
index 7a6e54a..df2f924 100644
--- a/examples/urlExtractorNew.py
+++ b/examples/urlExtractorNew.py
@@ -1,8 +1,10 @@
 # URL extractor
 # Copyright 2004, Paul McGuire
-from pyparsing import makeHTMLTags
-from urllib.request import urlopen
+from collections import Counter
 import pprint
+from urllib.request import urlopen
+
+from pyparsing import makeHTMLTags, pyparsing_common as ppc, FollowedBy, trace_parse_action
 
 # Define the pyparsing grammar for a URL, that is:
 #    URLlink ::= <a href= URL>linkText</a>
@@ -13,9 +15,24 @@ import pprint
 linkOpenTag, linkCloseTag = makeHTMLTags("a")
 link = linkOpenTag + linkOpenTag.tag_body("body") + linkCloseTag.suppress()
 
+
+# Add a parse action to expand relative URLs
+def expand_relative_url(t):
+    url = t.href
+    if url.startswith("//"):
+        url = "https:" + url
+    elif url.startswith(("/", "?", "#")):
+        url = "https://www.cnn.com" + url
+
+    # Put modified URL back into input tokens
+    t["href"] = url
+
+
+link.add_parse_action(expand_relative_url)
+
 # Go get some HTML with some links in it.
 with urlopen("https://www.cnn.com/") as serverListPage:
-    htmlText = serverListPage.read()
+    htmlText = serverListPage.read().decode()
 
 # scanString is a generator that loops through the input htmlText, and for each
 # match yields the tokens and start and end locations (for this application, we are
@@ -23,8 +40,33 @@ with urlopen("https://www.cnn.com/") as serverListPage:
 for toks, strt, end in link.scanString(htmlText):
     print(toks.startA.href, "->", toks.body)
 
-# Create dictionary from list comprehension, assembled from each pair of tokens returned
+# Create dictionary with a dict comprehension, assembled from each pair of tokens returned
 # from a matched URL.
-pprint.pprint(
-    {toks.body: toks.startA.href for toks, strt, end in link.scanString(htmlText)}
-)
+links = {toks.body: toks.href for toks, _, _ in link.scanString(htmlText)}
+pprint.pprint(links)
+
+# Parse the urls in the links using pyparsing_common.url, and tally up all
+# the different domains in a Counter.
+domains = Counter()
+for url in links.values():
+
+    print(url)
+    parsed = ppc.url.parseString(url)
+
+    # print parsed fields for each new url
+    if parsed.host not in domains:
+        print(parsed.dump())
+        print()
+
+    # update domain counter
+    domains[parsed.host] += 1
+
+
+# Print out a little table of all the domains in the urls
+max_domain_len = max(len(d) for d in domains)
+print()
+print("{:{}s}  {}".format("Domain", max_domain_len, "Count"))
+print("{:=<{}}  {:=<5}".format("", max_domain_len, ""))
+
+for domain, count in domains.most_common():
+    print("{:{}s}  {:5d}".format(domain, max_domain_len, count))