stripping the links to remove spaces and '\n'

This commit is contained in:
Nicolas Richart 2016-08-15 21:13:25 +02:00
parent 1e673bfa42
commit 37faa41b6f

View file

@ -109,7 +109,7 @@ def _spider(args):
while link_parser.links:
raw_link = link_parser.links.pop()
abs_link = urlparse.urljoin(response_url, raw_link)
abs_link = urlparse.urljoin(response_url, raw_link.strip())
links.add(abs_link)