Paddy/ka-news-yaa-crawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Adding try except block in case of connection errors

Browse Source
This commit is contained in:
Patrick Müller 2022-08-16 22:07:04 +02:00
parent fbdfdc287b
commit 2f69f1d3b6
Signed by: Paddy
GPG Key ID: 37ABC11275CAABCE
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crawler.py
View File

@ -35,7 +35,10 @@ class Crawler:
return articles return articles
def __download_url__(self, url): def __download_url__(self, url):
return requests.get(url, headers=HEADERS).text try:
return requests.get(url, headers=HEADERS).text
except requests.exceptions.ConnectionError:
print('An error has occured trying to download the HTML')
def __extract_article_urls__(self, source) -> []: def __extract_article_urls__(self, source) -> []:
soup = BeautifulSoup(source, 'html.parser') soup = BeautifulSoup(source, 'html.parser')