Paddy/Betterzon
1
0
Fork 0
mirror of https://github.com/Mueller-Patrick/Betterzon.git synced 2024-12-23 04:05:12 +00:00
Code Issues Projects Releases Wiki Activity

Added basic amazon crawler using beautifulsoup4

Browse Source
This commit is contained in:
henningxtro 2021-05-16 22:05:32 +02:00
parent dbc793cc08
commit 2067a47fb2
2 changed files with 93 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
Crawler/crawler.py
View File

@ -1,5 +1,10 @@
import sql import sql
import amazonspider import requests
from bs4 import BeautifulSoup
HEADERS = ({'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
'Safari/537.36'})
def crawl(product_ids: [int]) -> dict: def crawl(product_ids: [int]) -> dict:
@ -57,9 +62,14 @@ def __crawl_amazon__(product_info: dict) -> tuple:
:param product_info: A dict with product info containing product_id, vendor_id, url :param product_info: A dict with product info containing product_id, vendor_id, url
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
""" """
page = requests.get(product_info['url'], headers= HEADERS)
soup = BeautifulSoup(page.content, features="lxml")
try:
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
except RuntimeError:
price = ''
amazonspider.start_crawling() return (product_info['product_id'], product_info['vendor_id'], price)
return (product_info['product_id'], product_info['vendor_id'], 123)
def __crawl_apple__(product_info: dict) -> tuple: def __crawl_apple__(product_info: dict) -> tuple:

5
Crawler/requirements.txt
View File

@ -1,5 +1,6 @@
pymysql pymysql
flask flask==1.1.2
flask-sqlalchemy flask-sqlalchemy
flask_restful flask_restful
scrapy beautifulsoup4
requests