mirror of
https://github.com/Mueller-Patrick/Betterzon.git
synced 2024-12-22 19:55:12 +00:00
Added basic amazon crawler using beautifulsoup4
This commit is contained in:
parent
dbc793cc08
commit
2067a47fb2
|
|
@ -1,80 +1,90 @@
|
|||
import sql
|
||||
import amazonspider
|
||||
|
||||
|
||||
def crawl(product_ids: [int]) -> dict:
|
||||
"""
|
||||
Crawls the given list of products and saves the results to sql
|
||||
:param products: The list of product IDs to fetch
|
||||
:return: A dict with the following fields:
|
||||
total_crawls: number of total crawl tries (products * vendors per product)
|
||||
successful_crawls: number of successful products
|
||||
products_with_problems: list of products that have not been crawled successfully
|
||||
"""
|
||||
total_crawls = 0
|
||||
successful_crawls = 0
|
||||
products_with_problems = []
|
||||
|
||||
# Iterate over every product that has to be crawled
|
||||
for product_id in product_ids:
|
||||
# Get all links for this product
|
||||
product_links = sql.getProductLinksForProduct(product_id)
|
||||
|
||||
crawled_data = []
|
||||
|
||||
# Iterate over every link / vendor
|
||||
for product_vendor_info in product_links:
|
||||
total_crawls += 1
|
||||
|
||||
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
||||
if product_vendor_info['vendor_id'] == 1:
|
||||
# Amazon
|
||||
crawled_data.append(__crawl_amazon__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 2:
|
||||
# Apple
|
||||
crawled_data.append(__crawl_apple__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 3:
|
||||
# Media Markt
|
||||
crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
|
||||
else:
|
||||
products_with_problems.append(product_vendor_info)
|
||||
continue
|
||||
|
||||
successful_crawls += 1
|
||||
|
||||
# Insert data to SQL
|
||||
sql.insertData(crawled_data)
|
||||
|
||||
return {
|
||||
'total_crawls': total_crawls,
|
||||
'successful_crawls': successful_crawls,
|
||||
'products_with_problems': products_with_problems
|
||||
}
|
||||
|
||||
def __crawl_amazon__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from amazon
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
|
||||
amazonspider.start_crawling()
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
|
||||
|
||||
def __crawl_apple__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from apple
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
|
||||
|
||||
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from media markt
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
pass
|
||||
import sql
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
HEADERS = ({'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
|
||||
'Safari/537.36'})
|
||||
|
||||
|
||||
def crawl(product_ids: [int]) -> dict:
|
||||
"""
|
||||
Crawls the given list of products and saves the results to sql
|
||||
:param products: The list of product IDs to fetch
|
||||
:return: A dict with the following fields:
|
||||
total_crawls: number of total crawl tries (products * vendors per product)
|
||||
successful_crawls: number of successful products
|
||||
products_with_problems: list of products that have not been crawled successfully
|
||||
"""
|
||||
total_crawls = 0
|
||||
successful_crawls = 0
|
||||
products_with_problems = []
|
||||
|
||||
# Iterate over every product that has to be crawled
|
||||
for product_id in product_ids:
|
||||
# Get all links for this product
|
||||
product_links = sql.getProductLinksForProduct(product_id)
|
||||
|
||||
crawled_data = []
|
||||
|
||||
# Iterate over every link / vendor
|
||||
for product_vendor_info in product_links:
|
||||
total_crawls += 1
|
||||
|
||||
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
||||
if product_vendor_info['vendor_id'] == 1:
|
||||
# Amazon
|
||||
crawled_data.append(__crawl_amazon__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 2:
|
||||
# Apple
|
||||
crawled_data.append(__crawl_apple__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 3:
|
||||
# Media Markt
|
||||
crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
|
||||
else:
|
||||
products_with_problems.append(product_vendor_info)
|
||||
continue
|
||||
|
||||
successful_crawls += 1
|
||||
|
||||
# Insert data to SQL
|
||||
sql.insertData(crawled_data)
|
||||
|
||||
return {
|
||||
'total_crawls': total_crawls,
|
||||
'successful_crawls': successful_crawls,
|
||||
'products_with_problems': products_with_problems
|
||||
}
|
||||
|
||||
def __crawl_amazon__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from amazon
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
page = requests.get(product_info['url'], headers= HEADERS)
|
||||
soup = BeautifulSoup(page.content, features="lxml")
|
||||
try:
|
||||
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
||||
except RuntimeError:
|
||||
price = ''
|
||||
|
||||
return (product_info['product_id'], product_info['vendor_id'], price)
|
||||
|
||||
|
||||
def __crawl_apple__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from apple
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
|
||||
|
||||
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from media markt
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
pymysql
|
||||
flask
|
||||
flask==1.1.2
|
||||
flask-sqlalchemy
|
||||
flask_restful
|
||||
scrapy
|
||||
beautifulsoup4
|
||||
requests
|
||||
Loading…
Reference in New Issue
Block a user