From 2067a47fb23f63ad88fbb91c5585d9afbac9aa77 Mon Sep 17 00:00:00 2001 From: henningxtro Date: Sun, 16 May 2021 22:05:32 +0200 Subject: [PATCH] Added basic amazon crawler using beautifulsoup4 --- Crawler/crawler.py | 170 +++++++++++++++++++++------------------ Crawler/requirements.txt | 5 +- 2 files changed, 93 insertions(+), 82 deletions(-) diff --git a/Crawler/crawler.py b/Crawler/crawler.py index f3bea76..025bcd7 100644 --- a/Crawler/crawler.py +++ b/Crawler/crawler.py @@ -1,80 +1,90 @@ -import sql -import amazonspider - - -def crawl(product_ids: [int]) -> dict: - """ - Crawls the given list of products and saves the results to sql - :param products: The list of product IDs to fetch - :return: A dict with the following fields: - total_crawls: number of total crawl tries (products * vendors per product) - successful_crawls: number of successful products - products_with_problems: list of products that have not been crawled successfully - """ - total_crawls = 0 - successful_crawls = 0 - products_with_problems = [] - - # Iterate over every product that has to be crawled - for product_id in product_ids: - # Get all links for this product - product_links = sql.getProductLinksForProduct(product_id) - - crawled_data = [] - - # Iterate over every link / vendor - for product_vendor_info in product_links: - total_crawls += 1 - - # Call the appropriate vendor crawling function and append the result to the list of crawled data - if product_vendor_info['vendor_id'] == 1: - # Amazon - crawled_data.append(__crawl_amazon__(product_vendor_info)) - elif product_vendor_info['vendor_id'] == 2: - # Apple - crawled_data.append(__crawl_apple__(product_vendor_info)) - elif product_vendor_info['vendor_id'] == 3: - # Media Markt - crawled_data.append(__crawl_mediamarkt__(product_vendor_info)) - else: - products_with_problems.append(product_vendor_info) - continue - - successful_crawls += 1 - - # Insert data to SQL - sql.insertData(crawled_data) - - return { - 'total_crawls': total_crawls, - 'successful_crawls': successful_crawls, - 'products_with_problems': products_with_problems - } - -def __crawl_amazon__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from amazon - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - - amazonspider.start_crawling() - return (product_info['product_id'], product_info['vendor_id'], 123) - - -def __crawl_apple__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from apple - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - return (product_info['product_id'], product_info['vendor_id'], 123) - - -def __crawl_mediamarkt__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from media markt - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - pass +import sql +import requests +from bs4 import BeautifulSoup + +HEADERS = ({'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 ' + 'Safari/537.36'}) + + +def crawl(product_ids: [int]) -> dict: + """ + Crawls the given list of products and saves the results to sql + :param products: The list of product IDs to fetch + :return: A dict with the following fields: + total_crawls: number of total crawl tries (products * vendors per product) + successful_crawls: number of successful products + products_with_problems: list of products that have not been crawled successfully + """ + total_crawls = 0 + successful_crawls = 0 + products_with_problems = [] + + # Iterate over every product that has to be crawled + for product_id in product_ids: + # Get all links for this product + product_links = sql.getProductLinksForProduct(product_id) + + crawled_data = [] + + # Iterate over every link / vendor + for product_vendor_info in product_links: + total_crawls += 1 + + # Call the appropriate vendor crawling function and append the result to the list of crawled data + if product_vendor_info['vendor_id'] == 1: + # Amazon + crawled_data.append(__crawl_amazon__(product_vendor_info)) + elif product_vendor_info['vendor_id'] == 2: + # Apple + crawled_data.append(__crawl_apple__(product_vendor_info)) + elif product_vendor_info['vendor_id'] == 3: + # Media Markt + crawled_data.append(__crawl_mediamarkt__(product_vendor_info)) + else: + products_with_problems.append(product_vendor_info) + continue + + successful_crawls += 1 + + # Insert data to SQL + sql.insertData(crawled_data) + + return { + 'total_crawls': total_crawls, + 'successful_crawls': successful_crawls, + 'products_with_problems': products_with_problems + } + +def __crawl_amazon__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from amazon + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + page = requests.get(product_info['url'], headers= HEADERS) + soup = BeautifulSoup(page.content, features="lxml") + try: + price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip()) + except RuntimeError: + price = '' + + return (product_info['product_id'], product_info['vendor_id'], price) + + +def __crawl_apple__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from apple + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + return (product_info['product_id'], product_info['vendor_id'], 123) + + +def __crawl_mediamarkt__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from media markt + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + pass diff --git a/Crawler/requirements.txt b/Crawler/requirements.txt index ba99df1..f7b6180 100644 --- a/Crawler/requirements.txt +++ b/Crawler/requirements.txt @@ -1,5 +1,6 @@ pymysql -flask +flask==1.1.2 flask-sqlalchemy flask_restful -scrapy \ No newline at end of file +beautifulsoup4 +requests \ No newline at end of file