Crawler/api.py

@@ -1,8 +1,6 @@
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
-import crawler
-
 app = Flask(__name__)
 api = Api(app)
 
@@ -19,7 +17,6 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
-        crawler.crawl(args['products'])
         return args
 
 

Crawler/crawler.py

@@ -1,90 +1,80 @@
-import sql
+import sql
-import requests
+import amazonspider
-from bs4 import BeautifulSoup
+
-
+
-HEADERS = ({'User-Agent':
+def crawl(product_ids: [int]) -> dict:
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
+    """
-            'Safari/537.36'})
+    Crawls the given list of products and saves the results to sql
-
+    :param products: The list of product IDs to fetch
-
+    :return: A dict with the following fields:
-def crawl(product_ids: [int]) -> dict:
+                total_crawls: number of total crawl tries (products * vendors per product)
-    """
+                successful_crawls: number of successful products
-    Crawls the given list of products and saves the results to sql
+                products_with_problems: list of products that have not been crawled successfully
-    :param products: The list of product IDs to fetch
+    """
-    :return: A dict with the following fields:
+    total_crawls = 0
-                total_crawls: number of total crawl tries (products * vendors per product)
+    successful_crawls = 0
-                successful_crawls: number of successful products
+    products_with_problems = []
-                products_with_problems: list of products that have not been crawled successfully
+
-    """
+    # Iterate over every product that has to be crawled
-    total_crawls = 0
+    for product_id in product_ids:
-    successful_crawls = 0
+        # Get all links for this product
-    products_with_problems = []
+        product_links = sql.getProductLinksForProduct(product_id)
-
+
-    # Iterate over every product that has to be crawled
+        crawled_data = []
-    for product_id in product_ids:
+
-        # Get all links for this product
+        # Iterate over every link / vendor
-        product_links = sql.getProductLinksForProduct(product_id)
+        for product_vendor_info in product_links:
-
+            total_crawls += 1
-        crawled_data = []
+
-
+            # Call the appropriate vendor crawling function and append the result to the list of crawled data
-        # Iterate over every link / vendor
+            if product_vendor_info['vendor_id'] == 1:
-        for product_vendor_info in product_links:
+                # Amazon
-            total_crawls += 1
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
-
+            elif product_vendor_info['vendor_id'] == 2:
-            # Call the appropriate vendor crawling function and append the result to the list of crawled data
+                # Apple
-            if product_vendor_info['vendor_id'] == 1:
+                crawled_data.append(__crawl_apple__(product_vendor_info))
-                # Amazon
+            elif product_vendor_info['vendor_id'] == 3:
-                crawled_data.append(__crawl_amazon__(product_vendor_info))
+                # Media Markt
-            elif product_vendor_info['vendor_id'] == 2:
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
-                # Apple
+            else:
-                crawled_data.append(__crawl_apple__(product_vendor_info))
+                products_with_problems.append(product_vendor_info)
-            elif product_vendor_info['vendor_id'] == 3:
+                continue
-                # Media Markt
+
-                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
+            successful_crawls += 1
-            else:
+
-                products_with_problems.append(product_vendor_info)
+        # Insert data to SQL
-                continue
+        sql.insertData(crawled_data)
-
+
-            successful_crawls += 1
+    return {
-
+        'total_crawls': total_crawls,
-        # Insert data to SQL
+        'successful_crawls': successful_crawls,
-        sql.insertData(crawled_data)
+        'products_with_problems': products_with_problems
-
+    }
-    return {
+
-        'total_crawls': total_crawls,
+def __crawl_amazon__(product_info: dict) -> tuple:
-        'successful_crawls': successful_crawls,
+    """
-        'products_with_problems': products_with_problems
+    Crawls the price for the given product from amazon
-    }
+    :param product_info: A dict with product info containing product_id, vendor_id, url
-
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-def __crawl_amazon__(product_info: dict) -> tuple:
+    """
-    """
+
-    Crawls the price for the given product from amazon
+    amazonspider.start_crawling()
-    :param product_info: A dict with product info containing product_id, vendor_id, url
+    return (product_info['product_id'], product_info['vendor_id'], 123)
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+
-    """
+
-    page = requests.get(product_info['url'], headers= HEADERS)
+def __crawl_apple__(product_info: dict) -> tuple:
-    soup = BeautifulSoup(page.content, features="lxml")
+    """
-    try:
+    Crawls the price for the given product from apple
-        price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
+    :param product_info: A dict with product info containing product_id, vendor_id, url
-    except RuntimeError:
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-        price = ''
+    """
-
+    return (product_info['product_id'], product_info['vendor_id'], 123)
-    return (product_info['product_id'], product_info['vendor_id'], price)
+
-
+
-
+def __crawl_mediamarkt__(product_info: dict) -> tuple:
-def __crawl_apple__(product_info: dict) -> tuple:
+    """
-    """
+    Crawls the price for the given product from media markt
-    Crawls the price for the given product from apple
+    :param product_info: A dict with product info containing product_id, vendor_id, url
-    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
-    """
+    pass
-    return (product_info['product_id'], product_info['vendor_id'], 123)
-
-
-def __crawl_mediamarkt__(product_info: dict) -> tuple:
-    """
-    Crawls the price for the given product from media markt
-    :param product_info: A dict with product info containing product_id, vendor_id, url
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    """
-    pass

Crawler/requirements.txt

@@ -1,7 +1,5 @@
 pymysql
-flask==1.1.2
+flask
 flask-sqlalchemy
 flask_restful
-beautifulsoup4
+scrapy
-requests
-lxml

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
 
     cur.execute(query, (vendor_id,))
 
@@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
-    print(query)
+
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))