Crawler/api.py

@@ -1,8 +1,6 @@
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
-import crawler
-
 app = Flask(__name__)
 api = Api(app)
 
@@ -19,7 +17,6 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
-        crawler.crawl(args['products'])
         return args
 
 

Crawler/crawler.py

@@ -1,90 +1,80 @@
-import sql
-import requests
-from bs4 import BeautifulSoup
-
-HEADERS = ({'User-Agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
-            'Safari/537.36'})
-
-
-def crawl(product_ids: [int]) -> dict:
-    """
-    Crawls the given list of products and saves the results to sql
-    :param products: The list of product IDs to fetch
-    :return: A dict with the following fields:
-                total_crawls: number of total crawl tries (products * vendors per product)
-                successful_crawls: number of successful products
-                products_with_problems: list of products that have not been crawled successfully
-    """
-    total_crawls = 0
-    successful_crawls = 0
-    products_with_problems = []
-
-    # Iterate over every product that has to be crawled
-    for product_id in product_ids:
-        # Get all links for this product
-        product_links = sql.getProductLinksForProduct(product_id)
-
-        crawled_data = []
-
-        # Iterate over every link / vendor
-        for product_vendor_info in product_links:
-            total_crawls += 1
-
-            # Call the appropriate vendor crawling function and append the result to the list of crawled data
-            if product_vendor_info['vendor_id'] == 1:
-                # Amazon
-                crawled_data.append(__crawl_amazon__(product_vendor_info))
-            elif product_vendor_info['vendor_id'] == 2:
-                # Apple
-                crawled_data.append(__crawl_apple__(product_vendor_info))
-            elif product_vendor_info['vendor_id'] == 3:
-                # Media Markt
-                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
-            else:
-                products_with_problems.append(product_vendor_info)
-                continue
-
-            successful_crawls += 1
-
-        # Insert data to SQL
-        sql.insertData(crawled_data)
-
-    return {
-        'total_crawls': total_crawls,
-        'successful_crawls': successful_crawls,
-        'products_with_problems': products_with_problems
-    }
-
-def __crawl_amazon__(product_info: dict) -> tuple:
-    """
-    Crawls the price for the given product from amazon
-    :param product_info: A dict with product info containing product_id, vendor_id, url
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    """
-    page = requests.get(product_info['url'], headers= HEADERS)
-    soup = BeautifulSoup(page.content, features="lxml")
-    try:
-        price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
-    except RuntimeError:
-        price = ''
-
-    return (product_info['product_id'], product_info['vendor_id'], price)
-
-
-def __crawl_apple__(product_info: dict) -> tuple:
-    """
-    Crawls the price for the given product from apple
-    :param product_info: A dict with product info containing product_id, vendor_id, url
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    """
-    return (product_info['product_id'], product_info['vendor_id'], 123)
-
-
-def __crawl_mediamarkt__(product_info: dict) -> tuple:
-    """
-    Crawls the price for the given product from media markt
-    :param product_info: A dict with product info containing product_id, vendor_id, url
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    """
-    pass
+import sql
+import amazonspider
+
+
+def crawl(product_ids: [int]) -> dict:
+    """
+    Crawls the given list of products and saves the results to sql
+    :param products: The list of product IDs to fetch
+    :return: A dict with the following fields:
+                total_crawls: number of total crawl tries (products * vendors per product)
+                successful_crawls: number of successful products
+                products_with_problems: list of products that have not been crawled successfully
+    """
+    total_crawls = 0
+    successful_crawls = 0
+    products_with_problems = []
+
+    # Iterate over every product that has to be crawled
+    for product_id in product_ids:
+        # Get all links for this product
+        product_links = sql.getProductLinksForProduct(product_id)
+
+        crawled_data = []
+
+        # Iterate over every link / vendor
+        for product_vendor_info in product_links:
+            total_crawls += 1
+
+            # Call the appropriate vendor crawling function and append the result to the list of crawled data
+            if product_vendor_info['vendor_id'] == 1:
+                # Amazon
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 2:
+                # Apple
+                crawled_data.append(__crawl_apple__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 3:
+                # Media Markt
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
+            else:
+                products_with_problems.append(product_vendor_info)
+                continue
+
+            successful_crawls += 1
+
+        # Insert data to SQL
+        sql.insertData(crawled_data)
+
+    return {
+        'total_crawls': total_crawls,
+        'successful_crawls': successful_crawls,
+        'products_with_problems': products_with_problems
+    }
+
+def __crawl_amazon__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from amazon
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+
+    amazonspider.start_crawling()
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_apple__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from apple
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_mediamarkt__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from media markt
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    pass

Crawler/requirements.txt

@@ -1,7 +1,5 @@
 pymysql
-flask==1.1.2
+flask
 flask-sqlalchemy
 flask_restful
-beautifulsoup4
-requests
-lxml
+scrapy

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
 
     cur.execute(query, (vendor_id,))
 
@@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
-    print(query)
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))