Fixed string concatenation for sql statement in getProductLinksForProduct

Crawler/api.py

@@ -1,6 +1,8 @@
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
+import crawler
+
 app = Flask(__name__)
 api = Api(app)
 
@@ -17,6 +19,7 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
+        crawler.crawl(args['products'])
         return args
 
 

Crawler/crawler.py

@@ -1,80 +1,90 @@
-import sql
+import sql
-import amazonspider
+import requests
-
+from bs4 import BeautifulSoup
-
+
-def crawl(product_ids: [int]) -> dict:
+HEADERS = ({'User-Agent':
-    """
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
-    Crawls the given list of products and saves the results to sql
+            'Safari/537.36'})
-    :param products: The list of product IDs to fetch
+
-    :return: A dict with the following fields:
+
-                total_crawls: number of total crawl tries (products * vendors per product)
+def crawl(product_ids: [int]) -> dict:
-                successful_crawls: number of successful products
+    """
-                products_with_problems: list of products that have not been crawled successfully
+    Crawls the given list of products and saves the results to sql
-    """
+    :param products: The list of product IDs to fetch
-    total_crawls = 0
+    :return: A dict with the following fields:
-    successful_crawls = 0
+                total_crawls: number of total crawl tries (products * vendors per product)
-    products_with_problems = []
+                successful_crawls: number of successful products
-
+                products_with_problems: list of products that have not been crawled successfully
-    # Iterate over every product that has to be crawled
+    """
-    for product_id in product_ids:
+    total_crawls = 0
-        # Get all links for this product
+    successful_crawls = 0
-        product_links = sql.getProductLinksForProduct(product_id)
+    products_with_problems = []
-
+
-        crawled_data = []
+    # Iterate over every product that has to be crawled
-
+    for product_id in product_ids:
-        # Iterate over every link / vendor
+        # Get all links for this product
-        for product_vendor_info in product_links:
+        product_links = sql.getProductLinksForProduct(product_id)
-            total_crawls += 1
+
-
+        crawled_data = []
-            # Call the appropriate vendor crawling function and append the result to the list of crawled data
+
-            if product_vendor_info['vendor_id'] == 1:
+        # Iterate over every link / vendor
-                # Amazon
+        for product_vendor_info in product_links:
-                crawled_data.append(__crawl_amazon__(product_vendor_info))
+            total_crawls += 1
-            elif product_vendor_info['vendor_id'] == 2:
+
-                # Apple
+            # Call the appropriate vendor crawling function and append the result to the list of crawled data
-                crawled_data.append(__crawl_apple__(product_vendor_info))
+            if product_vendor_info['vendor_id'] == 1:
-            elif product_vendor_info['vendor_id'] == 3:
+                # Amazon
-                # Media Markt
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
-                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 2:
-            else:
+                # Apple
-                products_with_problems.append(product_vendor_info)
+                crawled_data.append(__crawl_apple__(product_vendor_info))
-                continue
+            elif product_vendor_info['vendor_id'] == 3:
-
+                # Media Markt
-            successful_crawls += 1
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
-
+            else:
-        # Insert data to SQL
+                products_with_problems.append(product_vendor_info)
-        sql.insertData(crawled_data)
+                continue
-
+
-    return {
+            successful_crawls += 1
-        'total_crawls': total_crawls,
+
-        'successful_crawls': successful_crawls,
+        # Insert data to SQL
-        'products_with_problems': products_with_problems
+        sql.insertData(crawled_data)
-    }
+
-
+    return {
-def __crawl_amazon__(product_info: dict) -> tuple:
+        'total_crawls': total_crawls,
-    """
+        'successful_crawls': successful_crawls,
-    Crawls the price for the given product from amazon
+        'products_with_problems': products_with_problems
-    :param product_info: A dict with product info containing product_id, vendor_id, url
+    }
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+
-    """
+def __crawl_amazon__(product_info: dict) -> tuple:
-
+    """
-    amazonspider.start_crawling()
+    Crawls the price for the given product from amazon
-    return (product_info['product_id'], product_info['vendor_id'], 123)
+    :param product_info: A dict with product info containing product_id, vendor_id, url
-
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-
+    """
-def __crawl_apple__(product_info: dict) -> tuple:
+    page = requests.get(product_info['url'], headers= HEADERS)
-    """
+    soup = BeautifulSoup(page.content, features="lxml")
-    Crawls the price for the given product from apple
+    try:
-    :param product_info: A dict with product info containing product_id, vendor_id, url
+        price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    except RuntimeError:
-    """
+        price = ''
-    return (product_info['product_id'], product_info['vendor_id'], 123)
+
-
+    return (product_info['product_id'], product_info['vendor_id'], price)
-
+
-def __crawl_mediamarkt__(product_info: dict) -> tuple:
+
-    """
+def __crawl_apple__(product_info: dict) -> tuple:
-    Crawls the price for the given product from media markt
+    """
-    :param product_info: A dict with product info containing product_id, vendor_id, url
+    Crawls the price for the given product from apple
-    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    :param product_info: A dict with product info containing product_id, vendor_id, url
-    """
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
-    pass
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_mediamarkt__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from media markt
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    pass

Crawler/requirements.txt

@@ -1,5 +1,7 @@
 pymysql
-flask
+flask==1.1.2
 flask-sqlalchemy
 flask_restful
-scrapy
+beautifulsoup4
+requests
+lxml

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
 
     cur.execute(query, (vendor_id,))
 
@@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
-
+    print(query)
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))