Crawler/api.py

@@ -1,5 +1,3 @@
-import os
-
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
@@ -10,8 +8,8 @@ api = Api(app)
 
 # To parse request data
 parser = reqparse.RequestParser()
-parser.add_argument('key', type=str)
-parser.add_argument('products', type=int, action='append')
+parser.add_argument('key')
+parser.add_argument('products')
 
 
 class CrawlerApi(Resource):
@@ -21,12 +19,8 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
-        access_key = os.getenv('CRAWLER_ACCESS_KEY')
-        if(args['key'] == access_key):
-            crawler.crawl(args['products'])
-            return {'message': 'success'}
-        else:
-            return {'message': 'Wrong access key'}
+        crawler.crawl(args['products'])
+        return args
 
 
 api.add_resource(CrawlerApi, '/')

Crawler/crawler.py

@@ -34,19 +34,13 @@ def crawl(product_ids: [int]) -> dict:
             # Call the appropriate vendor crawling function and append the result to the list of crawled data
             if product_vendor_info['vendor_id'] == 1:
                 # Amazon
-                data = __crawl_amazon__(product_vendor_info)
-                if data:
-                    crawled_data.append(data)
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
             elif product_vendor_info['vendor_id'] == 2:
                 # Apple
-                data = __crawl_apple__(product_vendor_info)
-                if data:
-                    crawled_data.append(data)
+                crawled_data.append(__crawl_apple__(product_vendor_info))
             elif product_vendor_info['vendor_id'] == 3:
                 # Media Markt
-                data = __crawl_mediamarkt__(product_vendor_info)
-                if data:
-                    crawled_data.append(data)
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
             else:
                 products_with_problems.append(product_vendor_info)
                 continue
@@ -73,14 +67,9 @@ def __crawl_amazon__(product_info: dict) -> tuple:
     try:
         price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
     except RuntimeError:
-        price = -1
-    except AttributeError:
-        price = -1
+        price = ''
 
-    if price != -1:
-        return (product_info['product_id'], product_info['vendor_id'], price)
-    else:
-        return None
+    return (product_info['product_id'], product_info['vendor_id'], price)
 
 
 def __crawl_apple__(product_info: dict) -> tuple:
@@ -89,8 +78,7 @@ def __crawl_apple__(product_info: dict) -> tuple:
     :param product_info: A dict with product info containing product_id, vendor_id, url
     :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
     """
-    #return (product_info['product_id'], product_info['vendor_id'], 123)
-    pass
+    return (product_info['product_id'], product_info['vendor_id'], 123)
 
 
 def __crawl_mediamarkt__(product_info: dict) -> tuple:

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
 
     cur.execute(query, (vendor_id,))
 
@@ -53,7 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
+    print(query)
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))