Fixed string concatenation for sql statement in getProductLinksForProduct

Crawler/api.py

@@ -1,6 +1,8 @@
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
+import crawler
+
 app = Flask(__name__)
 api = Api(app)
 
@@ -17,6 +19,7 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
+        crawler.crawl(args['products'])
         return args
 
 

Crawler/crawler.py

@@ -1,5 +1,10 @@
 import sql
-import amazonspider
+import requests
+from bs4 import BeautifulSoup
+
+HEADERS = ({'User-Agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
+            'Safari/537.36'})
 
 
 def crawl(product_ids: [int]) -> dict:
@@ -57,9 +62,14 @@ def __crawl_amazon__(product_info: dict) -> tuple:
     :param product_info: A dict with product info containing product_id, vendor_id, url
     :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
     """
+    page = requests.get(product_info['url'], headers= HEADERS)
+    soup = BeautifulSoup(page.content, features="lxml")
+    try:
+        price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
+    except RuntimeError:
+        price = ''
 
-    amazonspider.start_crawling()
+    return (product_info['product_id'], product_info['vendor_id'], price)
-    return (product_info['product_id'], product_info['vendor_id'], 123)
 
 
 def __crawl_apple__(product_info: dict) -> tuple:

Crawler/requirements.txt

@@ -1,5 +1,7 @@
 pymysql
-flask
+flask==1.1.2
 flask-sqlalchemy
 flask_restful
-scrapy
+beautifulsoup4
+requests
+lxml

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
 
     cur.execute(query, (vendor_id,))
 
@@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
-
+    print(query)
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))