Crawler/api.py

@@ -1,8 +1,6 @@
 from flask import Flask
 from flask_restful import Resource, Api, reqparse
 
-import crawler
-
 app = Flask(__name__)
 api = Api(app)
 
@@ -19,7 +17,6 @@ class CrawlerApi(Resource):
     def post(self):
         # Accept crawler request here
         args = parser.parse_args()
-        crawler.crawl(args['products'])
         return args
 
 

Crawler/crawler.py

@@ -1,10 +1,5 @@
 import sql
-import requests
-from bs4 import BeautifulSoup
-
-HEADERS = ({'User-Agent':
-            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
-            'Safari/537.36'})
+import amazonspider
 
 
 def crawl(product_ids: [int]) -> dict:
@@ -62,14 +57,9 @@ def __crawl_amazon__(product_info: dict) -> tuple:
     :param product_info: A dict with product info containing product_id, vendor_id, url
     :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
     """
-    page = requests.get(product_info['url'], headers= HEADERS)
-    soup = BeautifulSoup(page.content, features="lxml")
-    try:
-        price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
-    except RuntimeError:
-        price = ''
 
-    return (product_info['product_id'], product_info['vendor_id'], price)
+    amazonspider.start_crawling()
+    return (product_info['product_id'], product_info['vendor_id'], 123)
 
 
 def __crawl_apple__(product_info: dict) -> tuple:

Crawler/requirements.txt

@@ -1,7 +1,5 @@
 pymysql
-flask==1.1.2
+flask
 flask-sqlalchemy
 flask_restful
-beautifulsoup4
-requests
-lxml
+scrapy

Crawler/sql.py

@@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
 
     cur.execute(query, (vendor_id,))
 
@@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
     conn = __getConnection__()
     cur = conn.cursor()
 
-    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
-    print(query)
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+
     cur.execute(query, (product_id,))
 
     products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))