mirror of
https://github.com/Mueller-Patrick/Betterzon.git
synced 2024-12-23 12:15:11 +00:00
Compare commits
No commits in common. "63cbac5490ada4c97808bac84a6c28ef7aab5572" and "f98d1fdb24f9166ce571e6242b3ac2fb91ae6e61" have entirely different histories.
63cbac5490
...
f98d1fdb24
|
@ -1,5 +1,3 @@
|
||||||
import os
|
|
||||||
|
|
||||||
from flask import Flask
|
from flask import Flask
|
||||||
from flask_restful import Resource, Api, reqparse
|
from flask_restful import Resource, Api, reqparse
|
||||||
|
|
||||||
|
@ -10,8 +8,8 @@ api = Api(app)
|
||||||
|
|
||||||
# To parse request data
|
# To parse request data
|
||||||
parser = reqparse.RequestParser()
|
parser = reqparse.RequestParser()
|
||||||
parser.add_argument('key', type=str)
|
parser.add_argument('key')
|
||||||
parser.add_argument('products', type=int, action='append')
|
parser.add_argument('products')
|
||||||
|
|
||||||
|
|
||||||
class CrawlerApi(Resource):
|
class CrawlerApi(Resource):
|
||||||
|
@ -21,12 +19,8 @@ class CrawlerApi(Resource):
|
||||||
def post(self):
|
def post(self):
|
||||||
# Accept crawler request here
|
# Accept crawler request here
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
access_key = os.getenv('CRAWLER_ACCESS_KEY')
|
|
||||||
if(args['key'] == access_key):
|
|
||||||
crawler.crawl(args['products'])
|
crawler.crawl(args['products'])
|
||||||
return {'message': 'success'}
|
return args
|
||||||
else:
|
|
||||||
return {'message': 'Wrong access key'}
|
|
||||||
|
|
||||||
|
|
||||||
api.add_resource(CrawlerApi, '/')
|
api.add_resource(CrawlerApi, '/')
|
||||||
|
|
|
@ -34,19 +34,13 @@ def crawl(product_ids: [int]) -> dict:
|
||||||
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
||||||
if product_vendor_info['vendor_id'] == 1:
|
if product_vendor_info['vendor_id'] == 1:
|
||||||
# Amazon
|
# Amazon
|
||||||
data = __crawl_amazon__(product_vendor_info)
|
crawled_data.append(__crawl_amazon__(product_vendor_info))
|
||||||
if data:
|
|
||||||
crawled_data.append(data)
|
|
||||||
elif product_vendor_info['vendor_id'] == 2:
|
elif product_vendor_info['vendor_id'] == 2:
|
||||||
# Apple
|
# Apple
|
||||||
data = __crawl_apple__(product_vendor_info)
|
crawled_data.append(__crawl_apple__(product_vendor_info))
|
||||||
if data:
|
|
||||||
crawled_data.append(data)
|
|
||||||
elif product_vendor_info['vendor_id'] == 3:
|
elif product_vendor_info['vendor_id'] == 3:
|
||||||
# Media Markt
|
# Media Markt
|
||||||
data = __crawl_mediamarkt__(product_vendor_info)
|
crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
|
||||||
if data:
|
|
||||||
crawled_data.append(data)
|
|
||||||
else:
|
else:
|
||||||
products_with_problems.append(product_vendor_info)
|
products_with_problems.append(product_vendor_info)
|
||||||
continue
|
continue
|
||||||
|
@ -73,14 +67,9 @@ def __crawl_amazon__(product_info: dict) -> tuple:
|
||||||
try:
|
try:
|
||||||
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
||||||
except RuntimeError:
|
except RuntimeError:
|
||||||
price = -1
|
price = ''
|
||||||
except AttributeError:
|
|
||||||
price = -1
|
|
||||||
|
|
||||||
if price != -1:
|
|
||||||
return (product_info['product_id'], product_info['vendor_id'], price)
|
return (product_info['product_id'], product_info['vendor_id'], price)
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def __crawl_apple__(product_info: dict) -> tuple:
|
def __crawl_apple__(product_info: dict) -> tuple:
|
||||||
|
@ -89,8 +78,7 @@ def __crawl_apple__(product_info: dict) -> tuple:
|
||||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||||
"""
|
"""
|
||||||
#return (product_info['product_id'], product_info['vendor_id'], 123)
|
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
||||||
|
|
|
@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
|
||||||
conn = __getConnection__()
|
conn = __getConnection__()
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
|
query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
|
||||||
|
|
||||||
cur.execute(query, (vendor_id,))
|
cur.execute(query, (vendor_id,))
|
||||||
|
|
||||||
|
@ -53,7 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
|
||||||
conn = __getConnection__()
|
conn = __getConnection__()
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
||||||
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
|
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
|
||||||
|
print(query)
|
||||||
cur.execute(query, (product_id,))
|
cur.execute(query, (product_id,))
|
||||||
|
|
||||||
products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))
|
products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))
|
||||||
|
|
Loading…
Reference in New Issue
Block a user