Paddy/Betterzon
1
0
Fork 0
mirror of https://github.com/Mueller-Patrick/Betterzon.git synced 2024-12-23 12:15:11 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: Paddy:f98d1fdb24f9166ce571e6242b3ac2fb91ae6e61
Branches Tags
Paddy:develop
Paddy:master
..
compare: Paddy:dbc793cc0849048ee91c9419a6e41c5696f0da0a
Branches Tags
Paddy:master
Paddy:develop

No commits in common. "f98d1fdb24f9166ce571e6242b3ac2fb91ae6e61" and "dbc793cc0849048ee91c9419a6e41c5696f0da0a" have entirely different histories.
f98d1fdb24 ... dbc793cc08

4 changed files with 85 additions and 100 deletions
Show Stats Expand all files Collapse all files

3
Crawler/api.py
View File

@ -1,8 +1,6 @@
from flask import Flask from flask import Flask
from flask_restful import Resource, Api, reqparse from flask_restful import Resource, Api, reqparse
import crawler
app = Flask(__name__) app = Flask(__name__)
api = Api(app) api = Api(app)
@ -19,7 +17,6 @@ class CrawlerApi(Resource):
def post(self): def post(self):
# Accept crawler request here # Accept crawler request here
args = parser.parse_args() args = parser.parse_args()
crawler.crawl(args['products'])
return args return args

16
Crawler/crawler.py
View File

@ -1,10 +1,5 @@
import sql import sql
import requests import amazonspider
from bs4 import BeautifulSoup
HEADERS = ({'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
'Safari/537.36'})
def crawl(product_ids: [int]) -> dict: def crawl(product_ids: [int]) -> dict:
@ -62,14 +57,9 @@ def __crawl_amazon__(product_info: dict) -> tuple:
:param product_info: A dict with product info containing product_id, vendor_id, url :param product_info: A dict with product info containing product_id, vendor_id, url
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
""" """
page = requests.get(product_info['url'], headers= HEADERS)
soup = BeautifulSoup(page.content, features="lxml")
try:
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
except RuntimeError:
price = ''
return (product_info['product_id'], product_info['vendor_id'], price) amazonspider.start_crawling()
return (product_info['product_id'], product_info['vendor_id'], 123)
def __crawl_apple__(product_info: dict) -> tuple: def __crawl_apple__(product_info: dict) -> tuple:

6
Crawler/requirements.txt
View File

@ -1,7 +1,5 @@
pymysql pymysql
flask==1.1.2 flask
flask-sqlalchemy flask-sqlalchemy
flask_restful flask_restful
beautifulsoup4 scrapy
requests
lxml

6
Crawler/sql.py
View File

@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
conn = __getConnection__() conn = __getConnection__()
cur = conn.cursor() cur = conn.cursor()
query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
cur.execute(query, (vendor_id,)) cur.execute(query, (vendor_id,))
@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
conn = __getConnection__() conn = __getConnection__()
cur = conn.cursor() cur = conn.cursor()
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
print(query)
cur.execute(query, (product_id,)) cur.execute(query, (product_id,))
products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall())) products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))