mirror of
https://github.com/Mueller-Patrick/Betterzon.git
synced 2024-12-22 19:55:12 +00:00
Compare commits
3 Commits
dbc793cc08
...
f98d1fdb24
Author | SHA1 | Date | |
---|---|---|---|
f98d1fdb24 | |||
776b9a00f2 | |||
2067a47fb2 |
|
@ -1,6 +1,8 @@
|
|||
from flask import Flask
|
||||
from flask_restful import Resource, Api, reqparse
|
||||
|
||||
import crawler
|
||||
|
||||
app = Flask(__name__)
|
||||
api = Api(app)
|
||||
|
||||
|
@ -17,6 +19,7 @@ class CrawlerApi(Resource):
|
|||
def post(self):
|
||||
# Accept crawler request here
|
||||
args = parser.parse_args()
|
||||
crawler.crawl(args['products'])
|
||||
return args
|
||||
|
||||
|
||||
|
|
|
@ -1,5 +1,10 @@
|
|||
import sql
|
||||
import amazonspider
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
HEADERS = ({'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
|
||||
'Safari/537.36'})
|
||||
|
||||
|
||||
def crawl(product_ids: [int]) -> dict:
|
||||
|
@ -57,9 +62,14 @@ def __crawl_amazon__(product_info: dict) -> tuple:
|
|||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
page = requests.get(product_info['url'], headers= HEADERS)
|
||||
soup = BeautifulSoup(page.content, features="lxml")
|
||||
try:
|
||||
price = int(soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
||||
except RuntimeError:
|
||||
price = ''
|
||||
|
||||
amazonspider.start_crawling()
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
return (product_info['product_id'], product_info['vendor_id'], price)
|
||||
|
||||
|
||||
def __crawl_apple__(product_info: dict) -> tuple:
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
pymysql
|
||||
flask
|
||||
flask==1.1.2
|
||||
flask-sqlalchemy
|
||||
flask_restful
|
||||
scrapy
|
||||
beautifulsoup4
|
||||
requests
|
||||
lxml
|
|
@ -35,7 +35,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
|
|||
conn = __getConnection__()
|
||||
cur = conn.cursor()
|
||||
|
||||
query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
|
||||
query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s' % vendor_id
|
||||
|
||||
cur.execute(query, (vendor_id,))
|
||||
|
||||
|
@ -53,8 +53,8 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
|
|||
conn = __getConnection__()
|
||||
cur = conn.cursor()
|
||||
|
||||
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
|
||||
|
||||
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' % product_id
|
||||
print(query)
|
||||
cur.execute(query, (product_id,))
|
||||
|
||||
products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))
|
||||
|
|
Loading…
Reference in New Issue
Block a user