Paddy/Betterzon
1
0
Fork 0
mirror of https://github.com/Mueller-Patrick/Betterzon.git synced 2024-12-22 19:55:12 +00:00
Code Issues Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
henningxtro 2021-05-16 15:48:08 +02:00
parent 233a5ba140
commit f1d6487701
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Crawler/Crawler.iml
View File

@ -2,13 +2,13 @@
<module type="WEB_MODULE" version="4"> <module type="WEB_MODULE" version="4">
<component name="FacetManager"> <component name="FacetManager">
<facet type="Python" name="Python"> <facet type="Python" name="Python">
<configuration sdkName="Python 3.9" /> <configuration sdkName="Python 3.9 (venv)" />
</facet> </facet>
</component> </component>
<component name="NewModuleRootManager" inherit-compiler-output="true"> <component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output /> <exclude-output />
<content url="file://$MODULE_DIR$" /> <content url="file://$MODULE_DIR$" />
<orderEntry type="sourceFolder" forTests="false" /> <orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Python 3.9 interpreter library" level="application" /> <orderEntry type="library" name="Python 3.9 (venv) interpreter library" level="application" />
</component> </component>
</module> </module>

4
Crawler/crawler.py
View File

@ -1,4 +1,5 @@
import sql import sql
import amazonspider
def crawl(product_ids: [int]) -> dict: def crawl(product_ids: [int]) -> dict:
@ -50,13 +51,14 @@ def crawl(product_ids: [int]) -> dict:
'products_with_problems': products_with_problems 'products_with_problems': products_with_problems
} }
def __crawl_amazon__(product_info: dict) -> tuple: def __crawl_amazon__(product_info: dict) -> tuple:
""" """
Crawls the price for the given product from amazon Crawls the price for the given product from amazon
:param product_info: A dict with product info containing product_id, vendor_id, url :param product_info: A dict with product info containing product_id, vendor_id, url
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
""" """
amazonspider.start_crawling()
return (product_info['product_id'], product_info['vendor_id'], 123) return (product_info['product_id'], product_info['vendor_id'], 123)

2
Crawler/scrapy.cfg
View File

@ -8,4 +8,4 @@ default = crawler.settings
[deploy] [deploy]
#url = http://localhost:6800/ #url = http://localhost:6800/
project = crawler project = crawler