Paddy/Betterzon
1
0
Fork 0
mirror of https://github.com/Mueller-Patrick/Betterzon.git synced 2024-12-22 19:55:12 +00:00
Code Issues Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
henningxtro 2021-05-16 15:48:08 +02:00
parent 233a5ba140
commit f1d6487701
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Crawler/Crawler.iml
View File

@ -2,13 +2,13 @@
<module type="WEB_MODULE" version="4">
<component name="FacetManager">
<facet type="Python" name="Python">
<configuration sdkName="Python 3.9" />
<configuration sdkName="Python 3.9 (venv)" />
</facet>
</component>
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Python 3.9 interpreter library" level="application" />
<orderEntry type="library" name="Python 3.9 (venv) interpreter library" level="application" />
</component>
</module>

4
Crawler/crawler.py
View File

@ -1,4 +1,5 @@
import sql
import amazonspider
def crawl(product_ids: [int]) -> dict:
@ -50,13 +51,14 @@ def crawl(product_ids: [int]) -> dict:
'products_with_problems': products_with_problems
}
def __crawl_amazon__(product_info: dict) -> tuple:
"""
Crawls the price for the given product from amazon
:param product_info: A dict with product info containing product_id, vendor_id, url
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
"""
amazonspider.start_crawling()
return (product_info['product_id'], product_info['vendor_id'], 123)

2
Crawler/scrapy.cfg
View File

@ -8,4 +8,4 @@ default = crawler.settings
[deploy]
#url = http://localhost:6800/
project = crawler
project = crawler