From 610808ad03cd4372470feb3a98a4df81e71a4a12 Mon Sep 17 00:00:00 2001
From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com>
Date: Wed, 14 Apr 2021 21:51:36 +0200
Subject: [PATCH] BETTERZON-59: Adding crawler basic framework (#29)

---
 Crawler/crawler.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++
 Crawler/sql.py     |  2 +-
 2 files changed, 79 insertions(+), 1 deletion(-)
 create mode 100644 Crawler/crawler.py

diff --git a/Crawler/crawler.py b/Crawler/crawler.py
new file mode 100644
index 0000000..99ff867
--- /dev/null
+++ b/Crawler/crawler.py
@@ -0,0 +1,78 @@
+import sql
+
+
+def crawl(product_ids: [int]) -> dict:
+    """
+    Crawls the given list of products and saves the results to sql
+    :param products: The list of product IDs to fetch
+    :return: A dict with the following fields:
+                total_crawls: number of total crawl tries (products * vendors per product)
+                successful_crawls: number of successful products
+                products_with_problems: list of products that have not been crawled successfully
+    """
+    total_crawls = 0
+    successful_crawls = 0
+    products_with_problems = []
+
+    # Iterate over every product that has to be crawled
+    for product_id in product_ids:
+        # Get all links for this product
+        product_links = sql.getProductLinksForProduct(product_id)
+
+        crawled_data = []
+
+        # Iterate over every link / vendor
+        for product_vendor_info in product_links:
+            total_crawls += 1
+
+            # Call the appropriate vendor crawling function and append the result to the list of crawled data
+            if product_vendor_info['vendor_id'] == 1:
+                # Amazon
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 2:
+                # Apple
+                crawled_data.append(__crawl_apple__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 3:
+                # Media Markt
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
+            else:
+                products_with_problems.append(product_vendor_info)
+                continue
+
+            successful_crawls += 1
+
+        # Insert data to SQL
+        sql.insertData(crawled_data)
+
+    return {
+        'total_crawls': total_crawls,
+        'successful_crawls': successful_crawls,
+        'products_with_problems': products_with_problems
+    }
+
+
+def __crawl_amazon__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from amazon
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_apple__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from apple
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_mediamarkt__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from media markt
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    pass
diff --git a/Crawler/sql.py b/Crawler/sql.py
index e6f4e61..1cf3a58 100644
--- a/Crawler/sql.py
+++ b/Crawler/sql.py
@@ -43,7 +43,7 @@ def getProductsForVendor(vendor_id: int) -> [{}]:
 
     return products
 
-def getProductLinksForProduct(product_id: int) -> [{}]:
+def getProductLinksForProduct(product_id: int) -> [dict]:
     """
     Queries all the product links for the given product
     :param product_id: The product to query data for