Merge pull request #31 from Mueller-Patrick/develop

Master Deployment for API changes
2024-11-25 15:33:56 +00:00 · 2021-04-17 12:34:10 +02:00 · 2021-04-17 12:34:10 +02:00 · 197c39a61d
commit 197c39a61d
parent aaf829f090 21d5294a57
19 changed files with 686 additions and 80 deletions
--- a/.gitignore
+++ b/.gitignore
@ -27,6 +27,7 @@ speed-measure-plugin*.json
 !Backend.iml
 !CucumberTests.iml
 !Crawler.iml
+!Crawler-Loadbalancer.iml

 # Include IntelliJ modules
 !/.idea/modules.xml
--- a/.idea/modules.xml
+++ b/.idea/modules.xml
@ -5,6 +5,7 @@
      <module fileurl="file://$PROJECT_DIR$/Backend/Backend.iml" filepath="$PROJECT_DIR$/Backend/Backend.iml" />
      <module fileurl="file://$PROJECT_DIR$/Betterzon.iml" filepath="$PROJECT_DIR$/Betterzon.iml" />
      <module fileurl="file://$PROJECT_DIR$/Crawler/Crawler.iml" filepath="$PROJECT_DIR$/Crawler/Crawler.iml" />
+      <module fileurl="file://$PROJECT_DIR$/Crawler-Loadbalancer/Crawler-Loadbalancer.iml" filepath="$PROJECT_DIR$/Crawler-Loadbalancer/Crawler-Loadbalancer.iml" />
      <module fileurl="file://$PROJECT_DIR$/CucumberTests/CucumberTests.iml" filepath="$PROJECT_DIR$/CucumberTests/CucumberTests.iml" />
      <module fileurl="file://$PROJECT_DIR$/Frontend/Frontend.iml" filepath="$PROJECT_DIR$/Frontend/Frontend.iml" />
    </modules>
--- a/Backend/src/models/products/products.router.ts
+++ b/Backend/src/models/products/products.router.ts
@ -69,6 +69,25 @@ productsRouter.get('/search/:term', async (req: Request, res: Response) => {
    }
 });

+// GET products/list/[1,2,3]
+
+productsRouter.get('/list/:ids', async (req: Request, res: Response) => {
+    const ids: [number] = JSON.parse(req.params.ids);
+
+    if (!ids) {
+        res.status(400).send('Missing parameters.');
+        return;
+    }
+
+    try {
+        const products: Products = await ProductService.findList(ids);
+
+        res.status(200).send(products);
+    } catch (e) {
+        res.status(404).send(e.message);
+    }
+});
+
 // GET products/bestDeals


--- a/Backend/src/models/products/products.service.ts
+++ b/Backend/src/models/products/products.service.ts
@ -122,6 +122,29 @@ export const findBySearchTerm = async (term: string): Promise<Products> => {
    return prodRows;
 };

+export const findList = async (ids: [number]): Promise<Products> => {
+    let conn;
+    let prodRows = [];
+    try {
+        conn = await pool.getConnection();
+        const rows = await conn.query('SELECT product_id, name, asin, is_active, short_description, long_description, image_guid, date_added, last_modified, manufacturer_id, selling_rank, category_id FROM products WHERE product_id IN (?)', [ids]);
+        for (let row in rows) {
+            if (row !== 'meta') {
+                prodRows.push(rows[row]);
+            }
+        }
+
+    } catch (err) {
+        throw err;
+    } finally {
+        if (conn) {
+            conn.end();
+        }
+    }
+
+    return prodRows;
+};
+
 // export const create = async (newItem: Product): Promise<void> => {
 //     let conn;
 //     try {
--- a/Crawler-Loadbalancer/Crawler-Loadbalancer.iml
+++ b/Crawler-Loadbalancer/Crawler-Loadbalancer.iml
@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
--- a/Crawler-Loadbalancer/dailycrawl.py
+++ b/Crawler-Loadbalancer/dailycrawl.py
@ -0,0 +1,59 @@
+import json
+import requests
+import os
+
+import sql
+
+
+def call_crawlers() -> bool:
+    """
+    Fetches the list of all shops, does some load balancing magic and calls all registered crawler
+    instances to start them
+    :return: If the calls have been successful
+    """
+    product_ids = sql.getProductsToCrawl()
+
+    # crawler_urls = ['crawl.p4ddy.com', 'crawl.betterzon.xyz']
+    crawler_urls = ['http://localhost:22026']
+
+    balanced_lists = []
+
+    products_per_crawler = len(product_ids) // len(crawler_urls)
+    rest = len(product_ids) % len(crawler_urls)
+
+    # Distrubute available products over available crawler instances
+    for crawler_id in range(len(crawler_urls)):
+        amount_of_prods = products_per_crawler
+
+        # If we e.g. have 7 products but 2 crawlers, the first needs to crawl 4 products and the 2nd 3
+        if crawler_id < rest:
+            amount_of_prods += 1
+
+        # Assign the required amount of product ids to the current crawler and remove them from the
+        # list of all product ids
+        balanced_lists.append(product_ids[:amount_of_prods])
+        product_ids = product_ids[amount_of_prods:]
+
+    # Make the callouts to the instances
+    successful = 0
+    for crawler_id in range(len(crawler_urls)):
+        prods = balanced_lists[crawler_id]
+        url = crawler_urls[crawler_id]
+
+        # Send request
+        data = {
+            'key': os.environ['CRAWLER_ACCESS_KEY'],
+            'products': prods
+        }
+        headers = {'content-type': 'application/json', 'accept': 'application/json'}
+
+        resp = requests.post(url=url, data=json.dumps(data), headers=headers)
+
+        if resp.status_code == 200:
+            successful += 1
+
+    return successful == len(crawler_urls)
+
+
+if __name__ == '__main__':
+    call_crawlers()
--- a/Crawler-Loadbalancer/requirements.txt
+++ b/Crawler-Loadbalancer/requirements.txt
@ -0,0 +1 @@
+pymysql
--- a/Crawler-Loadbalancer/sql.py
+++ b/Crawler-Loadbalancer/sql.py
@ -0,0 +1,59 @@
+import pymysql
+import os
+import logging
+
+
+def __getConnection__() -> pymysql.Connection:
+    """
+    Opens a new pymysql connection and returns it
+    :return: A pymysql Connection object
+    """
+    logger = logging.getLogger()
+    try:
+        conn = pymysql.connect(
+            user=os.environ['BETTERZON_CRAWLER_USER'],
+            password=os.environ['BETTERZON_CRAWLER_PASSWORD'],
+            host=os.environ['BETTERZON_CRAWLER_HOST'],
+            port=3306,
+            database=os.environ['BETTERZON_CRAWLER_DB']
+        )
+
+        return conn
+    except pymysql.Error as e:
+        logger.error('SQL Connection error: %s', e)
+        return
+
+
+def getShopsToCrawl() -> [int]:
+    """
+    Queries the list of vendor IDs and returns them
+    :return: The list of IDs
+    """
+    conn = __getConnection__()
+    cur = conn.cursor()
+
+    query = 'SELECT vendor_id FROM vendors'
+
+    cur.execute(query)
+
+    # Extract the IDs from the returned tuples into a list
+    vendor_ids = list(map(lambda x: x[0], cur.fetchall()))
+
+    return vendor_ids
+
+def getProductsToCrawl() -> [int]:
+    """
+    Queries the list of product IDs and returns them
+    :return: The list of IDs
+    """
+    conn = __getConnection__()
+    cur = conn.cursor()
+
+    query = 'SELECT product_id FROM products'
+
+    cur.execute(query)
+
+    # Extract the IDs from the returned tuples into a list
+    product_ids = list(map(lambda x: x[0], cur.fetchall()))
+
+    return product_ids
--- a/Crawler/Crawler.iml
+++ b/Crawler/Crawler.iml
@ -2,13 +2,13 @@
 <module type="WEB_MODULE" version="4">
  <component name="FacetManager">
    <facet type="Python" name="Python">
-      <configuration sdkName="Python 3.9 (venv)" />
+      <configuration sdkName="Python 3.9" />
    </facet>
  </component>
  <component name="NewModuleRootManager" inherit-compiler-output="true">
    <exclude-output />
    <content url="file://$MODULE_DIR$" />
    <orderEntry type="sourceFolder" forTests="false" />
-    <orderEntry type="library" name="Python 3.9 (venv) interpreter library" level="application" />
+    <orderEntry type="library" name="Python 3.9 interpreter library" level="application" />
  </component>
 </module>
--- a/Crawler/api.py
+++ b/Crawler/api.py
@ -1,14 +1,24 @@
 from flask import Flask
-from flask_restful import Resource, Api
+from flask_restful import Resource, Api, reqparse

 app = Flask(__name__)
 api = Api(app)

+# To parse request data
+parser = reqparse.RequestParser()
+parser.add_argument('key')
+parser.add_argument('products')
+

 class CrawlerApi(Resource):
    def get(self):
        return {'Hallo': 'Betterzon'}

+    def post(self):
+        # Accept crawler request here
+        args = parser.parse_args()
+        return args
+

 api.add_resource(CrawlerApi, '/')

--- a/Crawler/crawler.py
+++ b/Crawler/crawler.py
@ -0,0 +1,78 @@
+import sql
+
+
+def crawl(product_ids: [int]) -> dict:
+    """
+    Crawls the given list of products and saves the results to sql
+    :param products: The list of product IDs to fetch
+    :return: A dict with the following fields:
+                total_crawls: number of total crawl tries (products * vendors per product)
+                successful_crawls: number of successful products
+                products_with_problems: list of products that have not been crawled successfully
+    """
+    total_crawls = 0
+    successful_crawls = 0
+    products_with_problems = []
+
+    # Iterate over every product that has to be crawled
+    for product_id in product_ids:
+        # Get all links for this product
+        product_links = sql.getProductLinksForProduct(product_id)
+
+        crawled_data = []
+
+        # Iterate over every link / vendor
+        for product_vendor_info in product_links:
+            total_crawls += 1
+
+            # Call the appropriate vendor crawling function and append the result to the list of crawled data
+            if product_vendor_info['vendor_id'] == 1:
+                # Amazon
+                crawled_data.append(__crawl_amazon__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 2:
+                # Apple
+                crawled_data.append(__crawl_apple__(product_vendor_info))
+            elif product_vendor_info['vendor_id'] == 3:
+                # Media Markt
+                crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
+            else:
+                products_with_problems.append(product_vendor_info)
+                continue
+
+            successful_crawls += 1
+
+        # Insert data to SQL
+        sql.insertData(crawled_data)
+
+    return {
+        'total_crawls': total_crawls,
+        'successful_crawls': successful_crawls,
+        'products_with_problems': products_with_problems
+    }
+
+
+def __crawl_amazon__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from amazon
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_apple__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from apple
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    return (product_info['product_id'], product_info['vendor_id'], 123)
+
+
+def __crawl_mediamarkt__(product_info: dict) -> tuple:
+    """
+    Crawls the price for the given product from media markt
+    :param product_info: A dict with product info containing product_id, vendor_id, url
+    :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
+    """
+    pass
--- a/Crawler/sql.py
+++ b/Crawler/sql.py
@ -0,0 +1,89 @@
+import logging
+
+import pymysql
+import os
+
+
+def __getConnection__() -> pymysql.Connection:
+    """
+    Opens a new pymysql connection and returns it
+    :return: A pymysql Connection object
+    """
+    logger = logging.getLogger()
+    try:
+        conn = pymysql.connect(
+            user=os.environ['BETTERZON_CRAWLER_USER'],
+            password=os.environ['BETTERZON_CRAWLER_PASSWORD'],
+            host=os.environ['BETTERZON_CRAWLER_HOST'],
+            port=3306,
+            database=os.environ['BETTERZON_CRAWLER_DB']
+        )
+
+        return conn
+    except pymysql.Error as e:
+        logger.error('SQL Connection error: %s', e)
+        return
+
+
+def getProductsForVendor(vendor_id: int) -> [{}]:
+    """
+    Queries the product links for all products of the given shop
+    :param vendor_id: The vendor / shop to query products for
+    :return: A list of product objects, each having the following parameters:
+                product_id, vendor_id, url
+    """
+    conn = __getConnection__()
+    cur = conn.cursor()
+
+    query = 'SELECT product_id, url FROM product_links WHERE vendor_id = %s'
+
+    cur.execute(query, (vendor_id,))
+
+    products = list(map(lambda x: {'product_id': x[0], 'vendor_id': vendor_id, 'url': x[1]}, cur.fetchall()))
+
+    return products
+
+def getProductLinksForProduct(product_id: int) -> [dict]:
+    """
+    Queries all the product links for the given product
+    :param product_id: The product to query data for
+    :return: A list of product objects, each having the following parameters:
+                product_id, vendor_id, url
+    """
+    conn = __getConnection__()
+    cur = conn.cursor()
+
+    query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
+
+    cur.execute(query, (product_id,))
+
+    products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))
+
+    return products
+
+
+
+def insertData(data_to_insert: [tuple]) -> bool:
+    """
+    Inserts the given list of tuples into the DB
+    :param dataToInsert: A list of tuples, where each tuple has to contain product id, vendor id and the price
+                         in exactly this order
+    :return: If the insert was successful
+    """
+    conn = __getConnection__()
+    cur = conn.cursor()
+
+    query = 'INSERT INTO prices (product_id, vendor_id, price_in_cents, timestamp) VALUES (%s, %s, %s, NOW())'
+
+    affectedRows = cur.executemany(query, data_to_insert)
+
+    if affectedRows != len(data_to_insert):
+        # Something went wrong, revert the changes
+        conn.rollback()
+    else:
+        conn.commit()
+
+    cur.close()
+    conn.close()
+
+    return affectedRows == len(data_to_insert)
--- a/doku/AC_AddProducts.drawio
+++ b/doku/AC_AddProducts.drawio
@ -0,0 +1,137 @@
+<mxfile host="app.diagrams.net" modified="2021-04-17T09:55:08.113Z" agent="5.0 (Windows)" etag="_jF0V2kgXPm31R3VhRsx" version="14.6.1" type="github">
+  <diagram id="93ncRcM3GDOl-1M6vxE-" name="Page-1">
+    <mxGraphModel dx="2062" dy="1163" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="9AhRm9Tl705j5lGg98FB-1" value="" style="ellipse;html=1;shape=startState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="360" y="80" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-2" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="375" y="170" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-3" value="User" style="shape=umlActor;verticalLabelPosition=bottom;verticalAlign=top;html=1;" vertex="1" parent="1">
+          <mxGeometry x="360" width="30" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-5" value="Searches for Product" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="315" y="170" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-6" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-5" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="375" y="270" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-9" value="Finds what he is looking for?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="315" y="270" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-10" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-9" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="520" y="290" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-11" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-9" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="375" y="370" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="375" y="370" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-13" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="520" y="275" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-14" value="Clicks on &#39;Add new product&#39; on search result page" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="315" y="370" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-15" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-14" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="375" y="470" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-16" value="Enters Amazon link of product he wants" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="315" y="470" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-17" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-16" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="375" y="570" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-26" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;fillColor=#f8cecc;strokeColor=#FF0000;" edge="1" parent="1" source="9AhRm9Tl705j5lGg98FB-18" target="9AhRm9Tl705j5lGg98FB-24">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-18" value="Clicks &#39;search&#39;" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="315" y="570" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-19" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-18" parent="1" target="9AhRm9Tl705j5lGg98FB-20">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="375" y="670" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-20" value="Crawler invoked, fetches data from Amazon" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="240" y="670" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-21" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-20" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="300" y="750" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-22" value="Crawler instances try to find product at other vendors" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="240" y="860" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-23" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=0.462;exitY=1.005;exitDx=0;exitDy=0;exitPerimeter=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-22" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="295" y="960" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-24" value="User is prompted to reload page in a few seconds" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="390" y="670" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-25" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-24" parent="1" target="9AhRm9Tl705j5lGg98FB-38">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="450" y="770" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-27" value="Data saved to SQL, can be accessed from the page from this point on" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="240" y="960" width="120" height="70" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-28" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-27" parent="1" target="9AhRm9Tl705j5lGg98FB-38">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="300" y="1060" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-33" style="edgeStyle=orthogonalEdgeStyle;rounded=1;orthogonalLoop=1;jettySize=auto;html=1;strokeColor=#FF0000;" edge="1" parent="1" source="9AhRm9Tl705j5lGg98FB-30">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="200" y="770" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-30" value="Product found" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="260" y="750" width="80" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-32" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" source="9AhRm9Tl705j5lGg98FB-30" parent="1" target="9AhRm9Tl705j5lGg98FB-22">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="260" y="850" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-34" value="no" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;" vertex="1" parent="1">
+          <mxGeometry x="240" y="770" width="30" height="20" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-35" value="Tell user that product could not be found at Amazon" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="80" y="750" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-36" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="9AhRm9Tl705j5lGg98FB-35" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="140" y="850" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-37" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="125" y="850" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="9AhRm9Tl705j5lGg98FB-38" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="360" y="1050" width="30" height="30" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>
--- a/doku/AC_AddProducts.png
+++ b/doku/AC_AddProducts.png
--- a/doku/AC_Administration.drawio
+++ b/doku/AC_Administration.drawio
@ -0,0 +1,72 @@
+<mxfile host="app.diagrams.net" modified="2021-04-17T10:00:57.510Z" agent="5.0 (Windows)" etag="V0KOXZzaBS25THcsVrrB" version="14.6.1" type="github">
+  <diagram id="ky918N-3rujnzzDEFhuE" name="Page-1">
+    <mxGraphModel dx="1178" dy="1834" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="0" />
+        <mxCell id="1" parent="0" />
+        <mxCell id="feyX4JLSk9dq7nyA62vH-1" value="" style="ellipse;html=1;shape=startState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="370" y="80" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-2" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="feyX4JLSk9dq7nyA62vH-1" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="385" y="170" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-3" value="Page Admin" style="shape=umlActor;verticalLabelPosition=bottom;verticalAlign=top;html=1;" vertex="1" parent="1">
+          <mxGeometry x="370" y="-10" width="30" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-4" value="Opens Administration Page" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="170" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-5" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="feyX4JLSk9dq7nyA62vH-4" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="385" y="270" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-6" value="Crawling process currently running?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="370" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-7" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="feyX4JLSk9dq7nyA62vH-6" parent="1" target="feyX4JLSk9dq7nyA62vH-9">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="545" y="390" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-8" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="feyX4JLSk9dq7nyA62vH-6" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="385" y="470" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-9" value="Can see status of latest crawling process" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="550" y="370" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-10" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="feyX4JLSk9dq7nyA62vH-9" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="610" y="470" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-11" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="595" y="470" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-12" value="Can see status of currently running crawling status" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="470" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-13" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" source="feyX4JLSk9dq7nyA62vH-12" parent="1" target="feyX4JLSk9dq7nyA62vH-14">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="384" y="578" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-14" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="370" y="580" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-15" value="Logs in with admin credentials" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="270" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="feyX4JLSk9dq7nyA62vH-16" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="feyX4JLSk9dq7nyA62vH-15" parent="1">
+          <mxGeometry relative="1" as="geometry">
+            <mxPoint x="385" y="370" as="targetPoint" />
+          </mxGeometry>
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>
--- a/doku/AC_Administration.png
+++ b/doku/AC_Administration.png
--- a/doku/AC_Crawler.drawio
+++ b/doku/AC_Crawler.drawio
@ -1,142 +1,190 @@
-<mxfile host="app.diagrams.net" modified="2020-10-22T10:48:28.862Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36" etag="Qwq2Hbdg23HRl1kpLlkL" version="13.8.1" type="github">
+<mxfile host="app.diagrams.net" modified="2021-04-16T06:49:53.018Z" agent="5.0 (Windows)" etag="5JD6Qb7bmkoe1ST7PLQh" version="13.10.9" type="github">
  <diagram id="HsOnwiffrXz8mLfPakhB" name="Page-1">
-    <mxGraphModel dx="1125" dy="807" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+    <mxGraphModel dx="2062" dy="1163" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
      <root>
        <mxCell id="0" />
        <mxCell id="1" parent="0" />
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-1" value="" style="ellipse;html=1;shape=startState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="ALDNomAj6A-5llFqMY2C-4" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;" vertex="1" parent="1">
+          <mxGeometry x="80" y="40" width="720" height="560" as="geometry" />
+        </mxCell>
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-1" value="" style="ellipse;html=1;shape=startState;fillColor=#000000;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="370" y="70" width="30" height="30" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-2" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-2" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-1" edge="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="385" y="160" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-15" value="Read Configuration" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-15" value="Read Configuration and Data from SQL" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="325" y="160" width="120" height="40" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-16" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-15" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-16" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-15" edge="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="385" y="260" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-17" value="Configuration valid?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-17" value="Configuration valid?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="315" y="260" width="140" height="60" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-18" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-17" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-18" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-17" edge="1">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="560" y="290" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-19" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-17" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-19" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-17" edge="1">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="385" y="400" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-20" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-20" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="560" y="275" width="30" height="30" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-21" value="Shop exists&lt;br&gt;&amp;nbsp;in Database?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-21" value="&amp;gt;=1 Crawler Instance registered?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="310" y="400" width="150" height="60" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-22" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-21" parent="1" target="qIYyL-Ke0HVsc26IhBTR-24">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-22" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-21" target="qIYyL-Ke0HVsc26IhBTR-24" edge="1">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="520" y="430" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-23" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-21" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-23" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-21" edge="1">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="385" y="520" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-24" value="Create Entry" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-24" value="Send Error email" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="530" y="410" width="120" height="40" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-25" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-24" parent="1" target="qIYyL-Ke0HVsc26IhBTR-26">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="470" y="550" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="590" y="540" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-26" value="Fetch Products from Category" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-26" value="Distribute tasks across all instances" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
          <mxGeometry x="325" y="520" width="120" height="40" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-27" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-26" parent="1" target="qIYyL-Ke0HVsc26IhBTR-44">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-27" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-26" target="qIYyL-Ke0HVsc26IhBTR-44" edge="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="385" y="630" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-30" value="Product available&lt;br&gt;on Amazon?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="296.25" y="710" width="180" height="80" as="geometry" />
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-33" value="Fetch Product Data from SQL" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
+          <mxGeometry x="325" y="760" width="120" height="40" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-31" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-30" parent="1">
-          <mxGeometry x="-1" relative="1" as="geometry">
-            <mxPoint x="531.25" y="750" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-32" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-30" parent="1">
-          <mxGeometry x="-1" relative="1" as="geometry">
-            <mxPoint x="386.25" y="850" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-33" value="Discard Product" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="531.25" y="730" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-37" value="Product in Productdatabase?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="290" y="850" width="192.5" height="70" as="geometry" />
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-38" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-37" parent="1">
-          <mxGeometry x="-1" relative="1" as="geometry">
-            <mxPoint x="541.25" y="885" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-39" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-37" parent="1">
-          <mxGeometry x="-1" relative="1" as="geometry">
-            <mxPoint x="386.25" y="960" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-40" value="Add to Database" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="541.25" y="860" width="120" height="40" as="geometry" />
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-41" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-40" parent="1">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="446.25" y="980" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="601.25" y="980" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-44" value="For Product in List" style="ellipse;shape=umlControl;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-44" value="For Product in List" style="ellipse;shape=umlControl;whiteSpace=wrap;html=1;" parent="1" vertex="1">
          <mxGeometry x="350" y="650" width="70" height="80" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-45" value="Last Product?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="329.75" y="1030" width="113" height="60" as="geometry" />
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-45" value="Last Product?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" parent="1" vertex="1">
+          <mxGeometry x="328.5" y="1140" width="113" height="60" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-46" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=1.029;entryY=0.55;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-45" parent="1" target="qIYyL-Ke0HVsc26IhBTR-44">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-46" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=1.029;entryY=0.55;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-45" target="qIYyL-Ke0HVsc26IhBTR-44" edge="1">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="710" y="650" as="targetPoint" />
            <Array as="points">
-              <mxPoint x="710" y="1060" />
+              <mxPoint x="710" y="1170" />
              <mxPoint x="710" y="694" />
            </Array>
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-47" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-45" parent="1">
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-47" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" parent="1" source="qIYyL-Ke0HVsc26IhBTR-45" edge="1" target="qIYyL-Ke0HVsc26IhBTR-48">
          <mxGeometry x="-1" relative="1" as="geometry">
            <mxPoint x="386.75" y="1130" as="targetPoint" />
          </mxGeometry>
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-48" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="371.75" y="1130" width="30" height="30" as="geometry" />
+        <mxCell id="qIYyL-Ke0HVsc26IhBTR-48" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" parent="1" vertex="1">
+          <mxGeometry x="370" y="1230" width="30" height="30" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-54" value="Add Price entry to Pricedatabase" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
-          <mxGeometry x="326.25" y="960" width="120" height="40" as="geometry" />
+        <mxCell id="ALDNomAj6A-5llFqMY2C-8" value="&lt;font style=&quot;font-size: 30px&quot;&gt;Load-Balancer&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
+          <mxGeometry x="120" y="280" width="120" height="80" as="geometry" />
        </mxCell>
-        <mxCell id="qIYyL-Ke0HVsc26IhBTR-55" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" source="qIYyL-Ke0HVsc26IhBTR-54" parent="1" target="qIYyL-Ke0HVsc26IhBTR-45">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="180" y="1040" as="targetPoint" />
+        <mxCell id="ALDNomAj6A-5llFqMY2C-9" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0.01;entryDx=0;entryDy=0;entryPerimeter=0;exitX=0.497;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;" edge="1" parent="1" source="qIYyL-Ke0HVsc26IhBTR-44" target="qIYyL-Ke0HVsc26IhBTR-33">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="455" y="810" as="targetPoint" />
+            <mxPoint x="385" y="732" as="sourcePoint" />
+            <Array as="points">
+              <mxPoint x="385" y="732" />
+              <mxPoint x="385" y="732" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-10" value="Crawl Price using appropriate crawling function" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="910" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-12" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0.01;entryDx=0;entryDy=0;entryPerimeter=0;" edge="1" parent="1" target="ALDNomAj6A-5llFqMY2C-10">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="452.75" y="960" as="targetPoint" />
+            <mxPoint x="385" y="890" as="sourcePoint" />
+            <Array as="points">
+              <mxPoint x="385" y="890" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-14" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=0.497;exitY=1;exitDx=0;exitDy=0;exitPerimeter=0;" edge="1" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="385" y="820" as="targetPoint" />
+            <mxPoint x="384.88000000000005" y="800" as="sourcePoint" />
+            <Array as="points">
+              <mxPoint x="385.09" y="802" />
+              <mxPoint x="385.09" y="802" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-11" value="For Vendor in List" style="ellipse;shape=umlControl;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="350" y="810" width="70" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-16" value="Last Vendor?" style="rhombus;whiteSpace=wrap;html=1;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="328.5" y="980" width="113" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-18" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="ALDNomAj6A-5llFqMY2C-10" target="ALDNomAj6A-5llFqMY2C-16">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="441.5" y="970" as="targetPoint" />
+            <mxPoint x="396.5" y="960" as="sourcePoint" />
+            <Array as="points" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-19" value="no" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="ALDNomAj6A-5llFqMY2C-16">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="420" y="850" as="targetPoint" />
+            <Array as="points">
+              <mxPoint x="610" y="1010" />
+              <mxPoint x="610" y="850" />
+            </Array>
+            <mxPoint x="459.47" y="1050" as="sourcePoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-20" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="385" y="980" as="targetPoint" />
+            <mxPoint x="385" y="950" as="sourcePoint" />
+            <Array as="points" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-21" value="yes" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="384.83" y="1070" as="targetPoint" />
+            <mxPoint x="384.83" y="1040" as="sourcePoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-22" value="Save price entries to SQL" style="rounded=1;whiteSpace=wrap;html=1;arcSize=40;fontColor=#000000;fillColor=#ffffc0;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="325" y="1070" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-24" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=top;endArrow=open;endSize=8;strokeColor=#ff0000;entryX=0.5;entryY=0;entryDx=0;entryDy=0;" edge="1" parent="1">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="384.8299999999999" y="1140" as="targetPoint" />
+            <mxPoint x="384.8299999999999" y="1110" as="sourcePoint" />
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-25" value="&lt;font style=&quot;font-size: 30px&quot;&gt;Load-Balancer&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
+          <mxGeometry x="120" y="280" width="120" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-26" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=none;" vertex="1" parent="1">
+          <mxGeometry x="80" y="640" width="720" height="640" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-27" value="&lt;font style=&quot;font-size: 29px&quot;&gt;Crawler Instance&lt;br&gt;&lt;/font&gt;" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
+          <mxGeometry x="120" y="920" width="120" height="80" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-2" value="" style="ellipse;html=1;shape=endState;fillColor=#000000;strokeColor=#ff0000;" vertex="1" parent="1">
+          <mxGeometry x="700" y="415" width="30" height="30" as="geometry" />
+        </mxCell>
+        <mxCell id="ALDNomAj6A-5llFqMY2C-28" value="" style="edgeStyle=orthogonalEdgeStyle;html=1;align=left;verticalAlign=bottom;endArrow=open;endSize=8;strokeColor=#ff0000;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="qIYyL-Ke0HVsc26IhBTR-24" target="ALDNomAj6A-5llFqMY2C-2">
+          <mxGeometry x="-1" relative="1" as="geometry">
+            <mxPoint x="756.25" y="430" as="targetPoint" />
+            <mxPoint x="650" y="430" as="sourcePoint" />
          </mxGeometry>
        </mxCell>
      </root>
--- a/doku/AC_Crawler.png
+++ b/doku/AC_Crawler.png