From 16ed1070c2ba5c1f68db3bad49ee58c92990f4a3 Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Tue, 18 May 2021 00:24:00 +0200 Subject: [PATCH 1/8] BETTERZON-97: Adding API endpoint to get all products listed by a specific vendor (#50) --- .../src/models/products/products.router.ts | 19 ++++++++++ .../src/models/products/products.service.ts | 38 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/Backend/src/models/products/products.router.ts b/Backend/src/models/products/products.router.ts index f0ea9a3..dd332e5 100644 --- a/Backend/src/models/products/products.router.ts +++ b/Backend/src/models/products/products.router.ts @@ -87,3 +87,22 @@ productsRouter.get('/list/:ids', async (req: Request, res: Response) => { res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); } }); + +// GET products/vendor/:id +productsRouter.get('/vendor/:id', async (req: Request, res: Response) => { + const id: number = parseInt(req.params.id, 10); + + if (!id) { + res.status(400).send('Missing parameters.'); + return; + } + + try { + const products: Products = await ProductService.findByVendor(id); + + res.status(200).send(products); + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); diff --git a/Backend/src/models/products/products.service.ts b/Backend/src/models/products/products.service.ts index c0bee72..7d397ab 100644 --- a/Backend/src/models/products/products.service.ts +++ b/Backend/src/models/products/products.service.ts @@ -159,3 +159,41 @@ export const findList = async (ids: [number]): Promise => { return prodRows; }; + +/** + * Fetches and returns the products that the given vendor has price entries for + * @param id The id of the vendor to fetch the products for + */ +export const findByVendor = async (id: number): Promise => { + let conn; + let prodRows = []; + try { + conn = await pool.getConnection(); + + // Get the relevant product ids + let relevant_prod_ids = []; + const relevantProds = await conn.query('SELECT product_id FROM prices WHERE vendor_id = ? GROUP BY product_id', id); + for (let row in relevantProds) { + if (row !== 'meta') { + relevant_prod_ids.push(relevantProds[row].product_id); + } + } + + // Fetch products + const rows = await conn.query('SELECT product_id, name, asin, is_active, short_description, long_description, image_guid, date_added, last_modified, manufacturer_id, selling_rank, category_id FROM products WHERE product_id IN (?)', [relevant_prod_ids]); + for (let row in rows) { + if (row !== 'meta') { + prodRows.push(rows[row]); + } + } + + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } + + return prodRows; +}; From 8f17ae789699650a8f6b2eaee82f03879154b88f Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Tue, 18 May 2021 00:40:24 +0200 Subject: [PATCH 2/8] BETTERZON-98: Adding API endpoint for adding price entries as a registered vendor manager (#51) --- Backend/src/models/prices/prices.router.ts | 26 +++++++++++++++++++ Backend/src/models/prices/prices.service.ts | 26 +++++++++++++++++++ Backend/src/models/vendors/vendors.router.ts | 2 +- Backend/src/models/vendors/vendors.service.ts | 5 +--- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/Backend/src/models/prices/prices.router.ts b/Backend/src/models/prices/prices.router.ts index 12ffaf1..047ce1b 100644 --- a/Backend/src/models/prices/prices.router.ts +++ b/Backend/src/models/prices/prices.router.ts @@ -6,6 +6,7 @@ import express, {Request, Response} from 'express'; import * as PriceService from './prices.service'; import {Price} from './price.interface'; import {Prices} from './prices.interface'; +import * as UserService from '../users/users.service'; /** @@ -100,3 +101,28 @@ pricesRouter.get('/byProduct/list/:ids', async (req: Request, res: Response) => res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); } }); + +// POST prices/ +pricesRouter.post('/', async (req: Request, res: Response) => { + try { + // Authenticate user + const user_ip = req.connection.remoteAddress ?? ''; + const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip); + + // Get required parameters + const vendor_id = req.body.vendor_id; + const product_id = req.body.product_id; + const price_in_cents = req.body.price_in_cents; + + const success = await PriceService.createPriceEntry(user.user_id, vendor_id, product_id, price_in_cents); + + if (success) { + res.sendStatus(200); + } else { + res.sendStatus(500); + } + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); diff --git a/Backend/src/models/prices/prices.service.ts b/Backend/src/models/prices/prices.service.ts index 405cfd4..0a5ab2a 100644 --- a/Backend/src/models/prices/prices.service.ts +++ b/Backend/src/models/prices/prices.service.ts @@ -376,3 +376,29 @@ export const findListByProducts = async (productIds: [number]): Promise return priceRows; }; + +export const createPriceEntry = async (user_id: number, vendor_id: number, product_id: number, price_in_cents: number): Promise => { + let conn; + try { + conn = await pool.getConnection(); + + // Check if the user is authorized to manage the requested vendor + const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]); + if (user_vendor_rows.length !== 1) { + return false; + } + + // Create price entry + const res = await conn.query('INSERT INTO prices (product_id, vendor_id, price_in_cents) VALUES (?,?,?)', [product_id, vendor_id, price_in_cents]); + + // If there are more / less than 1 affected rows, return false + return res.affectedRows === 1; + + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } +}; diff --git a/Backend/src/models/vendors/vendors.router.ts b/Backend/src/models/vendors/vendors.router.ts index 7ca180d..7440f4f 100644 --- a/Backend/src/models/vendors/vendors.router.ts +++ b/Backend/src/models/vendors/vendors.router.ts @@ -99,7 +99,7 @@ vendorsRouter.put('/manage/deactivatelisting', async (req: Request, res: Respons const success = await VendorService.deactivateListing(user.user_id, vendor_id, product_id); - if(success) { + if (success) { res.sendStatus(200); } else { res.sendStatus(500); diff --git a/Backend/src/models/vendors/vendors.service.ts b/Backend/src/models/vendors/vendors.service.ts index 1dc2220..39fee7d 100644 --- a/Backend/src/models/vendors/vendors.service.ts +++ b/Backend/src/models/vendors/vendors.service.ts @@ -171,10 +171,7 @@ export const deactivateListing = async (user_id: number, vendor_id: number, prod const status = await conn.query('UPDATE prices SET active_listing = false WHERE vendor_id = ? and product_id = ?', [vendor_id, product_id]); - if(status.affectedRows > 0){ - return true; - } - return false; + return status.affectedRows > 0; } catch (err) { throw err; } finally { From 3ae68b3df3453d99c2bdb2cc99ea8ba36e736b4b Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Tue, 18 May 2021 21:14:00 +0200 Subject: [PATCH 3/8] BETTERZON-95: Adding API endpoint for getting, inserting and updating contact persons (#52) --- Backend/src/index.ts | 2 + .../contact_person.interface.ts | 9 + .../contact_persons.interface.ts | 5 + .../contact_persons/contact_persons.router.ts | 129 +++++++++++++ .../contact_persons.service.ts | 175 ++++++++++++++++++ .../manufacturers/manufacturers.router.ts | 6 +- 6 files changed, 323 insertions(+), 3 deletions(-) create mode 100644 Backend/src/models/contact_persons/contact_person.interface.ts create mode 100644 Backend/src/models/contact_persons/contact_persons.interface.ts create mode 100644 Backend/src/models/contact_persons/contact_persons.router.ts create mode 100644 Backend/src/models/contact_persons/contact_persons.service.ts diff --git a/Backend/src/index.ts b/Backend/src/index.ts index 79173b3..b387dd9 100644 --- a/Backend/src/index.ts +++ b/Backend/src/index.ts @@ -15,6 +15,7 @@ import {errorHandler} from './middleware/error.middleware'; import {notFoundHandler} from './middleware/notFound.middleware'; import {usersRouter} from './models/users/users.router'; import {pricealarmsRouter} from './models/pricealarms/pricealarms.router'; +import {contactpersonsRouter} from './models/contact_persons/contact_persons.router'; const cookieParser = require('cookie-parser'); @@ -49,6 +50,7 @@ app.use('/prices', pricesRouter); app.use('/users', usersRouter); app.use('/vendors', vendorsRouter); app.use('/pricealarms', pricealarmsRouter); +app.use('/contactpersons', contactpersonsRouter); app.use(errorHandler); app.use(notFoundHandler); diff --git a/Backend/src/models/contact_persons/contact_person.interface.ts b/Backend/src/models/contact_persons/contact_person.interface.ts new file mode 100644 index 0000000..e777a40 --- /dev/null +++ b/Backend/src/models/contact_persons/contact_person.interface.ts @@ -0,0 +1,9 @@ +export interface Contact_Person { + contact_person_id: number; + first_name: string; + last_name: string; + gender: string; + email: string; + phone: string; + vendor_id: number; +} diff --git a/Backend/src/models/contact_persons/contact_persons.interface.ts b/Backend/src/models/contact_persons/contact_persons.interface.ts new file mode 100644 index 0000000..97f8393 --- /dev/null +++ b/Backend/src/models/contact_persons/contact_persons.interface.ts @@ -0,0 +1,5 @@ +import {Contact_Person} from './contact_person.interface'; + +export interface Contact_Persons { + [key: number]: Contact_Person; +} diff --git a/Backend/src/models/contact_persons/contact_persons.router.ts b/Backend/src/models/contact_persons/contact_persons.router.ts new file mode 100644 index 0000000..bb2d1a0 --- /dev/null +++ b/Backend/src/models/contact_persons/contact_persons.router.ts @@ -0,0 +1,129 @@ +/** + * Required External Modules and Interfaces + */ + +import express, {Request, Response} from 'express'; +import * as ContactPersonService from './contact_persons.service'; +import {Contact_Person} from './contact_person.interface'; +import {Contact_Persons} from './contact_persons.interface'; +import * as UserService from '../users/users.service'; +import * as PriceService from '../prices/prices.service'; + + +/** + * Router Definition + */ + +export const contactpersonsRouter = express.Router(); + + +/** + * Controller Definitions + */ + +// GET contactpersons/ +contactpersonsRouter.get('/', async (req: Request, res: Response) => { + try { + const contacts: Contact_Persons = await ContactPersonService.findAll(); + + res.status(200).send(contacts); + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); + +// GET contactpersons/:id +contactpersonsRouter.get('/:id', async (req: Request, res: Response) => { + const id: number = parseInt(req.params.id, 10); + + if (!id) { + res.status(400).send('Missing parameters.'); + return; + } + + try { + const contact: Contact_Person = await ContactPersonService.find(id); + + res.status(200).send(contact); + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); + +// GET contactpersons/byvendor/:id +contactpersonsRouter.get('/byvendor/:id', async (req: Request, res: Response) => { + const id: number = parseInt(req.params.id, 10); + + if (!id) { + res.status(400).send('Missing parameters.'); + return; + } + + try { + const contacts: Contact_Persons = await ContactPersonService.findByVendor(id); + + res.status(200).send(contacts); + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); + +// POST contactpersons/ +contactpersonsRouter.post('/', async (req: Request, res: Response) => { + try { + // Authenticate user + const user_ip = req.connection.remoteAddress ?? ''; + const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip); + + // Get required parameters + const vendor_id = req.body.vendor_id; + const first_name = req.body.first_name; + const last_name = req.body.last_name; + const gender = req.body.gender; + const email = req.body.email; + const phone = req.body.phone; + + const success = await ContactPersonService.createContactEntry(user.user_id, vendor_id, first_name, last_name, gender, email, phone); + + if (success) { + res.sendStatus(200); + } else { + res.sendStatus(500); + } + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); + +// PUT contactpersons/:id +contactpersonsRouter.put('/:id', async (req: Request, res: Response) => { + try { + // Authenticate user + const user_ip = req.connection.remoteAddress ?? ''; + const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip); + + // Get required parameters + const contact_person_id = parseInt(req.params.id, 10); + const vendor_id = req.body.vendor_id; + const first_name = req.body.first_name; + const last_name = req.body.last_name; + const gender = req.body.gender; + const email = req.body.email; + const phone = req.body.phone; + + const success = await ContactPersonService.updateContactEntry(user.user_id, contact_person_id, vendor_id, first_name, last_name, gender, email, phone); + + if (success) { + res.sendStatus(200); + } else { + res.sendStatus(500); + } + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); diff --git a/Backend/src/models/contact_persons/contact_persons.service.ts b/Backend/src/models/contact_persons/contact_persons.service.ts new file mode 100644 index 0000000..7e23191 --- /dev/null +++ b/Backend/src/models/contact_persons/contact_persons.service.ts @@ -0,0 +1,175 @@ +import * as dotenv from 'dotenv'; + +dotenv.config(); + +const mariadb = require('mariadb'); +const pool = mariadb.createPool({ + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_DATABASE, + connectionLimit: 5 +}); + +/** + * Data Model Interfaces + */ + +import {Contact_Person} from './contact_person.interface'; +import {Contact_Persons} from './contact_persons.interface'; + + +/** + * Service Methods + */ + +/** + * Fetches and returns all known contact persons + */ +export const findAll = async (): Promise => { + let conn; + let contRows = []; + try { + conn = await pool.getConnection(); + const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons'); + for (let row in rows) { + if (row !== 'meta') { + contRows.push(rows[row]); + } + } + + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } + + return contRows; +}; + +/** + * Fetches and returns the contact person with the specified id + * @param id The id of the contact person to fetch + */ +export const find = async (id: number): Promise => { + let conn; + let cont: any; + try { + conn = await pool.getConnection(); + const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons WHERE contact_person_id = ?', id); + for (let row in rows) { + if (row !== 'meta') { + cont = rows[row]; + } + } + + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } + + return cont; +}; + +/** + * Fetches and returns the contact persons for the specified vendor + * @param id The id of the vendor to fetch contact persons for + */ +export const findByVendor = async (id: number): Promise => { + let conn; + let contRows = []; + try { + conn = await pool.getConnection(); + const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons WHERE vendor_id = ?', id); + for (let row in rows) { + if (row !== 'meta') { + contRows.push(rows[row]); + } + } + + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } + + return contRows; +}; + +/** + * Creates a contact entry record + * @param user_id The user id of the issuing user + * @param vendor_id The vendor id of the vendor to create the record for + * @param first_name The first name of the contact person + * @param last_name The last name of the contact person + * @param gender The gender of the contact person + * @param email The email of the contact person + * @param phone The phone number of the contact person + */ +export const createContactEntry = async (user_id: number, vendor_id: number, first_name: string, last_name: string, gender: string, email: string, phone: string): Promise => { + let conn; + try { + conn = await pool.getConnection(); + + // Check if the user is authorized to manage the requested vendor + const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]); + if (user_vendor_rows.length !== 1) { + return false; + } + + // Create contact person entry + const res = await conn.query('INSERT INTO contact_persons (first_name, last_name, gender, email, phone, vendor_id) VALUES (?, ?, ?, ?, ?, ?)', [first_name, last_name, gender, email, phone, vendor_id]); + + // If there are more / less than 1 affected rows, return false + return res.affectedRows === 1; + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } +}; + +/** + * Updates a contact entry record + * @param user_id The user id of the issuing user + * @param contact_person_id The id of the record to update + * @param vendor_id The vendor id of the vendor to create the record for + * @param first_name The first name of the contact person + * @param last_name The last name of the contact person + * @param gender The gender of the contact person + * @param email The email of the contact person + * @param phone The phone number of the contact person + */ +export const updateContactEntry = async (user_id: number, contact_person_id: number, vendor_id: number, first_name: string, last_name: string, gender: string, email: string, phone: string): Promise => { + let conn; + try { + conn = await pool.getConnection(); + + // Check if the user is authorized to manage the requested vendor + const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]); + if (user_vendor_rows.length !== 1) { + return false; + } + + // Create contact person entry + const res = await conn.query('UPDATE contact_persons SET first_name = ?, last_name = ?, gender = ?, email = ?, phone = ? WHERE contact_person_id = ? AND vendor_id = ?', [first_name, last_name, gender, email, phone, contact_person_id, vendor_id]); + + // If there are more / less than 1 affected rows, return false + return res.affectedRows === 1; + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } +}; diff --git a/Backend/src/models/manufacturers/manufacturers.router.ts b/Backend/src/models/manufacturers/manufacturers.router.ts index e78815c..99b0875 100644 --- a/Backend/src/models/manufacturers/manufacturers.router.ts +++ b/Backend/src/models/manufacturers/manufacturers.router.ts @@ -19,7 +19,7 @@ export const manufacturersRouter = express.Router(); * Controller Definitions */ -// GET items/ +// GET manufacturers/ manufacturersRouter.get('/', async (req: Request, res: Response) => { try { const manufacturers: Manufacturers = await ManufacturerService.findAll(); @@ -31,7 +31,7 @@ manufacturersRouter.get('/', async (req: Request, res: Response) => { } }); -// GET items/:id +// GET manufacturers/:id manufacturersRouter.get('/:id', async (req: Request, res: Response) => { const id: number = parseInt(req.params.id, 10); @@ -50,7 +50,7 @@ manufacturersRouter.get('/:id', async (req: Request, res: Response) => { } }); -// GET items/:term +// GET manufacturers/:term manufacturersRouter.get('/search/:term', async (req: Request, res: Response) => { const term: string = req.params.term; From 26ba21156aba2a6d8de8766a095460d5b0ecf533 Mon Sep 17 00:00:00 2001 From: henningxtro Date: Wed, 19 May 2021 00:46:14 +0200 Subject: [PATCH 4/8] BETTERZON-58 (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * BETTERZON-58: Basic Functionality with scrapy * Added independent crawler function, yielding price * moved logic to amazon.py * . * moved scrapy files to unused folder * Added basic amazon crawler using beautifulsoup4 * Connected Api to Crawler * Fixed string concatenation for sql statement in getProductLinksForProduct * BETTERZON-58: Fixing SQL insert * BETTERZON-58: Adding access key verification * BETTERZON-58: Fixing API endpoint of the crawler - The list of products in the API request was treated like a string and henceforth, only the first product has been crawled * Added another selector for price on amazon (does not work for books) Co-authored-by: root Co-authored-by: Patrick Müller Co-authored-by: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> --- Crawler/Crawler.iml | 4 +- Crawler/api.py | 15 +- Crawler/crawler.py | 185 ++++++++++-------- Crawler/crawler/spiders/amazon.py | 66 ------- Crawler/requirements.txt | 6 +- Crawler/sql.py | 1 - Crawler/unused/scrapy/amazonspider.py | 33 ++++ .../{ => unused/scrapy}/crawler/__init__.py | 0 Crawler/{ => unused/scrapy}/crawler/items.py | 0 .../scrapy}/crawler/middlewares.py | 0 .../{ => unused/scrapy}/crawler/pipelines.py | 0 .../{ => unused/scrapy}/crawler/settings.py | 0 Crawler/{ => unused/scrapy}/scrapy.cfg | 2 +- .../scrapy}/spiders/__init__.py | 0 Crawler/unused/scrapy/spiders/amazon.py | 25 +++ 15 files changed, 184 insertions(+), 153 deletions(-) delete mode 100644 Crawler/crawler/spiders/amazon.py create mode 100644 Crawler/unused/scrapy/amazonspider.py rename Crawler/{ => unused/scrapy}/crawler/__init__.py (100%) rename Crawler/{ => unused/scrapy}/crawler/items.py (100%) rename Crawler/{ => unused/scrapy}/crawler/middlewares.py (100%) rename Crawler/{ => unused/scrapy}/crawler/pipelines.py (100%) rename Crawler/{ => unused/scrapy}/crawler/settings.py (100%) rename Crawler/{ => unused/scrapy}/scrapy.cfg (92%) rename Crawler/{crawler => unused/scrapy}/spiders/__init__.py (100%) create mode 100644 Crawler/unused/scrapy/spiders/amazon.py diff --git a/Crawler/Crawler.iml b/Crawler/Crawler.iml index 8568e2d..db1dd1c 100644 --- a/Crawler/Crawler.iml +++ b/Crawler/Crawler.iml @@ -2,13 +2,13 @@ - + - + \ No newline at end of file diff --git a/Crawler/api.py b/Crawler/api.py index 92617c4..7b7e0c2 100644 --- a/Crawler/api.py +++ b/Crawler/api.py @@ -1,13 +1,17 @@ +import os + from flask import Flask from flask_restful import Resource, Api, reqparse +import crawler + app = Flask(__name__) api = Api(app) # To parse request data parser = reqparse.RequestParser() -parser.add_argument('key') -parser.add_argument('products') +parser.add_argument('key', type=str) +parser.add_argument('products', type=int, action='append') class CrawlerApi(Resource): @@ -17,7 +21,12 @@ class CrawlerApi(Resource): def post(self): # Accept crawler request here args = parser.parse_args() - return args + access_key = os.getenv('CRAWLER_ACCESS_KEY') + if(args['key'] == access_key): + crawler.crawl(args['products']) + return {'message': 'success'} + else: + return {'message': 'Wrong access key'} api.add_resource(CrawlerApi, '/') diff --git a/Crawler/crawler.py b/Crawler/crawler.py index 99ff867..45bd15a 100644 --- a/Crawler/crawler.py +++ b/Crawler/crawler.py @@ -1,78 +1,107 @@ -import sql - - -def crawl(product_ids: [int]) -> dict: - """ - Crawls the given list of products and saves the results to sql - :param products: The list of product IDs to fetch - :return: A dict with the following fields: - total_crawls: number of total crawl tries (products * vendors per product) - successful_crawls: number of successful products - products_with_problems: list of products that have not been crawled successfully - """ - total_crawls = 0 - successful_crawls = 0 - products_with_problems = [] - - # Iterate over every product that has to be crawled - for product_id in product_ids: - # Get all links for this product - product_links = sql.getProductLinksForProduct(product_id) - - crawled_data = [] - - # Iterate over every link / vendor - for product_vendor_info in product_links: - total_crawls += 1 - - # Call the appropriate vendor crawling function and append the result to the list of crawled data - if product_vendor_info['vendor_id'] == 1: - # Amazon - crawled_data.append(__crawl_amazon__(product_vendor_info)) - elif product_vendor_info['vendor_id'] == 2: - # Apple - crawled_data.append(__crawl_apple__(product_vendor_info)) - elif product_vendor_info['vendor_id'] == 3: - # Media Markt - crawled_data.append(__crawl_mediamarkt__(product_vendor_info)) - else: - products_with_problems.append(product_vendor_info) - continue - - successful_crawls += 1 - - # Insert data to SQL - sql.insertData(crawled_data) - - return { - 'total_crawls': total_crawls, - 'successful_crawls': successful_crawls, - 'products_with_problems': products_with_problems - } - - -def __crawl_amazon__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from amazon - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - return (product_info['product_id'], product_info['vendor_id'], 123) - - -def __crawl_apple__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from apple - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - return (product_info['product_id'], product_info['vendor_id'], 123) - - -def __crawl_mediamarkt__(product_info: dict) -> tuple: - """ - Crawls the price for the given product from media markt - :param product_info: A dict with product info containing product_id, vendor_id, url - :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) - """ - pass +import sql +import requests +from bs4 import BeautifulSoup + +HEADERS = ({'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 ' + 'Safari/537.36'}) + + +def crawl(product_ids: [int]) -> dict: + """ + Crawls the given list of products and saves the results to sql + :param products: The list of product IDs to fetch + :return: A dict with the following fields: + total_crawls: number of total crawl tries (products * vendors per product) + successful_crawls: number of successful products + products_with_problems: list of products that have not been crawled successfully + """ + total_crawls = 0 + successful_crawls = 0 + products_with_problems = [] + + # Iterate over every product that has to be crawled + for product_id in product_ids: + # Get all links for this product + product_links = sql.getProductLinksForProduct(product_id) + + crawled_data = [] + + # Iterate over every link / vendor + for product_vendor_info in product_links: + total_crawls += 1 + + # Call the appropriate vendor crawling function and append the result to the list of crawled data + if product_vendor_info['vendor_id'] == 1: + # Amazon + data = __crawl_amazon__(product_vendor_info) + if data: + crawled_data.append(data) + elif product_vendor_info['vendor_id'] == 2: + # Apple + data = __crawl_apple__(product_vendor_info) + if data: + crawled_data.append(data) + elif product_vendor_info['vendor_id'] == 3: + # Media Markt + data = __crawl_mediamarkt__(product_vendor_info) + if data: + crawled_data.append(data) + else: + products_with_problems.append(product_vendor_info) + continue + + successful_crawls += 1 + + # Insert data to SQL + sql.insertData(crawled_data) + + return { + 'total_crawls': total_crawls, + 'successful_crawls': successful_crawls, + 'products_with_problems': products_with_problems + } + + +def __crawl_amazon__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from amazon + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + page = requests.get(product_info['url'], headers=HEADERS) + soup = BeautifulSoup(page.content, features="lxml") + try: + price = int( + soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip()) + if not price: + price = int(soup.find(id='price_inside_buybox').get_text().replace(".", "").replace(",", "").replace("€", "").strip()) + + except RuntimeError: + price = -1 + except AttributeError: + price = -1 + + if price != -1: + return (product_info['product_id'], product_info['vendor_id'], price) + else: + return None + + +def __crawl_apple__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from apple + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + # return (product_info['product_id'], product_info['vendor_id'], 123) + pass + + +def __crawl_mediamarkt__(product_info: dict) -> tuple: + """ + Crawls the price for the given product from media markt + :param product_info: A dict with product info containing product_id, vendor_id, url + :return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents) + """ + pass diff --git a/Crawler/crawler/spiders/amazon.py b/Crawler/crawler/spiders/amazon.py deleted file mode 100644 index 12ea3d5..0000000 --- a/Crawler/crawler/spiders/amazon.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -import scrapy -from urllib.parse import urlencode -from urllib.parse import urljoin -import re -import json - -queries = ['iphone'] -API = '' - - -def get_url(url): - payload = {'api_key': API, 'url': url, 'country_code': 'us'} - proxy_url = 'http://api.scraperapi.com/?' + urlencode(payload) - return proxy_url - - -class AmazonSpider(scrapy.Spider): - name = 'amazon' - - def start_requests(self): - for query in queries: - url = 'https://www.amazon.de/s?' + urlencode({'k': query}) - yield scrapy.Request(url=url, callback=self.parse_keyword_response) - - def parse_keyword_response(self, response): - products = response.xpath('//*[@data-asin]') - - for product in products: - asin = product.xpath('@data-asin').extract_first() - product_url = f"https://www.amazon.de/dp/{asin}" - yield scrapy.Request(url=product_url, callback=self.parse_product_page, meta={'asin': asin}) - - next_page = response.xpath('//li[@class="a-last"]/a/@href').extract_first() - if next_page: - url = urljoin("https://www.amazon.de", next_page) - yield scrapy.Request(url=url, callback=self.parse_keyword_response) - - def parse_product_page(self, response): - asin = response.meta['asin'] - title = response.xpath('//*[@id="productTitle"]/text()').extract_first() - image = re.search('"large":"(.*?)"', response.text).groups()[0] - rating = response.xpath('//*[@id="acrPopover"]/@title').extract_first() - number_of_reviews = response.xpath('//*[@id="acrCustomerReviewText"]/text()').extract_first() - price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first() - - if not price: - price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \ - response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first() - - temp = response.xpath('//*[@id="twister"]') - sizes = [] - colors = [] - if temp: - s = re.search('"variationValues" : ({.*})', response.text).groups()[0] - json_acceptable = s.replace("'", "\"") - di = json.loads(json_acceptable) - sizes = di.get('size_name', []) - colors = di.get('color_name', []) - - bullet_points = response.xpath('//*[@id="feature-bullets"]//li/span/text()').extract() - seller_rank = response.xpath( - '//*[text()="Amazon Best Sellers Rank:"]/parent::*//text()[not(parent::style)]').extract() - yield {'asin': asin, 'Title': title, 'MainImage': image, 'Rating': rating, 'NumberOfReviews': number_of_reviews, - 'Price': price, 'AvailableSizes': sizes, 'AvailableColors': colors, 'BulletPoints': bullet_points, - 'SellerRank': seller_rank} diff --git a/Crawler/requirements.txt b/Crawler/requirements.txt index ba99df1..a704f27 100644 --- a/Crawler/requirements.txt +++ b/Crawler/requirements.txt @@ -1,5 +1,7 @@ pymysql -flask +flask==1.1.2 flask-sqlalchemy flask_restful -scrapy \ No newline at end of file +beautifulsoup4 +requests +lxml \ No newline at end of file diff --git a/Crawler/sql.py b/Crawler/sql.py index 1cf3a58..c1b2669 100644 --- a/Crawler/sql.py +++ b/Crawler/sql.py @@ -54,7 +54,6 @@ def getProductLinksForProduct(product_id: int) -> [dict]: cur = conn.cursor() query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s' - cur.execute(query, (product_id,)) products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall())) diff --git a/Crawler/unused/scrapy/amazonspider.py b/Crawler/unused/scrapy/amazonspider.py new file mode 100644 index 0000000..5f88e20 --- /dev/null +++ b/Crawler/unused/scrapy/amazonspider.py @@ -0,0 +1,33 @@ +import scrapy +from scrapy.crawler import CrawlerProcess +import re + +class AmazonSpider(scrapy.Spider): + name = 'amazon' + allowed_domains = ['amazon.de'] + start_urls = ['https://amazon.de/dp/B083DRCPJG'] + + # def __init__(self, start_urls): + # self.start_urls = start_urls + + def parse(self, response): + price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first() + if not price: + price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \ + response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first() + + euros = re.match('(\d*),\d\d', price).group(1) + cents = re.match('\d*,(\d\d)', price).group(1) + priceincents = euros + cents + + yield {'price': priceincents} + + +def start_crawling(): + process = CrawlerProcess( + settings={'COOKIES_ENABLED': 'False', 'CONCURRENT_REQUESTS_PER_IP': 1, 'ROBOTSTXT_OBEY': False, + 'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36', + 'DOWNLOAD_DELAY': 3} + , install_root_handler=False) + process.crawl() + process.start() diff --git a/Crawler/crawler/__init__.py b/Crawler/unused/scrapy/crawler/__init__.py similarity index 100% rename from Crawler/crawler/__init__.py rename to Crawler/unused/scrapy/crawler/__init__.py diff --git a/Crawler/crawler/items.py b/Crawler/unused/scrapy/crawler/items.py similarity index 100% rename from Crawler/crawler/items.py rename to Crawler/unused/scrapy/crawler/items.py diff --git a/Crawler/crawler/middlewares.py b/Crawler/unused/scrapy/crawler/middlewares.py similarity index 100% rename from Crawler/crawler/middlewares.py rename to Crawler/unused/scrapy/crawler/middlewares.py diff --git a/Crawler/crawler/pipelines.py b/Crawler/unused/scrapy/crawler/pipelines.py similarity index 100% rename from Crawler/crawler/pipelines.py rename to Crawler/unused/scrapy/crawler/pipelines.py diff --git a/Crawler/crawler/settings.py b/Crawler/unused/scrapy/crawler/settings.py similarity index 100% rename from Crawler/crawler/settings.py rename to Crawler/unused/scrapy/crawler/settings.py diff --git a/Crawler/scrapy.cfg b/Crawler/unused/scrapy/scrapy.cfg similarity index 92% rename from Crawler/scrapy.cfg rename to Crawler/unused/scrapy/scrapy.cfg index 83a4eef..9c0c1bc 100644 --- a/Crawler/scrapy.cfg +++ b/Crawler/unused/scrapy/scrapy.cfg @@ -8,4 +8,4 @@ default = crawler.settings [deploy] #url = http://localhost:6800/ -project = crawler +project = crawler \ No newline at end of file diff --git a/Crawler/crawler/spiders/__init__.py b/Crawler/unused/scrapy/spiders/__init__.py similarity index 100% rename from Crawler/crawler/spiders/__init__.py rename to Crawler/unused/scrapy/spiders/__init__.py diff --git a/Crawler/unused/scrapy/spiders/amazon.py b/Crawler/unused/scrapy/spiders/amazon.py new file mode 100644 index 0000000..c74196b --- /dev/null +++ b/Crawler/unused/scrapy/spiders/amazon.py @@ -0,0 +1,25 @@ +import scrapy +import re + +class AmazonSpider(scrapy.Spider): + name = 'amazon' + allowed_domains = ['amazon.de'] + start_urls = ['https://amazon.de/dp/B083DRCPJG'] + + def parse(self, response): + price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first() + if not price: + price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \ + response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first() + + euros = re.match('(\d*),\d\d', price).group(1) + cents = re.match('\d*,(\d\d)', price).group(1) + priceincents = euros + cents + + yield {'price': priceincents} + + + + + + From 1581184b5742abd39228738e6c405e2772a7cfb0 Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Wed, 19 May 2021 09:08:52 +0200 Subject: [PATCH 5/8] BETTERZON-96: Adding API endpoint for delisting a whole vendor (#54) --- Backend/src/models/prices/prices.service.ts | 30 ++++++------ Backend/src/models/vendors/vendors.router.ts | 46 +++++++++++++++++++ Backend/src/models/vendors/vendors.service.ts | 38 +++++++++++++-- 3 files changed, 95 insertions(+), 19 deletions(-) diff --git a/Backend/src/models/prices/prices.service.ts b/Backend/src/models/prices/prices.service.ts index 0a5ab2a..acc6793 100644 --- a/Backend/src/models/prices/prices.service.ts +++ b/Backend/src/models/prices/prices.service.ts @@ -31,7 +31,7 @@ export const findAll = async (): Promise => { let priceRows = []; try { conn = await pool.getConnection(); - const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE active_listing = true'); + const rows = await conn.query('SELECT price_id, product_id, v.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE active_listing = true AND v.isActive = true'); for (let row in rows) { if (row !== 'meta') { let price: Price = { @@ -72,7 +72,7 @@ export const find = async (id: number): Promise => { let price: any; try { conn = await pool.getConnection(); - const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE price_id = ? AND active_listing = true', id); + const rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE price_id = ? AND active_listing = true AND v.isActive = true', id); for (let row in rows) { if (row !== 'meta') { price = rows[row]; @@ -99,7 +99,7 @@ export const findByProduct = async (product: number): Promise => { let priceRows = []; try { conn = await pool.getConnection(); - const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND active_listing = true', product); + const rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND active_listing = true AND v.isActive = true', product); for (let row in rows) { if (row !== 'meta') { priceRows.push(rows[row]); @@ -142,16 +142,17 @@ export const findByType = async (product: string, type: string): Promise 'PARTITION BY p.vendor_id ' + 'ORDER BY p.timestamp DESC) AS rk ' + 'FROM prices p ' + - 'WHERE product_id = ? AND vendor_id != 1 AND active_listing = true) ' + + 'LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id ' + + 'WHERE product_id = ? AND p.vendor_id != 1 AND active_listing = true AND v.isActive = true) ' + 'SELECT s.* ' + 'FROM summary s ' + 'WHERE s.rk = 1 '), product); } else if (type === 'lowest') { // Used to get the lowest prices for this product over a period of time - rows = await conn.query('SELECT price_id, product_id, vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id != 1 AND active_listing = true GROUP BY DAY(timestamp) ORDER BY timestamp', product); + rows = await conn.query('SELECT price_id, product_id, p.vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND v.vendor_id != 1 AND active_listing = true AND v.isActive = true GROUP BY DAY(timestamp) ORDER BY timestamp', product); } else { // If no type is given, return all prices for this product - rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id != 1 AND active_listing = true', product); + rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id != 1 AND active_listing = true AND v.isActive = true', product); } for (let row in rows) { @@ -188,13 +189,13 @@ export const findByVendor = async (product: string, vendor: string, type: string let rows = []; if (type === 'newest') { // Used to get the newest price for this product and vendor - rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true ORDER BY timestamp DESC LIMIT 1', [product, vendor]); + rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true ORDER BY timestamp DESC LIMIT 1', [product, vendor]); } else if (type === 'lowest') { // Used to get the lowest prices for this product and vendor in all time - rows = await conn.query('SELECT price_id, product_id, vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true LIMIT 1', [product, vendor]); + rows = await conn.query('SELECT price_id, product_id, p.vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true LIMIT 1', [product, vendor]); } else { // If no type is given, return all prices for this product and vendor - rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true', [product, vendor]); + rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true', [product, vendor]); } for (let row in rows) { @@ -237,7 +238,7 @@ export const getBestDeals = async (amount: number): Promise => { ' ROW_NUMBER() OVER(\n' + ' PARTITION BY p.product_id, p.vendor_id\n' + ' ORDER BY p.timestamp DESC) AS rk\n' + - ' FROM prices p WHERE active_listing = true)\n' + + ' FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE active_listing = true AND v.isActive = true)\n' + 'SELECT s.*\n' + 'FROM summary s\n' + 'WHERE s.rk = 1'); @@ -300,7 +301,6 @@ export const getBestDeals = async (amount: number): Promise => { for (let dealIndex = 0; dealIndex < maxAmt; dealIndex++) { priceRows.push(deals[dealIndex] as Price); } - } catch (err) { console.log(err); throw err; @@ -315,7 +315,7 @@ export const getBestDeals = async (amount: number): Promise => { /** * Fetches and returns the lowest, latest, non-amazon price for each given product - * @param ids the ids of the products + * @param productIds the ids of the products */ export const findListByProducts = async (productIds: [number]): Promise => { let conn; @@ -335,8 +335,8 @@ export const findListByProducts = async (productIds: [number]): Promise ' ROW_NUMBER() OVER(\n' + ' PARTITION BY p.product_id, p.vendor_id\n' + ' ORDER BY p.timestamp DESC) AS rk\n' + - ' FROM prices p' + - ' WHERE p.product_id IN (?)' + + ' FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id ' + + ' WHERE p.product_id IN (?) AND v.isActive = true' + ' AND p.vendor_id != 1 AND active_listing = true)\n' + 'SELECT s.*\n' + 'FROM summary s\n' + @@ -365,7 +365,6 @@ export const findListByProducts = async (productIds: [number]): Promise priceRows.push(pricesForProd[0]); } }); - } catch (err) { throw err; } finally { @@ -393,7 +392,6 @@ export const createPriceEntry = async (user_id: number, vendor_id: number, produ // If there are more / less than 1 affected rows, return false return res.affectedRows === 1; - } catch (err) { throw err; } finally { diff --git a/Backend/src/models/vendors/vendors.router.ts b/Backend/src/models/vendors/vendors.router.ts index 7440f4f..89d02fd 100644 --- a/Backend/src/models/vendors/vendors.router.ts +++ b/Backend/src/models/vendors/vendors.router.ts @@ -109,3 +109,49 @@ vendorsRouter.put('/manage/deactivatelisting', async (req: Request, res: Respons res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); } }); + +// PUT /manage/shop/deactivate/:id +vendorsRouter.put('/manage/shop/deactivate/:id', async (req: Request, res: Response) => { + try { + // Authenticate user + const user_ip = req.connection.remoteAddress ?? ''; + const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip); + + // Get required parameters + const vendor_id = parseInt(req.params.id, 10); + + const success = await VendorService.setShopStatus(user.user_id, vendor_id, false); + + if (success) { + res.sendStatus(200); + } else { + res.sendStatus(500); + } + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); + +// PUT /manage/shop/activate/:id +vendorsRouter.put('/manage/shop/activate/:id', async (req: Request, res: Response) => { + try { + // Authenticate user + const user_ip = req.connection.remoteAddress ?? ''; + const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip); + + // Get required parameters + const vendor_id = parseInt(req.params.id, 10); + + const success = await VendorService.setShopStatus(user.user_id, vendor_id, true); + + if (success) { + res.sendStatus(200); + } else { + res.sendStatus(500); + } + } catch (e) { + console.log('Error handling a request: ' + e.message); + res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'})); + } +}); diff --git a/Backend/src/models/vendors/vendors.service.ts b/Backend/src/models/vendors/vendors.service.ts index 39fee7d..2439093 100644 --- a/Backend/src/models/vendors/vendors.service.ts +++ b/Backend/src/models/vendors/vendors.service.ts @@ -32,7 +32,7 @@ export const findAll = async (): Promise => { let vendorRows = []; try { conn = await pool.getConnection(); - const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors'); + const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE isActive = true'); for (let row in rows) { if (row !== 'meta') { let vendor: Vendor = { @@ -79,7 +79,7 @@ export const find = async (id: number): Promise => { let vendor: any; try { conn = await pool.getConnection(); - const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE vendor_id = ?', id); + const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE vendor_id = ? AND isActive = true', id); for (let row in rows) { if (row !== 'meta') { vendor = rows[row]; @@ -107,7 +107,7 @@ export const findBySearchTerm = async (term: string): Promise => { try { conn = await pool.getConnection(); term = '%' + term + '%'; - const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE name LIKE ?', term); + const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE name LIKE ? AND isActive = true', term); for (let row in rows) { if (row !== 'meta') { vendorRows.push(rows[row]); @@ -182,3 +182,35 @@ export const deactivateListing = async (user_id: number, vendor_id: number, prod return false; }; + +/** + * Set the specified shop to either active or not active + * @param user_id The user id of the issuing user + * @param vendor_id The vendor id of the shop to update + * @param isActive The new active state + */ +export const setShopStatus = async (user_id: number, vendor_id: number, isActive: boolean): Promise => { + let conn; + try { + conn = await pool.getConnection(); + + // Check if the user is authorized to manage the requested vendor + const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]); + if (user_vendor_rows.length !== 1) { + return false; + } + + // Update the vendor state + const status = await conn.query('UPDATE vendors SET isActive = ? WHERE vendor_id = ?', [isActive, vendor_id]); + + return status.affectedRows > 0; + } catch (err) { + throw err; + } finally { + if (conn) { + conn.end(); + } + } + + return false; +}; From 712f6c9034dc2fb245ea9c3645ecea4fced16b31 Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Wed, 19 May 2021 19:46:30 +0200 Subject: [PATCH 6/8] BETTERZON-101: Adding service functions for pricealarms api (#55) - Not properly tested though as login functionality is required to test but not yet implemented --- .../models/pricealarms/pricealarms.router.ts | 4 +-- Frontend/src/app/models/pricealarm.ts | 6 ++++ Frontend/src/app/services/api.service.ts | 34 +++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 Frontend/src/app/models/pricealarm.ts diff --git a/Backend/src/models/pricealarms/pricealarms.router.ts b/Backend/src/models/pricealarms/pricealarms.router.ts index faa3c40..4ba1f9f 100644 --- a/Backend/src/models/pricealarms/pricealarms.router.ts +++ b/Backend/src/models/pricealarms/pricealarms.router.ts @@ -36,7 +36,7 @@ pricealarmsRouter.get('/', async (req: Request, res: Response) => { }); // POST pricealarms/create -pricealarmsRouter.post('/create', async (req: Request, res: Response) => { +pricealarmsRouter.post('/', async (req: Request, res: Response) => { try { // Authenticate user const user_ip = req.connection.remoteAddress ?? ''; @@ -69,7 +69,7 @@ pricealarmsRouter.post('/create', async (req: Request, res: Response) => { }); // PUT pricealarms/update -pricealarmsRouter.put('/update', async (req: Request, res: Response) => { +pricealarmsRouter.put('/', async (req: Request, res: Response) => { try { // Authenticate user const user_ip = req.connection.remoteAddress ?? ''; diff --git a/Frontend/src/app/models/pricealarm.ts b/Frontend/src/app/models/pricealarm.ts new file mode 100644 index 0000000..c8a1717 --- /dev/null +++ b/Frontend/src/app/models/pricealarm.ts @@ -0,0 +1,6 @@ +export interface PriceAlarm { + alarm_id: number; + user_id: number; + product_id: number; + defined_price: number; +} diff --git a/Frontend/src/app/services/api.service.ts b/Frontend/src/app/services/api.service.ts index ec91051..e80e1a8 100644 --- a/Frontend/src/app/services/api.service.ts +++ b/Frontend/src/app/services/api.service.ts @@ -5,6 +5,7 @@ import {Product} from '../models/product'; import {Price} from '../models/price'; import {Observable, of} from 'rxjs'; import {Vendor} from '../models/vendor'; +import {PriceAlarm} from '../models/pricealarm'; @Injectable({ providedIn: 'root' @@ -98,4 +99,37 @@ export class ApiService { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + + getPriceAlarms(): Observable { + try { + const alarms = this.http.get((this.apiUrl + '/pricealarms')); + return alarms; + } catch (exception) { + process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); + } + } + + createPriceAlarms(productId: number, definedPrice: number): Observable { + try { + const res = this.http.post((this.apiUrl + '/pricealarms'), JSON.stringify({ + product_id: productId, + defined_price: definedPrice + })); + return res; + } catch (exception) { + process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); + } + } + + updatePriceAlarms(alarmId: number, definedPrice: number): Observable { + try { + const res = this.http.put((this.apiUrl + '/pricealarms'), JSON.stringify({ + alarm_id: alarmId, + defined_price: definedPrice + })); + return res; + } catch (exception) { + process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); + } + } } From ead1f10b255114ee441b43d1cf9cc0eeaad575c8 Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Thu, 20 May 2021 10:20:50 +0200 Subject: [PATCH 7/8] BETTERZON-110: Refactoring, reformatting and commenting api service (#56) --- Frontend/src/app/services/api.service.ts | 119 ++++++++++++++++++----- 1 file changed, 97 insertions(+), 22 deletions(-) diff --git a/Frontend/src/app/services/api.service.ts b/Frontend/src/app/services/api.service.ts index e80e1a8..640b317 100644 --- a/Frontend/src/app/services/api.service.ts +++ b/Frontend/src/app/services/api.service.ts @@ -18,116 +18,191 @@ export class ApiService { ) { } + + /* ____ __ __ + / __ \_________ ____/ /_ _______/ /______ + / /_/ / ___/ __ \/ __ / / / / ___/ __/ ___/ + / ____/ / / /_/ / /_/ / /_/ / /__/ /_(__ ) + /_/ /_/ \____/\__,_/\__,_/\___/\__/____/ + */ + + /** + * Gets the specified product from the API + * @param id The id of the product to get + * @return Observable An observable containing a single product + */ getProduct(id): Observable { try { - const prod = this.http.get((this.apiUrl + '/products/' + id)); - return prod; + return this.http.get((this.apiUrl + '/products/' + id)); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + + /** + * Gets a list of products that match the given search term + * @param query The search term to match + * @return Observable An observable list of products + */ getProductsByQuery(query): Observable { try { - const prods = this.http.get((this.apiUrl + '/products/search/' + query)); - return prods; + return this.http.get((this.apiUrl + '/products/search/' + query)); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Gets a list of all products + * @return Observable An observable list of products + */ getProducts(): Observable { try { - const prods = this.http.get((this.apiUrl + '/products')); - return prods; + return this.http.get((this.apiUrl + '/products')); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + + /* ____ _ + / __ \_____(_)_______ _____ + / /_/ / ___/ / ___/ _ \/ ___/ + / ____/ / / / /__/ __(__ ) + /_/ /_/ /_/\___/\___/____/ + */ + + /** + * Gets a list of all prices + * @return Observable An observable list of prices + */ getPrices(): Observable { try { - const prices = this.http.get((this.apiUrl + '/prices')); - return prices; + return this.http.get((this.apiUrl + '/prices')); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Gets the lowest prices of every vendor for the given product + * @param productId The product id of the product to fetch the prices for + * @return Observable An observable list of prices + */ getLowestPrices(productId): Observable { try { let params = new HttpParams(); params = params.append('product', productId); params = params.append('type', 'lowest'); - const prices = this.http.get((this.apiUrl + '/prices'), {params}); - return prices; + return this.http.get((this.apiUrl + '/prices'), {params}); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Gets the latest amazon price for the given product + * @param productId The product id of the product to get the price for + * @return Observable An observable containing a single price + */ getAmazonPrice(productId): Observable { try { let params = new HttpParams(); params = params.append('product', productId); params = params.append('vendor', '1'); params = params.append('type', 'newest'); - const price = this.http.get((this.apiUrl + '/prices'), {params}); - return price; + return this.http.get((this.apiUrl + '/prices'), {params}); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Gets the newest prices of every vendor for the given product + * @param productId The product id of the product to fetch the prices for + * @return Observable An observable list of prices + */ getCurrentPricePerVendor(productId): Observable { try { let params = new HttpParams(); params = params.append('product', productId); params = params.append('type', 'newest'); - const prices = this.http.get((this.apiUrl + '/prices'), {params}); - return prices; + return this.http.get((this.apiUrl + '/prices'), {params}); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + + /* _ __ __ + | | / /__ ____ ____/ /___ __________ + | | / / _ \/ __ \/ __ / __ \/ ___/ ___/ + | |/ / __/ / / / /_/ / /_/ / / (__ ) + |___/\___/_/ /_/\__,_/\____/_/ /____/ + */ + + /** + * Gets a list of all vendors + * @return Observable An observable list of vendors + */ getVendors(): Observable { try { - const vendors = this.http.get((this.apiUrl + '/vendors')); - return vendors; + return this.http.get((this.apiUrl + '/vendors')); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + + /* ____ _ ___ __ + / __ \_____(_)_______ / | / /___ __________ ___ _____ + / /_/ / ___/ / ___/ _ \ / /| | / / __ `/ ___/ __ `__ \/ ___/ + / ____/ / / / /__/ __/ / ___ |/ / /_/ / / / / / / / (__ ) + /_/ /_/ /_/\___/\___/ /_/ |_/_/\__,_/_/ /_/ /_/ /_/____/ + */ + + /** + * Gets a list of all price alarms + * @return Observable An observable list of price alarms + */ getPriceAlarms(): Observable { try { - const alarms = this.http.get((this.apiUrl + '/pricealarms')); - return alarms; + return this.http.get((this.apiUrl + '/pricealarms')); } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Creates a new price alarm + * @param productId The product id of the product to create the alarm for + * @param definedPrice The defined target price + * @return Observable The observable response of the api + */ createPriceAlarms(productId: number, definedPrice: number): Observable { try { - const res = this.http.post((this.apiUrl + '/pricealarms'), JSON.stringify({ + return this.http.post((this.apiUrl + '/pricealarms'), JSON.stringify({ product_id: productId, defined_price: definedPrice })); - return res; } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } } + /** + * Updates the given price alarm + * @param alarmId The alarm id of the alarm to update + * @param definedPrice The defined target price + * @return Observable The observable response of the api + */ updatePriceAlarms(alarmId: number, definedPrice: number): Observable { try { - const res = this.http.put((this.apiUrl + '/pricealarms'), JSON.stringify({ + return this.http.put((this.apiUrl + '/pricealarms'), JSON.stringify({ alarm_id: alarmId, defined_price: definedPrice })); - return res; } catch (exception) { process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`); } From e7543e6430065075ed83e332376e2aa5f16ecf51 Mon Sep 17 00:00:00 2001 From: Patrick <50352812+Mueller-Patrick@users.noreply.github.com> Date: Thu, 20 May 2021 10:23:41 +0200 Subject: [PATCH 8/8] BETTERZON-107: Refactoring code with Proxy as design pattern (#49) --- Backend/src/models/prices/price.interface.ts | 23 +++++++++++++++++--- Backend/src/models/prices/prices.service.ts | 6 ++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Backend/src/models/prices/price.interface.ts b/Backend/src/models/prices/price.interface.ts index 956c9d5..702015a 100644 --- a/Backend/src/models/prices/price.interface.ts +++ b/Backend/src/models/prices/price.interface.ts @@ -4,7 +4,24 @@ export interface Price { vendor_id: number; price_in_cents: number; timestamp: Date; - // Only for deals - amazonDifference?: number; - amazonDifferencePercent?: number; +} + +export class Deal implements Price { + price_id: number; + product_id: number; + vendor_id: number; + price_in_cents: number; + timestamp: Date; + amazonDifference: number; + amazonDifferencePercent: number; + + constructor(price_id: number, product_id: number, vendor_id: number, price_in_cents: number, timestamp: Date, amazonDifference: number, amazonDifferencePercent: number) { + this.price_id = price_id; + this.product_id = product_id; + this.vendor_id = vendor_id; + this.price_in_cents = price_in_cents; + this.timestamp = timestamp; + this.amazonDifference = amazonDifference; + this.amazonDifferencePercent = amazonDifferencePercent; + } } diff --git a/Backend/src/models/prices/prices.service.ts b/Backend/src/models/prices/prices.service.ts index acc6793..3175b66 100644 --- a/Backend/src/models/prices/prices.service.ts +++ b/Backend/src/models/prices/prices.service.ts @@ -15,7 +15,7 @@ const pool = mariadb.createPool({ * Data Model Interfaces */ -import {Price} from './price.interface'; +import {Deal, Price} from './price.interface'; import {Prices} from './prices.interface'; @@ -255,7 +255,7 @@ export const getBestDeals = async (amount: number): Promise => { } // Iterate over all prices to find the products with the biggest difference between amazon and other vendor - let deals: Price[] = []; + let deals: Deal[] = []; Object.keys(allPrices).forEach(productId => { if (allPrices[parseInt(productId)]) { @@ -287,7 +287,7 @@ export const getBestDeals = async (amount: number): Promise => { // Push only deals were the amazon price is actually higher if (deal.amazonDifferencePercent > 0) { - deals.push(deal as Price); + deals.push(deal as Deal); } } });