mirror of
https://github.com/Mueller-Patrick/Betterzon.git
synced 2024-12-23 12:15:11 +00:00
Compare commits
9 Commits
493a8faf42
...
0a25edd58f
Author | SHA1 | Date | |
---|---|---|---|
|
0a25edd58f | ||
|
e7543e6430 | ||
|
ead1f10b25 | ||
|
712f6c9034 | ||
|
1581184b57 | ||
|
26ba21156a | ||
|
3ae68b3df3 | ||
|
8f17ae7896 | ||
|
16ed1070c2 |
|
@ -15,6 +15,7 @@ import {errorHandler} from './middleware/error.middleware';
|
|||
import {notFoundHandler} from './middleware/notFound.middleware';
|
||||
import {usersRouter} from './models/users/users.router';
|
||||
import {pricealarmsRouter} from './models/pricealarms/pricealarms.router';
|
||||
import {contactpersonsRouter} from './models/contact_persons/contact_persons.router';
|
||||
|
||||
const cookieParser = require('cookie-parser');
|
||||
|
||||
|
@ -49,6 +50,7 @@ app.use('/prices', pricesRouter);
|
|||
app.use('/users', usersRouter);
|
||||
app.use('/vendors', vendorsRouter);
|
||||
app.use('/pricealarms', pricealarmsRouter);
|
||||
app.use('/contactpersons', contactpersonsRouter);
|
||||
|
||||
app.use(errorHandler);
|
||||
app.use(notFoundHandler);
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
export interface Contact_Person {
|
||||
contact_person_id: number;
|
||||
first_name: string;
|
||||
last_name: string;
|
||||
gender: string;
|
||||
email: string;
|
||||
phone: string;
|
||||
vendor_id: number;
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
import {Contact_Person} from './contact_person.interface';
|
||||
|
||||
export interface Contact_Persons {
|
||||
[key: number]: Contact_Person;
|
||||
}
|
129
Backend/src/models/contact_persons/contact_persons.router.ts
Normal file
129
Backend/src/models/contact_persons/contact_persons.router.ts
Normal file
|
@ -0,0 +1,129 @@
|
|||
/**
|
||||
* Required External Modules and Interfaces
|
||||
*/
|
||||
|
||||
import express, {Request, Response} from 'express';
|
||||
import * as ContactPersonService from './contact_persons.service';
|
||||
import {Contact_Person} from './contact_person.interface';
|
||||
import {Contact_Persons} from './contact_persons.interface';
|
||||
import * as UserService from '../users/users.service';
|
||||
import * as PriceService from '../prices/prices.service';
|
||||
|
||||
|
||||
/**
|
||||
* Router Definition
|
||||
*/
|
||||
|
||||
export const contactpersonsRouter = express.Router();
|
||||
|
||||
|
||||
/**
|
||||
* Controller Definitions
|
||||
*/
|
||||
|
||||
// GET contactpersons/
|
||||
contactpersonsRouter.get('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const contacts: Contact_Persons = await ContactPersonService.findAll();
|
||||
|
||||
res.status(200).send(contacts);
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// GET contactpersons/:id
|
||||
contactpersonsRouter.get('/:id', async (req: Request, res: Response) => {
|
||||
const id: number = parseInt(req.params.id, 10);
|
||||
|
||||
if (!id) {
|
||||
res.status(400).send('Missing parameters.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const contact: Contact_Person = await ContactPersonService.find(id);
|
||||
|
||||
res.status(200).send(contact);
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// GET contactpersons/byvendor/:id
|
||||
contactpersonsRouter.get('/byvendor/:id', async (req: Request, res: Response) => {
|
||||
const id: number = parseInt(req.params.id, 10);
|
||||
|
||||
if (!id) {
|
||||
res.status(400).send('Missing parameters.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const contacts: Contact_Persons = await ContactPersonService.findByVendor(id);
|
||||
|
||||
res.status(200).send(contacts);
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// POST contactpersons/
|
||||
contactpersonsRouter.post('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip);
|
||||
|
||||
// Get required parameters
|
||||
const vendor_id = req.body.vendor_id;
|
||||
const first_name = req.body.first_name;
|
||||
const last_name = req.body.last_name;
|
||||
const gender = req.body.gender;
|
||||
const email = req.body.email;
|
||||
const phone = req.body.phone;
|
||||
|
||||
const success = await ContactPersonService.createContactEntry(user.user_id, vendor_id, first_name, last_name, gender, email, phone);
|
||||
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// PUT contactpersons/:id
|
||||
contactpersonsRouter.put('/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip);
|
||||
|
||||
// Get required parameters
|
||||
const contact_person_id = parseInt(req.params.id, 10);
|
||||
const vendor_id = req.body.vendor_id;
|
||||
const first_name = req.body.first_name;
|
||||
const last_name = req.body.last_name;
|
||||
const gender = req.body.gender;
|
||||
const email = req.body.email;
|
||||
const phone = req.body.phone;
|
||||
|
||||
const success = await ContactPersonService.updateContactEntry(user.user_id, contact_person_id, vendor_id, first_name, last_name, gender, email, phone);
|
||||
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
175
Backend/src/models/contact_persons/contact_persons.service.ts
Normal file
175
Backend/src/models/contact_persons/contact_persons.service.ts
Normal file
|
@ -0,0 +1,175 @@
|
|||
import * as dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const mariadb = require('mariadb');
|
||||
const pool = mariadb.createPool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_DATABASE,
|
||||
connectionLimit: 5
|
||||
});
|
||||
|
||||
/**
|
||||
* Data Model Interfaces
|
||||
*/
|
||||
|
||||
import {Contact_Person} from './contact_person.interface';
|
||||
import {Contact_Persons} from './contact_persons.interface';
|
||||
|
||||
|
||||
/**
|
||||
* Service Methods
|
||||
*/
|
||||
|
||||
/**
|
||||
* Fetches and returns all known contact persons
|
||||
*/
|
||||
export const findAll = async (): Promise<Contact_Persons> => {
|
||||
let conn;
|
||||
let contRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons');
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
contRows.push(rows[row]);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
return contRows;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetches and returns the contact person with the specified id
|
||||
* @param id The id of the contact person to fetch
|
||||
*/
|
||||
export const find = async (id: number): Promise<Contact_Person> => {
|
||||
let conn;
|
||||
let cont: any;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons WHERE contact_person_id = ?', id);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
cont = rows[row];
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
return cont;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetches and returns the contact persons for the specified vendor
|
||||
* @param id The id of the vendor to fetch contact persons for
|
||||
*/
|
||||
export const findByVendor = async (id: number): Promise<Contact_Persons> => {
|
||||
let conn;
|
||||
let contRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT contact_person_id, first_name, last_name, gender, email, phone, vendor_id FROM contact_persons WHERE vendor_id = ?', id);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
contRows.push(rows[row]);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
return contRows;
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a contact entry record
|
||||
* @param user_id The user id of the issuing user
|
||||
* @param vendor_id The vendor id of the vendor to create the record for
|
||||
* @param first_name The first name of the contact person
|
||||
* @param last_name The last name of the contact person
|
||||
* @param gender The gender of the contact person
|
||||
* @param email The email of the contact person
|
||||
* @param phone The phone number of the contact person
|
||||
*/
|
||||
export const createContactEntry = async (user_id: number, vendor_id: number, first_name: string, last_name: string, gender: string, email: string, phone: string): Promise<Boolean> => {
|
||||
let conn;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
|
||||
// Check if the user is authorized to manage the requested vendor
|
||||
const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]);
|
||||
if (user_vendor_rows.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Create contact person entry
|
||||
const res = await conn.query('INSERT INTO contact_persons (first_name, last_name, gender, email, phone, vendor_id) VALUES (?, ?, ?, ?, ?, ?)', [first_name, last_name, gender, email, phone, vendor_id]);
|
||||
|
||||
// If there are more / less than 1 affected rows, return false
|
||||
return res.affectedRows === 1;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Updates a contact entry record
|
||||
* @param user_id The user id of the issuing user
|
||||
* @param contact_person_id The id of the record to update
|
||||
* @param vendor_id The vendor id of the vendor to create the record for
|
||||
* @param first_name The first name of the contact person
|
||||
* @param last_name The last name of the contact person
|
||||
* @param gender The gender of the contact person
|
||||
* @param email The email of the contact person
|
||||
* @param phone The phone number of the contact person
|
||||
*/
|
||||
export const updateContactEntry = async (user_id: number, contact_person_id: number, vendor_id: number, first_name: string, last_name: string, gender: string, email: string, phone: string): Promise<Boolean> => {
|
||||
let conn;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
|
||||
// Check if the user is authorized to manage the requested vendor
|
||||
const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]);
|
||||
if (user_vendor_rows.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Create contact person entry
|
||||
const res = await conn.query('UPDATE contact_persons SET first_name = ?, last_name = ?, gender = ?, email = ?, phone = ? WHERE contact_person_id = ? AND vendor_id = ?', [first_name, last_name, gender, email, phone, contact_person_id, vendor_id]);
|
||||
|
||||
// If there are more / less than 1 affected rows, return false
|
||||
return res.affectedRows === 1;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
};
|
|
@ -19,7 +19,7 @@ export const manufacturersRouter = express.Router();
|
|||
* Controller Definitions
|
||||
*/
|
||||
|
||||
// GET items/
|
||||
// GET manufacturers/
|
||||
manufacturersRouter.get('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const manufacturers: Manufacturers = await ManufacturerService.findAll();
|
||||
|
@ -31,7 +31,7 @@ manufacturersRouter.get('/', async (req: Request, res: Response) => {
|
|||
}
|
||||
});
|
||||
|
||||
// GET items/:id
|
||||
// GET manufacturers/:id
|
||||
manufacturersRouter.get('/:id', async (req: Request, res: Response) => {
|
||||
const id: number = parseInt(req.params.id, 10);
|
||||
|
||||
|
@ -50,7 +50,7 @@ manufacturersRouter.get('/:id', async (req: Request, res: Response) => {
|
|||
}
|
||||
});
|
||||
|
||||
// GET items/:term
|
||||
// GET manufacturers/:term
|
||||
manufacturersRouter.get('/search/:term', async (req: Request, res: Response) => {
|
||||
const term: string = req.params.term;
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ pricealarmsRouter.get('/', async (req: Request, res: Response) => {
|
|||
});
|
||||
|
||||
// POST pricealarms/create
|
||||
pricealarmsRouter.post('/create', async (req: Request, res: Response) => {
|
||||
pricealarmsRouter.post('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
|
@ -69,7 +69,7 @@ pricealarmsRouter.post('/create', async (req: Request, res: Response) => {
|
|||
});
|
||||
|
||||
// PUT pricealarms/update
|
||||
pricealarmsRouter.put('/update', async (req: Request, res: Response) => {
|
||||
pricealarmsRouter.put('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
|
|
|
@ -4,7 +4,24 @@ export interface Price {
|
|||
vendor_id: number;
|
||||
price_in_cents: number;
|
||||
timestamp: Date;
|
||||
// Only for deals
|
||||
amazonDifference?: number;
|
||||
amazonDifferencePercent?: number;
|
||||
}
|
||||
|
||||
export class Deal implements Price {
|
||||
price_id: number;
|
||||
product_id: number;
|
||||
vendor_id: number;
|
||||
price_in_cents: number;
|
||||
timestamp: Date;
|
||||
amazonDifference: number;
|
||||
amazonDifferencePercent: number;
|
||||
|
||||
constructor(price_id: number, product_id: number, vendor_id: number, price_in_cents: number, timestamp: Date, amazonDifference: number, amazonDifferencePercent: number) {
|
||||
this.price_id = price_id;
|
||||
this.product_id = product_id;
|
||||
this.vendor_id = vendor_id;
|
||||
this.price_in_cents = price_in_cents;
|
||||
this.timestamp = timestamp;
|
||||
this.amazonDifference = amazonDifference;
|
||||
this.amazonDifferencePercent = amazonDifferencePercent;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,6 +6,7 @@ import express, {Request, Response} from 'express';
|
|||
import * as PriceService from './prices.service';
|
||||
import {Price} from './price.interface';
|
||||
import {Prices} from './prices.interface';
|
||||
import * as UserService from '../users/users.service';
|
||||
|
||||
|
||||
/**
|
||||
|
@ -100,3 +101,28 @@ pricesRouter.get('/byProduct/list/:ids', async (req: Request, res: Response) =>
|
|||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// POST prices/
|
||||
pricesRouter.post('/', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip);
|
||||
|
||||
// Get required parameters
|
||||
const vendor_id = req.body.vendor_id;
|
||||
const product_id = req.body.product_id;
|
||||
const price_in_cents = req.body.price_in_cents;
|
||||
|
||||
const success = await PriceService.createPriceEntry(user.user_id, vendor_id, product_id, price_in_cents);
|
||||
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
|
|
@ -15,7 +15,7 @@ const pool = mariadb.createPool({
|
|||
* Data Model Interfaces
|
||||
*/
|
||||
|
||||
import {Price} from './price.interface';
|
||||
import {Deal, Price} from './price.interface';
|
||||
import {Prices} from './prices.interface';
|
||||
|
||||
|
||||
|
@ -31,7 +31,7 @@ export const findAll = async (): Promise<Prices> => {
|
|||
let priceRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE active_listing = true');
|
||||
const rows = await conn.query('SELECT price_id, product_id, v.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE active_listing = true AND v.isActive = true');
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
let price: Price = {
|
||||
|
@ -72,7 +72,7 @@ export const find = async (id: number): Promise<Price> => {
|
|||
let price: any;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE price_id = ? AND active_listing = true', id);
|
||||
const rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE price_id = ? AND active_listing = true AND v.isActive = true', id);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
price = rows[row];
|
||||
|
@ -99,7 +99,7 @@ export const findByProduct = async (product: number): Promise<Prices> => {
|
|||
let priceRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND active_listing = true', product);
|
||||
const rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND active_listing = true AND v.isActive = true', product);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
priceRows.push(rows[row]);
|
||||
|
@ -142,16 +142,17 @@ export const findByType = async (product: string, type: string): Promise<Prices>
|
|||
'PARTITION BY p.vendor_id ' +
|
||||
'ORDER BY p.timestamp DESC) AS rk ' +
|
||||
'FROM prices p ' +
|
||||
'WHERE product_id = ? AND vendor_id != 1 AND active_listing = true) ' +
|
||||
'LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id ' +
|
||||
'WHERE product_id = ? AND p.vendor_id != 1 AND active_listing = true AND v.isActive = true) ' +
|
||||
'SELECT s.* ' +
|
||||
'FROM summary s ' +
|
||||
'WHERE s.rk = 1 '), product);
|
||||
} else if (type === 'lowest') {
|
||||
// Used to get the lowest prices for this product over a period of time
|
||||
rows = await conn.query('SELECT price_id, product_id, vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id != 1 AND active_listing = true GROUP BY DAY(timestamp) ORDER BY timestamp', product);
|
||||
rows = await conn.query('SELECT price_id, product_id, p.vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND v.vendor_id != 1 AND active_listing = true AND v.isActive = true GROUP BY DAY(timestamp) ORDER BY timestamp', product);
|
||||
} else {
|
||||
// If no type is given, return all prices for this product
|
||||
rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id != 1 AND active_listing = true', product);
|
||||
rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id != 1 AND active_listing = true AND v.isActive = true', product);
|
||||
}
|
||||
|
||||
for (let row in rows) {
|
||||
|
@ -188,13 +189,13 @@ export const findByVendor = async (product: string, vendor: string, type: string
|
|||
let rows = [];
|
||||
if (type === 'newest') {
|
||||
// Used to get the newest price for this product and vendor
|
||||
rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true ORDER BY timestamp DESC LIMIT 1', [product, vendor]);
|
||||
rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true ORDER BY timestamp DESC LIMIT 1', [product, vendor]);
|
||||
} else if (type === 'lowest') {
|
||||
// Used to get the lowest prices for this product and vendor in all time
|
||||
rows = await conn.query('SELECT price_id, product_id, vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true LIMIT 1', [product, vendor]);
|
||||
rows = await conn.query('SELECT price_id, product_id, p.vendor_id, MIN(price_in_cents) as price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true LIMIT 1', [product, vendor]);
|
||||
} else {
|
||||
// If no type is given, return all prices for this product and vendor
|
||||
rows = await conn.query('SELECT price_id, product_id, vendor_id, price_in_cents, timestamp FROM prices WHERE product_id = ? AND vendor_id = ? AND active_listing = true', [product, vendor]);
|
||||
rows = await conn.query('SELECT price_id, product_id, p.vendor_id, price_in_cents, timestamp FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE product_id = ? AND p.vendor_id = ? AND active_listing = true AND v.isActive = true', [product, vendor]);
|
||||
}
|
||||
|
||||
for (let row in rows) {
|
||||
|
@ -237,7 +238,7 @@ export const getBestDeals = async (amount: number): Promise<Prices> => {
|
|||
' ROW_NUMBER() OVER(\n' +
|
||||
' PARTITION BY p.product_id, p.vendor_id\n' +
|
||||
' ORDER BY p.timestamp DESC) AS rk\n' +
|
||||
' FROM prices p WHERE active_listing = true)\n' +
|
||||
' FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id WHERE active_listing = true AND v.isActive = true)\n' +
|
||||
'SELECT s.*\n' +
|
||||
'FROM summary s\n' +
|
||||
'WHERE s.rk = 1');
|
||||
|
@ -254,7 +255,7 @@ export const getBestDeals = async (amount: number): Promise<Prices> => {
|
|||
}
|
||||
|
||||
// Iterate over all prices to find the products with the biggest difference between amazon and other vendor
|
||||
let deals: Price[] = [];
|
||||
let deals: Deal[] = [];
|
||||
|
||||
Object.keys(allPrices).forEach(productId => {
|
||||
if (allPrices[parseInt(productId)]) {
|
||||
|
@ -286,7 +287,7 @@ export const getBestDeals = async (amount: number): Promise<Prices> => {
|
|||
|
||||
// Push only deals were the amazon price is actually higher
|
||||
if (deal.amazonDifferencePercent > 0) {
|
||||
deals.push(deal as Price);
|
||||
deals.push(deal as Deal);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
@ -300,7 +301,6 @@ export const getBestDeals = async (amount: number): Promise<Prices> => {
|
|||
for (let dealIndex = 0; dealIndex < maxAmt; dealIndex++) {
|
||||
priceRows.push(deals[dealIndex] as Price);
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
throw err;
|
||||
|
@ -315,7 +315,7 @@ export const getBestDeals = async (amount: number): Promise<Prices> => {
|
|||
|
||||
/**
|
||||
* Fetches and returns the lowest, latest, non-amazon price for each given product
|
||||
* @param ids the ids of the products
|
||||
* @param productIds the ids of the products
|
||||
*/
|
||||
export const findListByProducts = async (productIds: [number]): Promise<Prices> => {
|
||||
let conn;
|
||||
|
@ -335,8 +335,8 @@ export const findListByProducts = async (productIds: [number]): Promise<Prices>
|
|||
' ROW_NUMBER() OVER(\n' +
|
||||
' PARTITION BY p.product_id, p.vendor_id\n' +
|
||||
' ORDER BY p.timestamp DESC) AS rk\n' +
|
||||
' FROM prices p' +
|
||||
' WHERE p.product_id IN (?)' +
|
||||
' FROM prices p LEFT OUTER JOIN vendors v ON v.vendor_id = p.vendor_id ' +
|
||||
' WHERE p.product_id IN (?) AND v.isActive = true' +
|
||||
' AND p.vendor_id != 1 AND active_listing = true)\n' +
|
||||
'SELECT s.*\n' +
|
||||
'FROM summary s\n' +
|
||||
|
@ -365,7 +365,6 @@ export const findListByProducts = async (productIds: [number]): Promise<Prices>
|
|||
priceRows.push(pricesForProd[0]);
|
||||
}
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
|
@ -376,3 +375,28 @@ export const findListByProducts = async (productIds: [number]): Promise<Prices>
|
|||
|
||||
return priceRows;
|
||||
};
|
||||
|
||||
export const createPriceEntry = async (user_id: number, vendor_id: number, product_id: number, price_in_cents: number): Promise<Boolean> => {
|
||||
let conn;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
|
||||
// Check if the user is authorized to manage the requested vendor
|
||||
const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]);
|
||||
if (user_vendor_rows.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Create price entry
|
||||
const res = await conn.query('INSERT INTO prices (product_id, vendor_id, price_in_cents) VALUES (?,?,?)', [product_id, vendor_id, price_in_cents]);
|
||||
|
||||
// If there are more / less than 1 affected rows, return false
|
||||
return res.affectedRows === 1;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -87,3 +87,22 @@ productsRouter.get('/list/:ids', async (req: Request, res: Response) => {
|
|||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// GET products/vendor/:id
|
||||
productsRouter.get('/vendor/:id', async (req: Request, res: Response) => {
|
||||
const id: number = parseInt(req.params.id, 10);
|
||||
|
||||
if (!id) {
|
||||
res.status(400).send('Missing parameters.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const products: Products = await ProductService.findByVendor(id);
|
||||
|
||||
res.status(200).send(products);
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
|
|
@ -159,3 +159,41 @@ export const findList = async (ids: [number]): Promise<Products> => {
|
|||
|
||||
return prodRows;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetches and returns the products that the given vendor has price entries for
|
||||
* @param id The id of the vendor to fetch the products for
|
||||
*/
|
||||
export const findByVendor = async (id: number): Promise<Products> => {
|
||||
let conn;
|
||||
let prodRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
|
||||
// Get the relevant product ids
|
||||
let relevant_prod_ids = [];
|
||||
const relevantProds = await conn.query('SELECT product_id FROM prices WHERE vendor_id = ? GROUP BY product_id', id);
|
||||
for (let row in relevantProds) {
|
||||
if (row !== 'meta') {
|
||||
relevant_prod_ids.push(relevantProds[row].product_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch products
|
||||
const rows = await conn.query('SELECT product_id, name, asin, is_active, short_description, long_description, image_guid, date_added, last_modified, manufacturer_id, selling_rank, category_id FROM products WHERE product_id IN (?)', [relevant_prod_ids]);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
prodRows.push(rows[row]);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
return prodRows;
|
||||
};
|
||||
|
|
48
Backend/src/models/vendors/vendors.router.ts
vendored
48
Backend/src/models/vendors/vendors.router.ts
vendored
|
@ -99,7 +99,53 @@ vendorsRouter.put('/manage/deactivatelisting', async (req: Request, res: Respons
|
|||
|
||||
const success = await VendorService.deactivateListing(user.user_id, vendor_id, product_id);
|
||||
|
||||
if(success) {
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// PUT /manage/shop/deactivate/:id
|
||||
vendorsRouter.put('/manage/shop/deactivate/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip);
|
||||
|
||||
// Get required parameters
|
||||
const vendor_id = parseInt(req.params.id, 10);
|
||||
|
||||
const success = await VendorService.setShopStatus(user.user_id, vendor_id, false);
|
||||
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Error handling a request: ' + e.message);
|
||||
res.status(500).send(JSON.stringify({'message': 'Internal Server Error. Try again later.'}));
|
||||
}
|
||||
});
|
||||
|
||||
// PUT /manage/shop/activate/:id
|
||||
vendorsRouter.put('/manage/shop/activate/:id', async (req: Request, res: Response) => {
|
||||
try {
|
||||
// Authenticate user
|
||||
const user_ip = req.connection.remoteAddress ?? '';
|
||||
const user = await UserService.checkSessionWithCookie(req.cookies.betterauth, user_ip);
|
||||
|
||||
// Get required parameters
|
||||
const vendor_id = parseInt(req.params.id, 10);
|
||||
|
||||
const success = await VendorService.setShopStatus(user.user_id, vendor_id, true);
|
||||
|
||||
if (success) {
|
||||
res.sendStatus(200);
|
||||
} else {
|
||||
res.sendStatus(500);
|
||||
|
|
43
Backend/src/models/vendors/vendors.service.ts
vendored
43
Backend/src/models/vendors/vendors.service.ts
vendored
|
@ -32,7 +32,7 @@ export const findAll = async (): Promise<Vendors> => {
|
|||
let vendorRows = [];
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors');
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE isActive = true');
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
let vendor: Vendor = {
|
||||
|
@ -79,7 +79,7 @@ export const find = async (id: number): Promise<Vendor> => {
|
|||
let vendor: any;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE vendor_id = ?', id);
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE vendor_id = ? AND isActive = true', id);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
vendor = rows[row];
|
||||
|
@ -107,7 +107,7 @@ export const findBySearchTerm = async (term: string): Promise<Vendors> => {
|
|||
try {
|
||||
conn = await pool.getConnection();
|
||||
term = '%' + term + '%';
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE name LIKE ?', term);
|
||||
const rows = await conn.query('SELECT vendor_id, name, streetname, zip_code, city, country_code, phone, website FROM vendors WHERE name LIKE ? AND isActive = true', term);
|
||||
for (let row in rows) {
|
||||
if (row !== 'meta') {
|
||||
vendorRows.push(rows[row]);
|
||||
|
@ -171,10 +171,39 @@ export const deactivateListing = async (user_id: number, vendor_id: number, prod
|
|||
|
||||
const status = await conn.query('UPDATE prices SET active_listing = false WHERE vendor_id = ? and product_id = ?', [vendor_id, product_id]);
|
||||
|
||||
if(status.affectedRows > 0){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return status.affectedRows > 0;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
if (conn) {
|
||||
conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
/**
|
||||
* Set the specified shop to either active or not active
|
||||
* @param user_id The user id of the issuing user
|
||||
* @param vendor_id The vendor id of the shop to update
|
||||
* @param isActive The new active state
|
||||
*/
|
||||
export const setShopStatus = async (user_id: number, vendor_id: number, isActive: boolean): Promise<Boolean> => {
|
||||
let conn;
|
||||
try {
|
||||
conn = await pool.getConnection();
|
||||
|
||||
// Check if the user is authorized to manage the requested vendor
|
||||
const user_vendor_rows = await conn.query('SELECT vendor_id FROM vendors WHERE vendor_id = ? AND admin_id = ?', [vendor_id, user_id]);
|
||||
if (user_vendor_rows.length !== 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Update the vendor state
|
||||
const status = await conn.query('UPDATE vendors SET isActive = ? WHERE vendor_id = ?', [isActive, vendor_id]);
|
||||
|
||||
return status.affectedRows > 0;
|
||||
} catch (err) {
|
||||
throw err;
|
||||
} finally {
|
||||
|
|
|
@ -2,13 +2,13 @@
|
|||
<module type="WEB_MODULE" version="4">
|
||||
<component name="FacetManager">
|
||||
<facet type="Python" name="Python">
|
||||
<configuration sdkName="Python 3.9" />
|
||||
<configuration sdkName="Python 3.9 (venv)" />
|
||||
</facet>
|
||||
</component>
|
||||
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
||||
<exclude-output />
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
<orderEntry type="library" name="Python 3.9 interpreter library" level="application" />
|
||||
<orderEntry type="library" name="Python 3.9 (venv) interpreter library" level="application" />
|
||||
</component>
|
||||
</module>
|
|
@ -1,13 +1,17 @@
|
|||
import os
|
||||
|
||||
from flask import Flask
|
||||
from flask_restful import Resource, Api, reqparse
|
||||
|
||||
import crawler
|
||||
|
||||
app = Flask(__name__)
|
||||
api = Api(app)
|
||||
|
||||
# To parse request data
|
||||
parser = reqparse.RequestParser()
|
||||
parser.add_argument('key')
|
||||
parser.add_argument('products')
|
||||
parser.add_argument('key', type=str)
|
||||
parser.add_argument('products', type=int, action='append')
|
||||
|
||||
|
||||
class CrawlerApi(Resource):
|
||||
|
@ -17,7 +21,12 @@ class CrawlerApi(Resource):
|
|||
def post(self):
|
||||
# Accept crawler request here
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
access_key = os.getenv('CRAWLER_ACCESS_KEY')
|
||||
if(args['key'] == access_key):
|
||||
crawler.crawl(args['products'])
|
||||
return {'message': 'success'}
|
||||
else:
|
||||
return {'message': 'Wrong access key'}
|
||||
|
||||
|
||||
api.add_resource(CrawlerApi, '/')
|
||||
|
|
|
@ -1,78 +1,107 @@
|
|||
import sql
|
||||
|
||||
|
||||
def crawl(product_ids: [int]) -> dict:
|
||||
"""
|
||||
Crawls the given list of products and saves the results to sql
|
||||
:param products: The list of product IDs to fetch
|
||||
:return: A dict with the following fields:
|
||||
total_crawls: number of total crawl tries (products * vendors per product)
|
||||
successful_crawls: number of successful products
|
||||
products_with_problems: list of products that have not been crawled successfully
|
||||
"""
|
||||
total_crawls = 0
|
||||
successful_crawls = 0
|
||||
products_with_problems = []
|
||||
|
||||
# Iterate over every product that has to be crawled
|
||||
for product_id in product_ids:
|
||||
# Get all links for this product
|
||||
product_links = sql.getProductLinksForProduct(product_id)
|
||||
|
||||
crawled_data = []
|
||||
|
||||
# Iterate over every link / vendor
|
||||
for product_vendor_info in product_links:
|
||||
total_crawls += 1
|
||||
|
||||
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
||||
if product_vendor_info['vendor_id'] == 1:
|
||||
# Amazon
|
||||
crawled_data.append(__crawl_amazon__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 2:
|
||||
# Apple
|
||||
crawled_data.append(__crawl_apple__(product_vendor_info))
|
||||
elif product_vendor_info['vendor_id'] == 3:
|
||||
# Media Markt
|
||||
crawled_data.append(__crawl_mediamarkt__(product_vendor_info))
|
||||
else:
|
||||
products_with_problems.append(product_vendor_info)
|
||||
continue
|
||||
|
||||
successful_crawls += 1
|
||||
|
||||
# Insert data to SQL
|
||||
sql.insertData(crawled_data)
|
||||
|
||||
return {
|
||||
'total_crawls': total_crawls,
|
||||
'successful_crawls': successful_crawls,
|
||||
'products_with_problems': products_with_problems
|
||||
}
|
||||
|
||||
|
||||
def __crawl_amazon__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from amazon
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
|
||||
|
||||
def __crawl_apple__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from apple
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
|
||||
|
||||
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from media markt
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
pass
|
||||
import sql
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
HEADERS = ({'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 '
|
||||
'Safari/537.36'})
|
||||
|
||||
|
||||
def crawl(product_ids: [int]) -> dict:
|
||||
"""
|
||||
Crawls the given list of products and saves the results to sql
|
||||
:param products: The list of product IDs to fetch
|
||||
:return: A dict with the following fields:
|
||||
total_crawls: number of total crawl tries (products * vendors per product)
|
||||
successful_crawls: number of successful products
|
||||
products_with_problems: list of products that have not been crawled successfully
|
||||
"""
|
||||
total_crawls = 0
|
||||
successful_crawls = 0
|
||||
products_with_problems = []
|
||||
|
||||
# Iterate over every product that has to be crawled
|
||||
for product_id in product_ids:
|
||||
# Get all links for this product
|
||||
product_links = sql.getProductLinksForProduct(product_id)
|
||||
|
||||
crawled_data = []
|
||||
|
||||
# Iterate over every link / vendor
|
||||
for product_vendor_info in product_links:
|
||||
total_crawls += 1
|
||||
|
||||
# Call the appropriate vendor crawling function and append the result to the list of crawled data
|
||||
if product_vendor_info['vendor_id'] == 1:
|
||||
# Amazon
|
||||
data = __crawl_amazon__(product_vendor_info)
|
||||
if data:
|
||||
crawled_data.append(data)
|
||||
elif product_vendor_info['vendor_id'] == 2:
|
||||
# Apple
|
||||
data = __crawl_apple__(product_vendor_info)
|
||||
if data:
|
||||
crawled_data.append(data)
|
||||
elif product_vendor_info['vendor_id'] == 3:
|
||||
# Media Markt
|
||||
data = __crawl_mediamarkt__(product_vendor_info)
|
||||
if data:
|
||||
crawled_data.append(data)
|
||||
else:
|
||||
products_with_problems.append(product_vendor_info)
|
||||
continue
|
||||
|
||||
successful_crawls += 1
|
||||
|
||||
# Insert data to SQL
|
||||
sql.insertData(crawled_data)
|
||||
|
||||
return {
|
||||
'total_crawls': total_crawls,
|
||||
'successful_crawls': successful_crawls,
|
||||
'products_with_problems': products_with_problems
|
||||
}
|
||||
|
||||
|
||||
def __crawl_amazon__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from amazon
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
page = requests.get(product_info['url'], headers=HEADERS)
|
||||
soup = BeautifulSoup(page.content, features="lxml")
|
||||
try:
|
||||
price = int(
|
||||
soup.find(id='priceblock_ourprice').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
||||
if not price:
|
||||
price = int(soup.find(id='price_inside_buybox').get_text().replace(".", "").replace(",", "").replace("€", "").strip())
|
||||
|
||||
except RuntimeError:
|
||||
price = -1
|
||||
except AttributeError:
|
||||
price = -1
|
||||
|
||||
if price != -1:
|
||||
return (product_info['product_id'], product_info['vendor_id'], price)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def __crawl_apple__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from apple
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
# return (product_info['product_id'], product_info['vendor_id'], 123)
|
||||
pass
|
||||
|
||||
|
||||
def __crawl_mediamarkt__(product_info: dict) -> tuple:
|
||||
"""
|
||||
Crawls the price for the given product from media markt
|
||||
:param product_info: A dict with product info containing product_id, vendor_id, url
|
||||
:return: A tuple with the crawled data, containing (product_id, vendor_id, price_in_cents)
|
||||
"""
|
||||
pass
|
||||
|
|
|
@ -1,66 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import scrapy
|
||||
from urllib.parse import urlencode
|
||||
from urllib.parse import urljoin
|
||||
import re
|
||||
import json
|
||||
|
||||
queries = ['iphone']
|
||||
API = ''
|
||||
|
||||
|
||||
def get_url(url):
|
||||
payload = {'api_key': API, 'url': url, 'country_code': 'us'}
|
||||
proxy_url = 'http://api.scraperapi.com/?' + urlencode(payload)
|
||||
return proxy_url
|
||||
|
||||
|
||||
class AmazonSpider(scrapy.Spider):
|
||||
name = 'amazon'
|
||||
|
||||
def start_requests(self):
|
||||
for query in queries:
|
||||
url = 'https://www.amazon.de/s?' + urlencode({'k': query})
|
||||
yield scrapy.Request(url=url, callback=self.parse_keyword_response)
|
||||
|
||||
def parse_keyword_response(self, response):
|
||||
products = response.xpath('//*[@data-asin]')
|
||||
|
||||
for product in products:
|
||||
asin = product.xpath('@data-asin').extract_first()
|
||||
product_url = f"https://www.amazon.de/dp/{asin}"
|
||||
yield scrapy.Request(url=product_url, callback=self.parse_product_page, meta={'asin': asin})
|
||||
|
||||
next_page = response.xpath('//li[@class="a-last"]/a/@href').extract_first()
|
||||
if next_page:
|
||||
url = urljoin("https://www.amazon.de", next_page)
|
||||
yield scrapy.Request(url=url, callback=self.parse_keyword_response)
|
||||
|
||||
def parse_product_page(self, response):
|
||||
asin = response.meta['asin']
|
||||
title = response.xpath('//*[@id="productTitle"]/text()').extract_first()
|
||||
image = re.search('"large":"(.*?)"', response.text).groups()[0]
|
||||
rating = response.xpath('//*[@id="acrPopover"]/@title').extract_first()
|
||||
number_of_reviews = response.xpath('//*[@id="acrCustomerReviewText"]/text()').extract_first()
|
||||
price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first()
|
||||
|
||||
if not price:
|
||||
price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \
|
||||
response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first()
|
||||
|
||||
temp = response.xpath('//*[@id="twister"]')
|
||||
sizes = []
|
||||
colors = []
|
||||
if temp:
|
||||
s = re.search('"variationValues" : ({.*})', response.text).groups()[0]
|
||||
json_acceptable = s.replace("'", "\"")
|
||||
di = json.loads(json_acceptable)
|
||||
sizes = di.get('size_name', [])
|
||||
colors = di.get('color_name', [])
|
||||
|
||||
bullet_points = response.xpath('//*[@id="feature-bullets"]//li/span/text()').extract()
|
||||
seller_rank = response.xpath(
|
||||
'//*[text()="Amazon Best Sellers Rank:"]/parent::*//text()[not(parent::style)]').extract()
|
||||
yield {'asin': asin, 'Title': title, 'MainImage': image, 'Rating': rating, 'NumberOfReviews': number_of_reviews,
|
||||
'Price': price, 'AvailableSizes': sizes, 'AvailableColors': colors, 'BulletPoints': bullet_points,
|
||||
'SellerRank': seller_rank}
|
|
@ -1,5 +1,7 @@
|
|||
pymysql
|
||||
flask
|
||||
flask==1.1.2
|
||||
flask-sqlalchemy
|
||||
flask_restful
|
||||
scrapy
|
||||
beautifulsoup4
|
||||
requests
|
||||
lxml
|
|
@ -54,7 +54,6 @@ def getProductLinksForProduct(product_id: int) -> [dict]:
|
|||
cur = conn.cursor()
|
||||
|
||||
query = 'SELECT vendor_id, url FROM product_links WHERE product_id = %s'
|
||||
|
||||
cur.execute(query, (product_id,))
|
||||
|
||||
products = list(map(lambda x: {'product_id': product_id, 'vendor_id': x[0], 'url': x[1]}, cur.fetchall()))
|
||||
|
|
33
Crawler/unused/scrapy/amazonspider.py
Normal file
33
Crawler/unused/scrapy/amazonspider.py
Normal file
|
@ -0,0 +1,33 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
import re
|
||||
|
||||
class AmazonSpider(scrapy.Spider):
|
||||
name = 'amazon'
|
||||
allowed_domains = ['amazon.de']
|
||||
start_urls = ['https://amazon.de/dp/B083DRCPJG']
|
||||
|
||||
# def __init__(self, start_urls):
|
||||
# self.start_urls = start_urls
|
||||
|
||||
def parse(self, response):
|
||||
price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first()
|
||||
if not price:
|
||||
price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \
|
||||
response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first()
|
||||
|
||||
euros = re.match('(\d*),\d\d', price).group(1)
|
||||
cents = re.match('\d*,(\d\d)', price).group(1)
|
||||
priceincents = euros + cents
|
||||
|
||||
yield {'price': priceincents}
|
||||
|
||||
|
||||
def start_crawling():
|
||||
process = CrawlerProcess(
|
||||
settings={'COOKIES_ENABLED': 'False', 'CONCURRENT_REQUESTS_PER_IP': 1, 'ROBOTSTXT_OBEY': False,
|
||||
'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
|
||||
'DOWNLOAD_DELAY': 3}
|
||||
, install_root_handler=False)
|
||||
process.crawl()
|
||||
process.start()
|
|
@ -8,4 +8,4 @@ default = crawler.settings
|
|||
|
||||
[deploy]
|
||||
#url = http://localhost:6800/
|
||||
project = crawler
|
||||
project = crawler
|
25
Crawler/unused/scrapy/spiders/amazon.py
Normal file
25
Crawler/unused/scrapy/spiders/amazon.py
Normal file
|
@ -0,0 +1,25 @@
|
|||
import scrapy
|
||||
import re
|
||||
|
||||
class AmazonSpider(scrapy.Spider):
|
||||
name = 'amazon'
|
||||
allowed_domains = ['amazon.de']
|
||||
start_urls = ['https://amazon.de/dp/B083DRCPJG']
|
||||
|
||||
def parse(self, response):
|
||||
price = response.xpath('//*[@id="priceblock_ourprice"]/text()').extract_first()
|
||||
if not price:
|
||||
price = response.xpath('//*[@data-asin-price]/@data-asin-price').extract_first() or \
|
||||
response.xpath('//*[@id="price_inside_buybox"]/text()').extract_first()
|
||||
|
||||
euros = re.match('(\d*),\d\d', price).group(1)
|
||||
cents = re.match('\d*,(\d\d)', price).group(1)
|
||||
priceincents = euros + cents
|
||||
|
||||
yield {'price': priceincents}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
6
Frontend/src/app/models/pricealarm.ts
Normal file
6
Frontend/src/app/models/pricealarm.ts
Normal file
|
@ -0,0 +1,6 @@
|
|||
export interface PriceAlarm {
|
||||
alarm_id: number;
|
||||
user_id: number;
|
||||
product_id: number;
|
||||
defined_price: number;
|
||||
}
|
|
@ -5,6 +5,7 @@ import {Product} from '../models/product';
|
|||
import {Price} from '../models/price';
|
||||
import {Observable, of} from 'rxjs';
|
||||
import {Vendor} from '../models/vendor';
|
||||
import {PriceAlarm} from '../models/pricealarm';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
|
@ -17,83 +18,191 @@ export class ApiService {
|
|||
) {
|
||||
}
|
||||
|
||||
|
||||
/* ____ __ __
|
||||
/ __ \_________ ____/ /_ _______/ /______
|
||||
/ /_/ / ___/ __ \/ __ / / / / ___/ __/ ___/
|
||||
/ ____/ / / /_/ / /_/ / /_/ / /__/ /_(__ )
|
||||
/_/ /_/ \____/\__,_/\__,_/\___/\__/____/
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gets the specified product from the API
|
||||
* @param id The id of the product to get
|
||||
* @return Observable<Product> An observable containing a single product
|
||||
*/
|
||||
getProduct(id): Observable<Product> {
|
||||
try {
|
||||
const prod = this.http.get<Product>((this.apiUrl + '/products/' + id));
|
||||
return prod;
|
||||
return this.http.get<Product>((this.apiUrl + '/products/' + id));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Gets a list of products that match the given search term
|
||||
* @param query The search term to match
|
||||
* @return Observable<Product[]> An observable list of products
|
||||
*/
|
||||
getProductsByQuery(query): Observable<Product[]> {
|
||||
try {
|
||||
const prods = this.http.get<Product[]>((this.apiUrl + '/products/search/' + query));
|
||||
return prods;
|
||||
return this.http.get<Product[]>((this.apiUrl + '/products/search/' + query));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a list of all products
|
||||
* @return Observable<Product[]> An observable list of products
|
||||
*/
|
||||
getProducts(): Observable<Product[]> {
|
||||
try {
|
||||
const prods = this.http.get<Product[]>((this.apiUrl + '/products'));
|
||||
return prods;
|
||||
return this.http.get<Product[]>((this.apiUrl + '/products'));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ____ _
|
||||
/ __ \_____(_)_______ _____
|
||||
/ /_/ / ___/ / ___/ _ \/ ___/
|
||||
/ ____/ / / / /__/ __(__ )
|
||||
/_/ /_/ /_/\___/\___/____/
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gets a list of all prices
|
||||
* @return Observable<Price[]> An observable list of prices
|
||||
*/
|
||||
getPrices(): Observable<Price[]> {
|
||||
try {
|
||||
const prices = this.http.get<Price[]>((this.apiUrl + '/prices'));
|
||||
return prices;
|
||||
return this.http.get<Price[]>((this.apiUrl + '/prices'));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the lowest prices of every vendor for the given product
|
||||
* @param productId The product id of the product to fetch the prices for
|
||||
* @return Observable<Price[]> An observable list of prices
|
||||
*/
|
||||
getLowestPrices(productId): Observable<Price[]> {
|
||||
try {
|
||||
let params = new HttpParams();
|
||||
params = params.append('product', productId);
|
||||
params = params.append('type', 'lowest');
|
||||
const prices = this.http.get<Price[]>((this.apiUrl + '/prices'), {params});
|
||||
return prices;
|
||||
return this.http.get<Price[]>((this.apiUrl + '/prices'), {params});
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the latest amazon price for the given product
|
||||
* @param productId The product id of the product to get the price for
|
||||
* @return Observable<Price> An observable containing a single price
|
||||
*/
|
||||
getAmazonPrice(productId): Observable<Price> {
|
||||
try {
|
||||
let params = new HttpParams();
|
||||
params = params.append('product', productId);
|
||||
params = params.append('vendor', '1');
|
||||
params = params.append('type', 'newest');
|
||||
const price = this.http.get<Price>((this.apiUrl + '/prices'), {params});
|
||||
return price;
|
||||
return this.http.get<Price>((this.apiUrl + '/prices'), {params});
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the newest prices of every vendor for the given product
|
||||
* @param productId The product id of the product to fetch the prices for
|
||||
* @return Observable<Price[]> An observable list of prices
|
||||
*/
|
||||
getCurrentPricePerVendor(productId): Observable<Price[]> {
|
||||
try {
|
||||
let params = new HttpParams();
|
||||
params = params.append('product', productId);
|
||||
params = params.append('type', 'newest');
|
||||
const prices = this.http.get<Price[]>((this.apiUrl + '/prices'), {params});
|
||||
return prices;
|
||||
return this.http.get<Price[]>((this.apiUrl + '/prices'), {params});
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* _ __ __
|
||||
| | / /__ ____ ____/ /___ __________
|
||||
| | / / _ \/ __ \/ __ / __ \/ ___/ ___/
|
||||
| |/ / __/ / / / /_/ / /_/ / / (__ )
|
||||
|___/\___/_/ /_/\__,_/\____/_/ /____/
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gets a list of all vendors
|
||||
* @return Observable<Vendor[]> An observable list of vendors
|
||||
*/
|
||||
getVendors(): Observable<Vendor[]> {
|
||||
try {
|
||||
const vendors = this.http.get<Vendor[]>((this.apiUrl + '/vendors'));
|
||||
return vendors;
|
||||
return this.http.get<Vendor[]>((this.apiUrl + '/vendors'));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/* ____ _ ___ __
|
||||
/ __ \_____(_)_______ / | / /___ __________ ___ _____
|
||||
/ /_/ / ___/ / ___/ _ \ / /| | / / __ `/ ___/ __ `__ \/ ___/
|
||||
/ ____/ / / / /__/ __/ / ___ |/ / /_/ / / / / / / / (__ )
|
||||
/_/ /_/ /_/\___/\___/ /_/ |_/_/\__,_/_/ /_/ /_/ /_/____/
|
||||
*/
|
||||
|
||||
/**
|
||||
* Gets a list of all price alarms
|
||||
* @return Observable<PriceAlarm[]> An observable list of price alarms
|
||||
*/
|
||||
getPriceAlarms(): Observable<PriceAlarm[]> {
|
||||
try {
|
||||
return this.http.get<PriceAlarm[]>((this.apiUrl + '/pricealarms'));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new price alarm
|
||||
* @param productId The product id of the product to create the alarm for
|
||||
* @param definedPrice The defined target price
|
||||
* @return Observable<any> The observable response of the api
|
||||
*/
|
||||
createPriceAlarms(productId: number, definedPrice: number): Observable<any> {
|
||||
try {
|
||||
return this.http.post((this.apiUrl + '/pricealarms'), JSON.stringify({
|
||||
product_id: productId,
|
||||
defined_price: definedPrice
|
||||
}));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the given price alarm
|
||||
* @param alarmId The alarm id of the alarm to update
|
||||
* @param definedPrice The defined target price
|
||||
* @return Observable<any> The observable response of the api
|
||||
*/
|
||||
updatePriceAlarms(alarmId: number, definedPrice: number): Observable<any> {
|
||||
try {
|
||||
return this.http.put((this.apiUrl + '/pricealarms'), JSON.stringify({
|
||||
alarm_id: alarmId,
|
||||
defined_price: definedPrice
|
||||
}));
|
||||
} catch (exception) {
|
||||
process.stderr.write(`ERROR received from ${this.apiUrl}: ${exception}\n`);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user