41 lines
988 B
Python
41 lines
988 B
Python
import sql
|
|
from crawler import Crawler
|
|
|
|
|
|
def __check_and_insert_in_database__(conn, article) -> bool:
|
|
"""
|
|
Checks, if the article is already known. If not, inserts it into the db.
|
|
:param conn: SQL connection
|
|
:param article: The article to check / insert
|
|
:return: If the article is already known
|
|
"""
|
|
cur = conn.cursor()
|
|
cur.execute('SELECT article_id FROM yaa_articles WHERE url = %s', article.url)
|
|
|
|
res = cur.fetchall()
|
|
|
|
if len(res) > 0:
|
|
cur.close()
|
|
return True
|
|
else:
|
|
cur.execute('INSERT INTO yaa_articles (title, summary, url, image_url) VALUES (%s, %s, %s, %s)',
|
|
(article.title, article.summary, article.url, article.image_url))
|
|
conn.commit()
|
|
cur.close()
|
|
return False
|
|
|
|
|
|
if __name__ == '__main__':
|
|
crawl = Crawler('https://www.ka-news.de')
|
|
conn = sql.get_connection()
|
|
|
|
articles = crawl.check_for_new_yaa_articles()
|
|
|
|
for article in articles:
|
|
if not __check_and_insert_in_database__(conn, article):
|
|
print('New!')
|
|
else:
|
|
print('Old!')
|
|
|
|
conn.close()
|