48 lines
1.2 KiB
Python
48 lines
1.2 KiB
Python
import sql
|
|
import os
|
|
from crawler import Crawler
|
|
from firebonk_api import FirebonkApi
|
|
|
|
|
|
def __check_and_insert_in_database__(conn, article) -> bool:
|
|
"""
|
|
Checks, if the article is already known. If not, inserts it into the db.
|
|
:param conn: SQL connection
|
|
:param article: The article to check / insert
|
|
:return: If the article is already known
|
|
"""
|
|
cur = conn.cursor()
|
|
cur.execute('SELECT article_id FROM yaa_articles WHERE url = %s', article.url)
|
|
|
|
res = cur.fetchall()
|
|
|
|
if len(res) > 0:
|
|
cur.close()
|
|
return True
|
|
else:
|
|
cur.execute('INSERT INTO yaa_articles (title, summary, url, image_url) VALUES (%s, %s, %s, %s)',
|
|
(article.title, article.summary, article.url, article.image_url))
|
|
conn.commit()
|
|
cur.close()
|
|
return False
|
|
|
|
|
|
if __name__ == '__main__':
|
|
crawl = Crawler('https://www.ka-news.de')
|
|
conn = sql.get_connection()
|
|
|
|
user = os.environ.get('FIREBONK_USER')
|
|
pw = os.environ.get('FIREBONK_PASSWORD')
|
|
|
|
api = FirebonkApi('https://www.firebonk.de/api/v1', user, pw)
|
|
|
|
|
|
articles = crawl.check_for_new_yaa_articles()
|
|
|
|
for article in articles:
|
|
if not __check_and_insert_in_database__(conn, article):
|
|
# Post is new
|
|
api.create_post(article.title, article.summary, article.url, article.image_url)
|
|
|
|
conn.close()
|