🐛 Fixed data fetcher

- Those fucking idiots at data.europa.eu somehow managed to fuck the data up by including some random string at the very beginning of the csv file.
- Also some small changes impacting performance
This commit is contained in:
Patrick Müller 2020-03-31 16:24:42 +02:00
parent 334f71e1b7
commit bd0bf56a1a
4 changed files with 256 additions and 44 deletions

View File

@ -46,12 +46,13 @@ class Analyser:
retList.sort() retList.sort()
return retList return retList
def getCasesGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def getCasesGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
""" """
Get a graph with the absolute number of cases by day for the entered country Get a graph with the absolute number of cases by day for the entered country
:param country: The country you wish to get the graph for :param country: The country you wish to get the graph for
:param start_date: The start date of the graph :param start_date: The start date of the graph
:param end_date: The end date of the graph :param end_date: The end date of the graph
:param showPlot: Whether to show the plot or only return the file path
:return: The path for the picture of the graph :return: The path for the picture of the graph
""" """
if country in self.getAvailableCountries(): if country in self.getAvailableCountries():
@ -67,21 +68,24 @@ class Analyser:
countryTimeData.plot(ax=ax, x='dateRep', y='totalCases') countryTimeData.plot(ax=ax, x='dateRep', y='totalCases')
plt.show(block=True) if showPlot:
plt.show(block=True)
filePath = ('graphs/casesGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d')) filePath = ('graphs/casesGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d'))
fig.savefig(filePath) fig.savefig(filePath)
plt.close(fig)
return filePath return filePath
else: else:
print('Unknown country') print('Unknown country')
return '-1' return '-1'
def getCaseIncreaseGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def getCaseIncreaseGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
""" """
Get a graph with the daily increase number of cases for the entered country Get a graph with the daily increase number of cases for the entered country
:param country: The country you wish to get the graph for :param country: The country you wish to get the graph for
:param start_date: The start date of the graph :param start_date: The start date of the graph
:param end_date: The end date of the graph :param end_date: The end date of the graph
:param showPlot: Whether to show the plot or only return the file path
:return: The path for the picture of the graph :return: The path for the picture of the graph
""" """
if country in self.getAvailableCountries(): if country in self.getAvailableCountries():
@ -95,9 +99,11 @@ class Analyser:
countryTimeData.plot(ax=ax, x='dateRep', y='cases') countryTimeData.plot(ax=ax, x='dateRep', y='cases')
plt.show(block=True) if showPlot:
plt.show(block=True)
filePath = ('graphs/casesIncreaseGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d')) filePath = ('graphs/casesIncreaseGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d'))
fig.savefig(filePath) fig.savefig(filePath)
plt.close(fig)
return filePath return filePath
else: else:
@ -116,12 +122,13 @@ class Analyser:
countryTimeData = countryData.loc[mask] countryTimeData = countryData.loc[mask]
return countryTimeData['cases'].sum() return countryTimeData['cases'].sum()
def getDeathGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def getDeathGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
""" """
Get a graph with the absolute number of cases by day for the entered country Get a graph with the absolute number of cases by day for the entered country
:param country: The country you wish to get the graph for :param country: The country you wish to get the graph for
:param start_date: The start date of the graph :param start_date: The start date of the graph
:param end_date: The end date of the graph :param end_date: The end date of the graph
:param showPlot: Whether to show the plot or only return the file path
:return: The path for the picture of the graph :return: The path for the picture of the graph
""" """
if country in self.getAvailableCountries(): if country in self.getAvailableCountries():
@ -137,21 +144,24 @@ class Analyser:
countryTimeData.plot(ax=ax, x='dateRep', y='totalDeaths') countryTimeData.plot(ax=ax, x='dateRep', y='totalDeaths')
plt.show(block=True) if showPlot:
plt.show(block=True)
filePath = ('graphs/deathsGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d')) filePath = ('graphs/deathsGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d'))
fig.savefig(filePath) fig.savefig(filePath)
plt.close(fig)
return filePath return filePath
else: else:
print('Unknown country') print('Unknown country')
return '-1' return '-1'
def getDeathIncreaseGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def getDeathIncreaseGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
""" """
Get a graph with the daily increase number of cases for the entered country Get a graph with the daily increase number of cases for the entered country
:param country: The country you wish to get the graph for :param country: The country you wish to get the graph for
:param start_date: The start date of the graph :param start_date: The start date of the graph
:param end_date: The end date of the graph :param end_date: The end date of the graph
:param showPlot: Whether to show the plot or only return the file path
:return: The path for the picture of the graph :return: The path for the picture of the graph
""" """
if country in self.getAvailableCountries(): if country in self.getAvailableCountries():
@ -165,9 +175,11 @@ class Analyser:
countryTimeData.plot(ax=ax, x='dateRep', y='deaths') countryTimeData.plot(ax=ax, x='dateRep', y='deaths')
plt.show(block=True) if showPlot:
plt.show(block=True)
filePath = ('graphs/deathsIncreaseGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d')) filePath = ('graphs/deathsIncreaseGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d'))
fig.savefig(filePath) fig.savefig(filePath)
plt.close(fig)
return filePath return filePath
else: else:
@ -186,12 +198,13 @@ class Analyser:
countryTimeData = countryData.loc[mask] countryTimeData = countryData.loc[mask]
return countryTimeData['deaths'].sum() return countryTimeData['deaths'].sum()
def getDailyDeathRateGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def getDailyDeathRateGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
""" """
Get a graph with the daily increase number of cases for the entered country Get a graph with the daily increase number of cases for the entered country
:param country: The country you wish to get the graph for :param country: The country you wish to get the graph for
:param start_date: The start date of the graph :param start_date: The start date of the graph
:param end_date: The end date of the graph :param end_date: The end date of the graph
:param showPlot: Whether to show the plot or only return the file path
:return: The path for the picture of the graph :return: The path for the picture of the graph
""" """
if country in self.getAvailableCountries(): if country in self.getAvailableCountries():
@ -209,9 +222,11 @@ class Analyser:
countryTimeData.plot(ax=ax, x='dateRep', y='deathRate') countryTimeData.plot(ax=ax, x='dateRep', y='deathRate')
plt.show(block=True) if showPlot:
plt.show(block=True)
filePath = ('graphs/dailyDeathRateGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d')) filePath = ('graphs/dailyDeathRateGraph_' + country + '_' + datetime.now().strftime('%Y-%m-%d'))
fig.savefig(filePath) fig.savefig(filePath)
plt.close(fig)
return filePath return filePath
else: else:
@ -230,7 +245,7 @@ class Analyser:
countryTimeData = countryData.loc[mask] countryTimeData = countryData.loc[mask]
return (countryTimeData['deaths'].sum()/countryTimeData['cases'].sum()*100) return (countryTimeData['deaths'].sum()/countryTimeData['cases'].sum()*100)
def testGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d')) -> str: def testGraph(self, country, start_date='2019-12-31', end_date=datetime.now().strftime('%Y-%m-%d'), showPlot=False) -> str:
countryData = self.df[self.df['countriesAndTerritories'].isin([country])] countryData = self.df[self.df['countriesAndTerritories'].isin([country])]
mask = (countryData['dateRep'] >= start_date) & (countryData['dateRep'] <= end_date) mask = (countryData['dateRep'] >= start_date) & (countryData['dateRep'] <= end_date)
countryTimeData = countryData.loc[mask] countryTimeData = countryData.loc[mask]
@ -241,4 +256,6 @@ class Analyser:
countryTimeData.plot.scatter(x='totalCases', y='cases', c='deaths') countryTimeData.plot.scatter(x='totalCases', y='cases', c='deaths')
plt.show(block=True) if showPlot:
plt.show(block=True)
plt.close()

View File

@ -13,8 +13,9 @@ def updateStatsFile():
with open('statsfile.csv', 'w') as file: with open('statsfile.csv', 'w') as file:
# Save the fetched csv String to a variable to be able to mutate it # Save the fetched csv String to a variable to be able to mutate it
writeString = str(resp.content) writeString = str(resp.content)
dateRepIndex = writeString.index('dateRep')
# Remove the first 2 characters and the very last character because these are some crap that we dont need # Remove the first 2 characters and the very last character because these are some crap that we dont need
writeString = writeString[2:len(writeString)-1] writeString = writeString[dateRepIndex:len(writeString)-1]
# Replace the crappy windows newline thingy with the beautiful unix newline char # Replace the crappy windows newline thingy with the beautiful unix newline char
writeString = writeString.replace('\\r\\n', '\n') writeString = writeString.replace('\\r\\n', '\n')
# Write the final string to the file # Write the final string to the file

View File

@ -10,5 +10,5 @@ import UserInterface as UI
if __name__ == '__main__': if __name__ == '__main__':
fetcher.updateStatsFile() fetcher.updateStatsFile()
ana.Analyser().testGraph('Germany') ana.Analyser().testGraph('Germany', showPlot=True)
UI.main() UI.main()

File diff suppressed because it is too large Load Diff