From d418f156faf2374c7279f33c881f56ba853af803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20M=C3=BCller?= Date: Sat, 12 Sep 2026 11:15:37 +0200 Subject: [PATCH] German date display + optional street address for precise geocoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Date pickers (entry form + edit dialog) now display TT.MM.JJJJ instead of ISO (DATE_DISPLAY_FORMAT). Storage stays YYYY-MM-DD; parse_date() already accepted both formats, so existing data and the self-updater are unaffected. - New optional "Straße" field (street + house number) in the entry form and edit dialog, backed by a new `street` CSV column. geocode_city() and GeocoderWorker.enqueue() gained a street parameter: when set, a full-address query is tried first for a much more precise map point, falling back automatically to the existing city/PLZ search if it doesn't resolve. - Tab 2 and the entry queue show a Straße column; Tab 2 search now also matches on street. - Fix: pandas turns a blank CSV cell into NaN even for a dtype=str column, so every existing (blank-street) row would have shown literal "nan" in Tab 2. DataStore._load now does street.fillna("") after every read. Verified with a non-GUI test suite (date parsing, query construction, CSV round-trip incl. the NaN case) and a full GUI build/drive test on Python 3.12/Tk 9. Docs updated (changelog, overview, architecture, data-model, dev-notes, improvements). Co-Authored-By: Claude Sonnet 5 --- app.py | 106 +++++++++++++++++++++++++++++-------------- docs/architecture.md | 6 +-- docs/changelog.md | 22 +++++++++ docs/data-model.md | 14 ++++-- docs/dev-notes.md | 26 +++++++++++ docs/improvements.md | 14 ++++++ docs/overview.md | 58 ++++++++++++++--------- 7 files changed, 185 insertions(+), 61 deletions(-) diff --git a/app.py b/app.py index 9449e9f..39f4970 100755 --- a/app.py +++ b/app.py @@ -43,7 +43,10 @@ LOG_PATH = DATA_DIR / "app.log" ICON_PATH = BASE_DIR / "icon.png" # mitgeliefert (für Fenster + Startmenü) ICON_FALLBACK_PATH = DATA_DIR / "icon.png" # zur Laufzeit gezeichnet, falls obiges fehlt WINDOW_STATE_PATH = DATA_DIR / "window.json" -CSV_COLUMNS = ["date", "city", "postal_code", "lat", "lon"] +CSV_COLUMNS = ["date", "city", "postal_code", "street", "lat", "lon"] + +#: Anzeigeformat des Datums-Feldes (deutsch). Gespeichert wird trotzdem immer ISO. +DATE_DISPLAY_FORMAT = "%d.%m.%Y" #: Wie viele automatische Backups von orte.csv aufbewahrt werden. MAX_BACKUPS = 20 @@ -203,6 +206,9 @@ class DataStore: if col not in self.df.columns: self.df[col] = "" self.df = self.df[CSV_COLUMNS] + # Leere Zellen liest pandas als NaN ein (auch bei dtype=str) – "street" + # ist optional und daher oft leer, das würde sonst als "nan" angezeigt. + self.df["street"] = self.df["street"].fillna("") self._refresh_known() def _load_from_newest_backup(self) -> pd.DataFrame: @@ -321,14 +327,24 @@ class GeocodingUnavailable(Exception): """Alle Geokodier-Anfragen sind an einem Dienst-/Netzwerkfehler gescheitert.""" -def geocode_city(city: str, postal_code: str) -> tuple[float, float] | None: - """Sucht Koordinaten für ``city``/``postal_code``. +def geocode_city(city: str, postal_code: str, street: str = "") -> tuple[float, float] | None: + """Sucht Koordinaten für ``city``/``postal_code``, optional mit ``street``. + + Ist eine Straße (inkl. Hausnummer) angegeben, wird zuerst die genaue Adresse + versucht – das liefert einen deutlich präziseren Punkt als nur Ort/PLZ. + Schlägt das fehl (z. B. Tippfehler, unbekannte Hausnummer), fällt die Suche + automatisch auf Ort/PLZ zurück, genau wie ohne Straßenangabe. Rückgabe ``None`` bedeutet "Ort nicht gefunden". Ein :class:`GeocodingUnavailable` wird geworfen, wenn *jede* Anfrage an einem Dienst- oder Netzwerkfehler gescheitert ist (z. B. keine Internetverbindung). """ queries: list[str] = [] + if street: + if postal_code: + queries.append(f"{street}, {postal_code} {city}, Germany") + else: + queries.append(f"{street}, {city}, Germany") if postal_code: queries.append(f"{postal_code} {city}, Germany") queries += [f"{city}, {region}, Germany" for region in GEOCODE_REGIONS] @@ -365,16 +381,16 @@ class GeocoderWorker(threading.Thread): def __init__(self, on_service_error=None) -> None: super().__init__(daemon=True) - self._items: list[tuple[str, str, str, object]] = [] + self._items: list[tuple[str, str, str, str, object]] = [] self._lock = threading.Lock() self._event = threading.Event() self._stopped = False self._on_service_error = on_service_error self._consecutive_errors = 0 - def enqueue(self, row_id: str, city: str, plz: str, callback) -> None: + def enqueue(self, row_id: str, city: str, plz: str, street: str, callback) -> None: with self._lock: - self._items.append((row_id, city, plz, callback)) + self._items.append((row_id, city, plz, street, callback)) self._event.set() def run(self) -> None: @@ -385,11 +401,11 @@ class GeocoderWorker(threading.Thread): with self._lock: if not self._items: break - row_id, city, plz, callback = self._items.pop(0) + row_id, city, plz, street, callback = self._items.pop(0) coords: tuple[float, float] | None = None try: - coords = geocode_city(city, plz) + coords = geocode_city(city, plz, street) self._consecutive_errors = 0 except GeocodingUnavailable: self._consecutive_errors += 1 @@ -577,12 +593,9 @@ class EditDialog(tk.Toplevel): start = parse_date(str(row.get("date", ""))) start_dt = datetime.strptime(start, "%Y-%m-%d") if start else None self._date_entry = ttk.DateEntry( - self, dateformat="%Y-%m-%d", width=12, startdate=start_dt + self, dateformat=DATE_DISPLAY_FORMAT, width=12, startdate=start_dt ) self._date_entry.grid(row=0, column=1, sticky=W, **pad) - if start: - self._date_entry.entry.delete(0, END) - self._date_entry.entry.insert(0, start) ttk.Label(self, text="Ort:").grid(row=1, column=0, sticky=W, **pad) self._city_var = tk.StringVar(value=str(row.get("city", ""))) @@ -596,14 +609,20 @@ class EditDialog(tk.Toplevel): self._plz_var = tk.StringVar(value=str(row.get("postal_code", ""))) ttk.Entry(self, textvariable=self._plz_var, width=10).grid(row=2, column=1, sticky=W, **pad) + ttk.Label(self, text="Straße:").grid(row=3, column=0, sticky=W, **pad) + self._street_var = tk.StringVar(value=str(row.get("street", ""))) + ttk.Entry(self, textvariable=self._street_var, width=28).grid( + row=3, column=1, sticky=W, **pad + ) + self._regeocode_var = tk.BooleanVar(value=False) ttk.Checkbutton( self, text="Koordinaten automatisch neu suchen", variable=self._regeocode_var, bootstyle="round-toggle", command=self._on_regeocode_toggle, - ).grid(row=3, column=0, columnspan=2, sticky=W, **pad) + ).grid(row=4, column=0, columnspan=2, sticky=W, **pad) coord_frame = ttk.LabelFrame(self, text=" Koordinaten (manuell) ", padding=(8, 4)) - coord_frame.grid(row=4, column=0, columnspan=2, sticky=EW, padx=8, pady=(4, 4)) + coord_frame.grid(row=5, column=0, columnspan=2, sticky=EW, padx=8, pady=(4, 4)) ttk.Label(coord_frame, text="Breitengrad:").grid(row=0, column=0, sticky=W, padx=(0, 4), pady=2) self._lat_var = tk.StringVar(value=_coord_str(row.get("lat", ""))) self._lat_entry = ttk.Entry(coord_frame, textvariable=self._lat_var, width=14) @@ -617,7 +636,7 @@ class EditDialog(tk.Toplevel): ).grid(row=2, column=0, columnspan=2, sticky=W, pady=(2, 0)) btn_frame = ttk.Frame(self) - btn_frame.grid(row=5, column=0, columnspan=2, pady=(8, 6)) + btn_frame.grid(row=6, column=0, columnspan=2, pady=(8, 6)) ttk.Button(btn_frame, text="Speichern", bootstyle="success", command=self._on_save).pack(side=LEFT, padx=6) ttk.Button(btn_frame, text="Abbrechen", bootstyle="secondary-outline", @@ -654,7 +673,7 @@ class EditDialog(tk.Toplevel): if iso is None: messagebox.showwarning( "Ungültiges Datum", - "Bitte ein gültiges Datum im Format JJJJ-MM-TT eingeben.", + "Bitte ein gültiges Datum im Format TT.MM.JJJJ eingeben.", parent=self, ) return @@ -686,6 +705,7 @@ class EditDialog(tk.Toplevel): "date": iso, "city": city, "postal_code": self._plz_var.get().strip(), + "street": self._street_var.get().strip(), "regeocode": regeocode, "lat": lat, "lon": lon, @@ -1004,7 +1024,7 @@ class App(ttk.Window): plz_ok = (self.register(lambda P: P == "" or (P.isdigit() and len(P) <= 5)), "%P") ttk.Label(input_frame, text="Datum:").grid(row=0, column=0, sticky=W, padx=(0, 4), pady=4) - self._date_entry = ttk.DateEntry(input_frame, dateformat="%Y-%m-%d", width=12) + self._date_entry = ttk.DateEntry(input_frame, dateformat=DATE_DISPLAY_FORMAT, width=12) self._date_entry.grid(row=0, column=1, sticky=W, padx=(0, 16), pady=4) ttk.Label(input_frame, text="Ort:").grid(row=0, column=2, sticky=W, padx=(0, 4), pady=4) @@ -1029,25 +1049,36 @@ class App(ttk.Window): ) add_btn.grid(row=0, column=6, pady=4) - for w in (self._date_entry.entry, self.city_cb, plz_entry): + ttk.Label(input_frame, text="Straße:").grid(row=1, column=0, sticky=W, padx=(0, 4), pady=4) + self._street_var = ttk.StringVar() + street_entry = ttk.Entry(input_frame, textvariable=self._street_var, width=40) + street_entry.grid(row=1, column=1, columnspan=4, sticky=EW, pady=4) + ttk.Label( + input_frame, text="optional, für genauere Kartenpunkte", bootstyle="secondary", + ).grid(row=1, column=5, columnspan=2, sticky=W, padx=(8, 0), pady=4) + + for w in (self._date_entry.entry, self.city_cb, plz_entry, street_entry): w.bind("", lambda _e: self._on_add_row()) # Warteschlange queue_frame = ttk.LabelFrame(parent, text=" Warteschlange (noch nicht gespeichert) ", padding=(8, 4)) queue_frame.pack(fill=BOTH, expand=True, padx=4, pady=4) - cols = ("date", "city", "postal_code", "coords", "del") + cols = ("date", "city", "postal_code", "street", "coords", "del") self.tree = ttk.Treeview(queue_frame, columns=cols, show="headings", height=9) self.tree.heading("date", text="Datum") self.tree.heading("city", text="Ort") self.tree.heading("postal_code", text="PLZ") + self.tree.heading("street", text="Straße") self.tree.heading("coords", text="Koordinaten") self.tree.heading("del", text="") self.tree.column("date", width=100, minwidth=90, stretch=False) - self.tree.column("city", width=180, minwidth=120) + self.tree.column("city", width=150, minwidth=110) self.tree.column("postal_code", width=65, minwidth=50, stretch=False) + self.tree.column("street", width=150, minwidth=100) self.tree.column("coords", width=185, minwidth=130) self.tree.column("del", width=32, minwidth=32, stretch=False, anchor=CENTER) + self._queue_del_col = f"#{cols.index('del') + 1}" sb = ttk.Scrollbar(queue_frame, orient=VERTICAL, command=self.tree.yview) self.tree.configure(yscrollcommand=sb.set) @@ -1104,7 +1135,8 @@ class App(ttk.Window): hist_frame.pack(fill=BOTH, expand=True) self._hist_headings = { - "date": "Datum", "city": "Ort", "postal_code": "PLZ", "coords": "Karte", + "date": "Datum", "city": "Ort", "postal_code": "PLZ", + "street": "Straße", "coords": "Karte", } cols = tuple(self._hist_headings) self.hist_tree = ttk.Treeview(hist_frame, columns=cols, show="headings", height=16) @@ -1114,8 +1146,9 @@ class App(ttk.Window): command=lambda k=key: self._sort_history(k), ) self.hist_tree.column("date", width=115, minwidth=95, stretch=False) - self.hist_tree.column("city", width=260, minwidth=140) + self.hist_tree.column("city", width=190, minwidth=120) self.hist_tree.column("postal_code", width=80, minwidth=55, stretch=False, anchor=CENTER) + self.hist_tree.column("street", width=180, minwidth=110) self.hist_tree.column("coords", width=90, minwidth=70, stretch=False, anchor=CENTER) sb2 = ttk.Scrollbar(hist_frame, orient=VERTICAL, command=self.hist_tree.yview) @@ -1170,7 +1203,8 @@ class App(ttk.Window): mask = ( df["date"].astype(str).str.lower().str.contains(query) | df["city"].astype(str).str.lower().str.contains(query) | - df["postal_code"].astype(str).str.lower().str.contains(query) + df["postal_code"].astype(str).str.lower().str.contains(query) | + df["street"].astype(str).str.lower().str.contains(query) ) df = df[mask] if self._only_missing_var.get(): @@ -1193,6 +1227,7 @@ class App(ttk.Window): row.get("date", ""), row.get("city", ""), row.get("postal_code", ""), + row.get("street", ""), "fehlt" if row_missing else "✓", ), ) @@ -1244,11 +1279,12 @@ class App(ttk.Window): row = self.store.df.loc[df_idx] city = str(row.get("city", "")).strip() plz = str(row.get("postal_code", "")).strip() + street = str(row.get("street", "")).strip() if not city: self._set_status("Eintrag hat keinen Ort – Koordinatensuche nicht möglich.") return self.geocoder.enqueue( - f"edit_{df_idx}", city, plz, + f"edit_{df_idx}", city, plz, street, lambda _rid, coords, idx=df_idx: self.after(0, self._apply_edit_geocode, idx, coords), ) self._set_status(f"Koordinaten werden gesucht für {city}…") @@ -1271,6 +1307,7 @@ class App(ttk.Window): "date": dlg.result["date"], "city": dlg.result["city"], "postal_code": dlg.result["postal_code"], + "street": dlg.result["street"], } if dlg.result["regeocode"]: @@ -1282,6 +1319,7 @@ class App(ttk.Window): f"edit_{df_idx}", dlg.result["city"], dlg.result["postal_code"], + dlg.result["street"], lambda _rid, coords, idx=df_idx: self.after(0, self._apply_edit_geocode, idx, coords), ) self._set_status(f"Koordinaten werden gesucht für {new_vals['city']}…") @@ -1354,16 +1392,17 @@ class App(ttk.Window): def _on_add_row(self) -> None: city = self._city_var.get().strip() plz = self._plz_var.get().strip() + street = self._street_var.get().strip() date_str = parse_date(self._date_entry.entry.get()) if not city: self._set_status("Bitte einen Ort eingeben.", "warning") return if date_str is None: - self._set_status("Bitte ein gültiges Datum (JJJJ-MM-TT) eingeben.", "warning") + self._set_status("Bitte ein gültiges Datum (TT.MM.JJJJ) eingeben.", "warning") messagebox.showwarning( "Ungültiges Datum", - "Bitte ein gültiges Datum im Format JJJJ-MM-TT eingeben " + "Bitte ein gültiges Datum im Format TT.MM.JJJJ eingeben " "oder den Kalender-Knopf benutzen.", parent=self, ) @@ -1372,15 +1411,16 @@ class App(ttk.Window): row_id = str(self._next_id) self._next_id += 1 row = {"_id": row_id, "date": date_str, "city": city, - "postal_code": plz, "lat": "", "lon": ""} + "postal_code": plz, "street": street, "lat": "", "lon": ""} self._queue.append(row) self.tree.insert("", END, iid=row_id, - values=(date_str, city, plz, "⏳ wird gesucht…", "✕")) - self.geocoder.enqueue(row_id, city, plz, self._geocode_done) + values=(date_str, city, plz, street, "⏳ wird gesucht…", "✕")) + self.geocoder.enqueue(row_id, city, plz, street, self._geocode_done) self._city_var.set("") self._plz_var.set("") + self._street_var.set("") self.city_cb.focus_set() self._set_status( f"{len(self._queue)} Einträge in der Warteschlange.", sticky=True @@ -1404,13 +1444,13 @@ class App(ttk.Window): if self.tree.exists(row_id): vals = list(self.tree.item(row_id, "values")) - vals[3] = coord_str + vals[4] = coord_str self.tree.item(row_id, values=vals) def _on_tree_click(self, event) -> None: if self.tree.identify_region(event.x, event.y) != "cell": return - if self.tree.identify_column(event.x) == "#5": + if self.tree.identify_column(event.x) == self._queue_del_col: row_id = self.tree.identify_row(event.y) if row_id: self._delete_queue_row(row_id) @@ -1448,10 +1488,10 @@ class App(ttk.Window): if row["_id"] == row_id: if self.tree.exists(row_id): vals = list(self.tree.item(row_id, "values")) - vals[3] = "⏳ wird gesucht…" + vals[4] = "⏳ wird gesucht…" self.tree.item(row_id, values=vals) self.geocoder.enqueue(row_id, row["city"], row["postal_code"], - self._geocode_done) + row.get("street", ""), self._geocode_done) break def _on_clear_queue(self) -> None: diff --git a/docs/architecture.md b/docs/architecture.md index f83694a..b70a129 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,8 +7,8 @@ Everything lives in [`app.py`](../app.py). There is no package structure. | Class / function | Responsibility | |------------------|----------------| | `DataStore` | Owns `orte.csv`. Loads it into a pandas `DataFrame`, exposes read helpers (`get_map_data`, `get_autocomplete_strings`, `total_count`, `find_duplicates`) and write helpers (`append_rows`, `update_row`, `delete_rows`). All writes go through `_save_csv` → rotating backup + atomic write. All columns are read as strings. Falls back to an empty frame (or the newest backup) if `orte.csv` is missing/empty/corrupt. | -| `geocode_city()` | Pure function: `(city, plz) -> (lat, lon) | None`. Tries several Nominatim queries with a configurable regional bias (`GEOCODE_REGIONS`). Raises `GeocodingUnavailable` if *every* query failed on a service/network error (vs. simply not finding the place). | -| `GeocoderWorker` | A `threading.Thread` (daemon). Holds a FIFO list of `(row_id, city, plz, callback)` items guarded by a lock, woken by an `Event`. Processes one item per second, calls `geocode_city`, then invokes `callback` **from the worker thread**. Each item is wrapped in `try/except` so one failure never kills the thread; after `SERVICE_ERROR_THRESHOLD` consecutive service errors it fires the optional `on_service_error` callback. | +| `geocode_city()` | Pure function: `(city, plz, street="") -> (lat, lon) | None`. Tries several Nominatim queries with a configurable regional bias (`GEOCODE_REGIONS`); if `street` is given, a full-address query is tried first and falls back to the city/PLZ-only queries on no match. Raises `GeocodingUnavailable` if *every* query failed on a service/network error (vs. simply not finding the place). | +| `GeocoderWorker` | A `threading.Thread` (daemon). Holds a FIFO list of `(row_id, city, plz, street, callback)` items guarded by a lock, woken by an `Event`. Processes one item per second, calls `geocode_city`, then invokes `callback` **from the worker thread**. Each item is wrapped in `try/except` so one failure never kills the thread; after `SERVICE_ERROR_THRESHOLD` consecutive service errors it fires the optional `on_service_error` callback. | | `parse_date()` / `parse_coord()` / helpers | Module-level pure functions for normalizing user input (date → ISO, coordinate strings → float, `"City (PLZ)"` splitting). Shared by the entry tab and `EditDialog`. | | `generate_map()` | Builds `karte.html` from `DataStore.get_map_data()` with Folium. | | `EditDialog` | `tk.Toplevel` modal dialog for editing one row. Returns its result via `self.result`. | @@ -22,7 +22,7 @@ Everything lives in [`app.py`](../app.py). There is no package structure. user types -> _on_add_row() -> append dict to self._queue -> insert row into Tab-1 Treeview (coords = "⏳ wird gesucht…") - -> geocoder.enqueue(row_id, city, plz, self._geocode_done) + -> geocoder.enqueue(row_id, city, plz, street, self._geocode_done) GeocoderWorker thread (1/sec): coords = geocode_city(...) diff --git a/docs/changelog.md b/docs/changelog.md index 13d23c3..38b665e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,28 @@ Notable changes to the app. Newest first. +## 2026-09-12 — German date display + optional street address for geocoding + +- **Date fields now display `TT.MM.JJJJ`** (e.g. `12.09.2026`) instead of ISO, + in both the entry form and the edit dialog (`DATE_DISPLAY_FORMAT`). Storage on + disk is unchanged (`YYYY-MM-DD`) — `parse_date()` still accepts either format + when typed, so nothing about existing data changes. +- **New optional "Straße" field** (street + house number) on the entry form and + in the edit dialog, and a new `street` CSV column. When set, geocoding tries + the full address first for a much more precise map point, falling back to the + existing city/PLZ search if the address doesn't resolve — see + [overview.md](overview.md#geocoding-behaviour). `geocode_city()` and + `GeocoderWorker.enqueue()` both gained a `street` parameter. +- Tab 2 and the entry queue both show a **Straße** column; the Tab 2 search box + now also matches on street. +- Fixed a bug this surfaced immediately: pandas turns a **blank CSV cell into + `NaN`, even for a column forced to `dtype=str`** — every pre-existing row (all + blank on `street`) would have displayed the literal text `nan`. `DataStore._load` + now does `self.df["street"] = self.df["street"].fillna("")`. See + [dev-notes.md](dev-notes.md#pandas-turns-blank-csv-cells-into-nan-even-with-dtypestr). +- `find_duplicates` intentionally still ignores `street` (unchanged: date + city + + PLZ only). + ## 2026-09-07 — fix: right-click context menu instantly triggered its first item On Linux the Tab 2 (and queue) right-click menu flashed and immediately ran diff --git a/docs/data-model.md b/docs/data-model.md index f5edc71..06d6a2e 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -7,14 +7,19 @@ just the header if it does not exist. | Column | Type on disk | Meaning | Notes | |--------|--------------|---------|-------| -| `date` | string | Assignment date | Always stored as `YYYY-MM-DD`. Input is validated and normalized by `parse_date()` (accepts `YYYY-MM-DD`, `DD.MM.YYYY`, `DD.MM.YY`, `YYYY/MM/DD`); invalid input is rejected before saving. Rows written before this change may still hold non-ISO strings. | +| `date` | string | Assignment date | Always **stored as `YYYY-MM-DD`**, regardless of display. Input is validated and normalized by `parse_date()` (accepts `YYYY-MM-DD`, `DD.MM.YYYY`, `DD.MM.YY`, `YYYY/MM/DD`); invalid input is rejected before saving. The date pickers *display* `TT.MM.JJJJ` (`DATE_DISPLAY_FORMAT`) since the 2026-09-12 change, but that's cosmetic — parsing/storage is unchanged, so old and new rows are identical on disk. | | `city` | string | Place name | Free text. Used for autocomplete and duplicate detection (trimmed, case-insensitive). | | `postal_code` | string | German PLZ | Optional. Kept as a string so leading zeros survive. Regex elsewhere accepts 4–5 digits. | +| `street` | string | Street + house number | Optional (e.g. `"Hauptstraße 12"`). Added 2026-09-12 for more precise geocoding — see [overview.md](overview.md#geocoding-behaviour). Rows written before that date have it blank. | | `lat` | string | Latitude | Blank until geocoded. Rounded to 5 dp. Parsed with `pd.to_numeric(errors="coerce")` when building the map. | | `lon` | string | Longitude | As above. | Every column is read as a string (`dtype={"postal_code": str, "lat": str, "lon": str}` -plus `city`/`date` default object). Numeric conversion happens only where needed. +plus `city`/`date` default object) — **except this doesn't stop a blank cell from +coming back as `NaN`** (a float), even for a column forced to `dtype=str`. This +bit `street` immediately (all pre-existing rows have it blank): `_load()` now +does `self.df["street"] = self.df["street"].fillna("")` right after reading. +See [dev-notes.md](dev-notes.md#pandas-turns-blank-csv-cells-into-nan-even-with-dtypestr). ### Row identity @@ -33,7 +38,10 @@ that ends up wrong. `find_duplicates` flags a queued row when an existing row matches on all three of: `date` (string-equal), `city` (trimmed, lower-cased), `postal_code` -(trimmed). It only produces a status-bar hint; duplicates are still written. +(trimmed). `street` is **not** part of the match (unchanged by the 2026-09-12 +address feature) — two visits to the same city/PLZ on the same day are flagged +as possible duplicates even with different streets. A modal now asks whether to +save anyway (see [changelog.md](changelog.md)). ### Map aggregation diff --git a/docs/dev-notes.md b/docs/dev-notes.md index cf508cf..9976cc8 100644 --- a/docs/dev-notes.md +++ b/docs/dev-notes.md @@ -57,6 +57,32 @@ Lesson: a tile source that works from `curl` or an `http://` page can still fail from `file://`. Test the map by **opening the generated `karte.html` directly**, not just by checking the URL in the HTML. +## pandas turns blank CSV cells into NaN even with `dtype=str` (2026-09-12) + +Assumption that turned out wrong: forcing a column's dtype (e.g. +`dtype={"street": str}`) does **not** stop `pd.read_csv` from parsing a truly +empty field as `NaN` (a float) instead of `""`. Verified directly: + +```python +>>> pd.read_csv(io.StringIO("a\n\n"), dtype={"a": str})["a"][0] +nan # not '' +>>> str(pd.read_csv(io.StringIO("a\n\n"), dtype={"a": str})["a"][0]) +'nan' # !! +``` + +This is *why* `lat`/`lon` already needed `_is_blank()` / `_coord_str()` helpers +— but the new `street` column (added 2026-09-12) hit it immediately and much +harder: it's optional, so *every* existing row has it blank, and without a fix +every one of them would show the literal text `nan` in Tab 2. Fix: after +`pd.read_csv` (all three load paths — normal, empty-file, backup-recovery), +`DataStore._load` runs `self.df["street"] = self.df["street"].fillna("")`. + +Rule for this codebase: **any column that can legitimately be blank needs an +explicit `.fillna("")` (or the `_is_blank`/`_coord_str` treatment) right after +`pd.read_csv`** — `dtype=str` alone is not enough. `city`/`postal_code` haven't +needed this only because they're rarely actually blank in practice, not because +they're immune. + ## Tk popup menus: bind ``, not `` (2026-09-07) `widget.bind("", …)` + `menu.tk_popup(x, y)` + `finally: diff --git a/docs/improvements.md b/docs/improvements.md index 5524d87..f9782bb 100644 --- a/docs/improvements.md +++ b/docs/improvements.md @@ -215,6 +215,20 @@ Open follow-ups: no rollback if a pushed update is broken (mitigated by `--ff-only` from a branch Patrick controls + pip errors caught pre-restart); `run.sh` / `.desktop` still not committed (item 5). +## German date + street address (2026-09-12) — ✅ done, one idea deferred + +Date pickers display `TT.MM.JJJJ`; storage untouched. New optional `street` +CSV column feeds a full-address geocode attempt before falling back to +city/PLZ. See [changelog.md](changelog.md). + +**Deferred idea, not requested:** since the same person is often logged at the +same city repeatedly, the address autocomplete could remember and suggest the +last-used street for a selected city (the way selecting a city already +auto-fills its PLZ). Would remove re-typing the same street each time, at the +cost of a bit more state to reason about. Worth doing if re-typing the address +turns out to be annoying in practice — not implemented now to keep the change +minimal. + ## Suggested order of remaining work 1. Deploy to the laptop: `git clone` + `./install.sh` (see [setup.md](setup.md)). diff --git a/docs/overview.md b/docs/overview.md index ceea0f8..e5c406c 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -16,14 +16,16 @@ Workflow: build up a **queue** of entries, let them geocode in the background, then save the whole batch at once. 1. **Entry form** – date (a **calendar picker**, `ttkbootstrap.DateEntry`, - pre-filled with today; typed input is accepted in `JJJJ-MM-TT` or - `TT.MM.JJJJ` and **validated** – an invalid date is rejected with a dialog), - city (autocomplete combobox), postal code (optional). Pressing `Return` in any - field, or the **+ Hinzufügen** button, adds the row to the queue. - The PLZ field only accepts digits. -2. **Queue table** – shows date, city, PLZ, and a live coordinate column that - updates from `⏳ wird gesucht…` to either `lat / lon` or `⚠ nicht gefunden` - as the background geocoder works through the queue. + pre-filled with today and **displayed as `TT.MM.JJJJ`**, e.g. `12.09.2026`; + typed input also accepts `JJJJ-MM-TT` and is **validated** – an invalid date + is rejected with a dialog), city (autocomplete combobox), postal code + (optional, digits only), and **Straße** (street + house number, optional, + second row of the form) — see [Geocoding behaviour](#geocoding-behaviour) for + why it's worth filling in. Pressing `Return` in any field, or the + **+ Hinzufügen** button, adds the row to the queue. +2. **Queue table** – shows date, city, PLZ, street, and a live coordinate column + that updates from `⏳ wird gesucht…` to either `lat / lon` or + `⚠ nicht gefunden` as the background geocoder works through the queue. - Select a row and press Delete, use the **"Auswahl entfernen"** button, or click the `✕` cell. - Right-click a row for *Entfernen* / *Koordinaten erneut suchen*. @@ -38,10 +40,12 @@ then save the whole batch at once. A table view of everything in `orte.csv`. -- **Columns:** Datum, Ort, PLZ, and **Karte** – a status column showing `✓` when - the row has coordinates or a red `fehlt` when it doesn't (the whole row is red - too). The raw lat/lon numbers live in the edit dialog, not this table. -- **Search box** – live filter across date, city, and PLZ (substring match). +- **Columns:** Datum, Ort, PLZ, Straße, and **Karte** – a status column showing + `✓` when the row has coordinates or a red `fehlt` when it doesn't (the whole + row is red too). The raw lat/lon numbers live in the edit dialog, not this + table. +- **Search box** – live filter across date, city, PLZ, **and street** + (substring match). - **Sortable columns** – click a header to sort; clicking again reverses. The active column shows a ▲/▼ arrow. Default sort is by date, newest first. Sorting by **Karte** groups the rows without coordinates together. @@ -49,10 +53,11 @@ A table view of everything in `orte.csv`. coordinates yet. The hint line always shows the total count, plus how many are missing coordinates. - **Edit** – double-click a row (or *Bearbeiten*) opens a modal dialog to change - date (calendar picker, validated) / city / PLZ. Two ways to fix coordinates: - a "Koordinaten automatisch neu suchen" toggle re-runs geocoding after saving, - or the **Breitengrad / Längengrad** fields let you type them in by hand - (both empty = no map marker). + date (calendar picker, `TT.MM.JJJJ`, validated) / city / PLZ / **Straße**. Two + ways to fix coordinates: a "Koordinaten automatisch neu suchen" toggle re-runs + geocoding after saving (now using the street if one is set), or the + **Breitengrad / Längengrad** fields let you type them in by hand (both empty = + no map marker). - **Right-click → Koordinaten suchen** – runs geocoding for that one row (handy for rows that failed the first time). - **Delete** – *Löschen* removes the selected row(s) after a confirmation @@ -87,13 +92,22 @@ Generated by `generate_map()`: ## Geocoding behaviour -`geocode_city(city, postal_code)` tries a series of queries in order and returns -the first hit: +`geocode_city(city, postal_code, street="")` tries a series of queries in order +and returns the **first hit**: -1. `" , Germany"` (only if a PLZ was given) -2. `", Baden-Württemberg, Germany"` -3. `", Hessen, Germany"` -4. `", Germany"` +1. `", , Germany"` — only if a **street** was given (with + PLZ if there is one, without it if not). This is what the optional Straße + field is for: a full address geocodes far more precisely than a city name + alone (a point in the right town vs. the right building). +2. `" , Germany"` (only if a PLZ was given) +3. `", Baden-Württemberg, Germany"` +4. `", Hessen, Germany"` +5. `", Germany"` + +If the street-level query (1) doesn't match anything in Nominatim (typo, house +number Nominatim doesn't know, etc.), it **falls back automatically** to the +same city/PLZ-level searches used when no street is given — a bad street never +makes geocoding *worse* than before, only better when it resolves. The regional bias is the `GEOCODE_REGIONS` constant (`Baden-Württemberg`, `Hessen`) near the top of `app.py`. Results are rounded to five decimal places.