From 177148c94735e6ab9b8343d219824e319a45d245 Mon Sep 17 00:00:00 2001 From: Mark Kors Date: Thu, 9 Apr 2026 09:42:46 +0200 Subject: [PATCH] transactions from laadpaal parsing updates --- uitlezen_laadpaal.py | 81 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 68 insertions(+), 13 deletions(-) diff --git a/uitlezen_laadpaal.py b/uitlezen_laadpaal.py index bc4109d..4796f87 100644 --- a/uitlezen_laadpaal.py +++ b/uitlezen_laadpaal.py @@ -11,8 +11,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # ── Configuratie ────────────────────────────────────────────── HOST = os.getenv("HOST", "https://192.168.178.184") -USERNAME = os.getenv("USERNAME", "admin") -PASSWORD = os.getenv("PASSWORD") +USERNAME = os.getenv("CHARGER_USERNAME", "admin") +PASSWORD = os.getenv("CHARGER_PASSWORD") # ───────────────────────────────────────────────────────────── session = requests.Session() @@ -45,33 +45,65 @@ def fetch_page(offset): r.raise_for_status() return r.text +def _probe_max_offset(): + """Doe één request met een enorm offset-nummer om het hoogste record-nummer te vinden.""" + raw = fetch_page(999_999_999) + offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE) + return max(int(x) for x in offsets) if offsets else None + +def _print_progress(current, total, width=40): + import sys + pct = min(current / total, 1.0) + filled = int(width * pct) + bar = '#' * filled + '-' * (width - filled) + line = f" [{bar}] {pct*100:5.1f}% (record {current}/{total})" + sys.stdout.write(f"\r{line:<70}") + sys.stdout.flush() + def get_all_raw(): """Haalt alle transactiepagina's op via paginering. Stopt zodra de record-nummers terugvallen (circulaire buffer bereikt). """ + import sys + + print(" Bepalen totaal aantal records...", end='', flush=True) + max_record = _probe_max_offset() + if max_record: + print(f" max record = {max_record}") + else: + print(" onbekend, toon teller") + all_raw = "" offset = 0 while True: raw = fetch_page(offset) - print(f" Offset {offset:>6}: {len(raw):>6} bytes") stripped = raw.strip().rstrip('}').strip() if not stripped or stripped in ('{"version":2,', '{"version":2'): - print("Lege response, klaar.") + if max_record: + _print_progress(max_record, max_record) + print("\n Klaar.") break all_raw += raw offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE) if not offsets: + print("\n Klaar.") break next_offset = max(int(x) for x in offsets) + 1 + if max_record: + _print_progress(next_offset, max_record) + else: + sys.stdout.write(f"\r Record {next_offset}...") + sys.stdout.flush() + # Circulaire buffer: nummers vallen terug → we hebben alles gehad if next_offset <= offset: - print("Circulaire data gedetecteerd, klaar.") + print("\n Klaar (circulaire buffer).") break offset = next_offset @@ -86,12 +118,18 @@ def parse_transactions(raw): current_tx = None stop_parsing = False + # Verwijder pagina-headers eerst, zodat }{"version":2,305085_ correct wordt gesplitst + raw = re.sub(r'\{"version":\d+,\s*', '', raw) + # Records worden soms op één regel samengevoegd: "...N}305085_txstop2:..." + # Splits op } gevolgd door optionele spaties en een recordnummer + raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw) + for line in raw.splitlines(): if stop_parsing: break line = line.strip().rstrip('}').strip() - if not line or line.startswith('{"version"'): + if not line: continue match = re.match(r'^(\d+)_(\w+):\s*(.+)$', line) @@ -108,10 +146,16 @@ def parse_transactions(raw): data ) if m: + start_time = datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S') + # Cirkelbuffer: als de starttijd eerder is dan de hoogste geziene starttijd, + # zijn we in herhaalde data beland → stop met parsen + latest_start = max((tx['start_time'] for tx in transactions), default=None) + if latest_start and start_time < latest_start: + break current_tx = { 'rfid': m.group(1), 'socket': int(m.group(2)), - 'start_time': datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S'), + 'start_time': start_time, 'start_kwh': float(m.group(4)), 'tag': m.group(5), 'end_time': None, @@ -155,7 +199,8 @@ def parse_transactions(raw): } current_tx['measurements'].append(meting) # Alleen bijwerken als sessie nog niet afgesloten via txstop2 - if current_tx['status'] == 'lopend': + # en de meettijd na de starttijd ligt (circulaire buffer kan oude data bevatten) + if current_tx['status'] == 'lopend' and meting['time'] >= current_tx['start_time']: current_tx['end_time'] = meting['time'] current_tx['end_kwh'] = meting['energy_wh'] / 1000 @@ -209,18 +254,28 @@ def save_as_csv(raw, device_id, filename=None): f"# Generated, {now}", ] - for line in raw.splitlines(): - # Verwijder {"version":2, aan het begin van een pagina - line = re.sub(r'\{"version":\d+,', '', line) - # Verwijder afsluitende } - line = line.rstrip('}').strip() + # Zelfde normalisatie als parse_transactions + raw = re.sub(r'\{"version":\d+,\s*', '', raw) + raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw) + latest_start = None + for line in raw.splitlines(): + line = line.rstrip('}').strip() if not line: continue # Verwijder offset-nummers: "76_mv:" → "mv:", "0_txstart2:" → "txstart2:" line = re.sub(r'^\d+_', '', line) + # Cirkelbuffer: stop als txstart2 terug in de tijd gaat + if line.startswith('txstart2:'): + m = re.search(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line) + if m: + start_time = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S') + if latest_start and start_time < latest_start: + break + latest_start = start_time + lines.append(line) with open(filename, 'w', encoding='utf-8') as f: