transactions from laadpaal parsing updates

uitlezen_laadpaal.py

@@ -11,8 +11,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # ── Configuratie ──────────────────────────────────────────────
 HOST       = os.getenv("HOST", "https://192.168.178.184")
-USERNAME   = os.getenv("USERNAME", "admin")
+USERNAME   = os.getenv("CHARGER_USERNAME", "admin")
-PASSWORD   = os.getenv("PASSWORD")
+PASSWORD   = os.getenv("CHARGER_PASSWORD")
 # ─────────────────────────────────────────────────────────────
 
 session = requests.Session()
@@ -45,33 +45,65 @@ def fetch_page(offset):
     r.raise_for_status()
     return r.text
 
+def _probe_max_offset():
+    """Doe één request met een enorm offset-nummer om het hoogste record-nummer te vinden."""
+    raw = fetch_page(999_999_999)
+    offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE)
+    return max(int(x) for x in offsets) if offsets else None
+
+def _print_progress(current, total, width=40):
+    import sys
+    pct = min(current / total, 1.0)
+    filled = int(width * pct)
+    bar = '#' * filled + '-' * (width - filled)
+    line = f"  [{bar}] {pct*100:5.1f}%  (record {current}/{total})"
+    sys.stdout.write(f"\r{line:<70}")
+    sys.stdout.flush()
+
 def get_all_raw():
     """Haalt alle transactiepagina's op via paginering.
     Stopt zodra de record-nummers terugvallen (circulaire buffer bereikt).
     """
+    import sys
+
+    print("  Bepalen totaal aantal records...", end='', flush=True)
+    max_record = _probe_max_offset()
+    if max_record:
+        print(f" max record = {max_record}")
+    else:
+        print(" onbekend, toon teller")
+
     all_raw = ""
     offset = 0
 
     while True:
         raw = fetch_page(offset)
-        print(f"  Offset {offset:>6}: {len(raw):>6} bytes")
 
         stripped = raw.strip().rstrip('}').strip()
         if not stripped or stripped in ('{"version":2,', '{"version":2'):
-            print("Lege response, klaar.")
+            if max_record:
+                _print_progress(max_record, max_record)
+            print("\n  Klaar.")
             break
 
         all_raw += raw
 
         offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE)
         if not offsets:
+            print("\n  Klaar.")
             break
 
         next_offset = max(int(x) for x in offsets) + 1
 
+        if max_record:
+            _print_progress(next_offset, max_record)
+        else:
+            sys.stdout.write(f"\r  Record {next_offset}...")
+            sys.stdout.flush()
+
         # Circulaire buffer: nummers vallen terug → we hebben alles gehad
         if next_offset <= offset:
-            print("Circulaire data gedetecteerd, klaar.")
+            print("\n  Klaar (circulaire buffer).")
             break
 
         offset = next_offset
@@ -86,12 +118,18 @@ def parse_transactions(raw):
     current_tx = None
     stop_parsing = False
 
+    # Verwijder pagina-headers eerst, zodat }{"version":2,305085_ correct wordt gesplitst
+    raw = re.sub(r'\{"version":\d+,\s*', '', raw)
+    # Records worden soms op één regel samengevoegd: "...N}305085_txstop2:..."
+    # Splits op } gevolgd door optionele spaties en een recordnummer
+    raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
+
     for line in raw.splitlines():
         if stop_parsing:
             break
 
         line = line.strip().rstrip('}').strip()
-        if not line or line.startswith('{"version"'):
+        if not line:
             continue
 
         match = re.match(r'^(\d+)_(\w+):\s*(.+)$', line)
@@ -108,10 +146,16 @@ def parse_transactions(raw):
                 data
             )
             if m:
+                start_time = datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S')
+                # Cirkelbuffer: als de starttijd eerder is dan de hoogste geziene starttijd,
+                # zijn we in herhaalde data beland → stop met parsen
+                latest_start = max((tx['start_time'] for tx in transactions), default=None)
+                if latest_start and start_time < latest_start:
+                    break
                 current_tx = {
                     'rfid':         m.group(1),
                     'socket':       int(m.group(2)),
-                    'start_time':   datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S'),
+                    'start_time':   start_time,
                     'start_kwh':    float(m.group(4)),
                     'tag':          m.group(5),
                     'end_time':     None,
@@ -155,7 +199,8 @@ def parse_transactions(raw):
                 }
                 current_tx['measurements'].append(meting)
                 # Alleen bijwerken als sessie nog niet afgesloten via txstop2
-                if current_tx['status'] == 'lopend':
+                # en de meettijd na de starttijd ligt (circulaire buffer kan oude data bevatten)
+                if current_tx['status'] == 'lopend' and meting['time'] >= current_tx['start_time']:
                     current_tx['end_time'] = meting['time']
                     current_tx['end_kwh']  = meting['energy_wh'] / 1000
 
@@ -209,18 +254,28 @@ def save_as_csv(raw, device_id, filename=None):
         f"# Generated, {now}",
     ]
 
-    for line in raw.splitlines():
+    # Zelfde normalisatie als parse_transactions
-        # Verwijder {"version":2, aan het begin van een pagina
+    raw = re.sub(r'\{"version":\d+,\s*', '', raw)
-        line = re.sub(r'\{"version":\d+,', '', line)
+    raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
-        # Verwijder afsluitende }
-        line = line.rstrip('}').strip()
 
+    latest_start = None
+    for line in raw.splitlines():
+        line = line.rstrip('}').strip()
         if not line:
             continue
 
         # Verwijder offset-nummers: "76_mv:" → "mv:", "0_txstart2:" → "txstart2:"
         line = re.sub(r'^\d+_', '', line)
 
+        # Cirkelbuffer: stop als txstart2 terug in de tijd gaat
+        if line.startswith('txstart2:'):
+            m = re.search(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line)
+            if m:
+                start_time = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')
+                if latest_start and start_time < latest_start:
+                    break
+                latest_start = start_time
+
         lines.append(line)
 
     with open(filename, 'w', encoding='utf-8') as f: