mark/import_alfen_transactions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

transactions from laadpaal parsing updates

Browse Source
This commit is contained in:
Mark Kors
2026-04-09 09:42:46 +02:00
parent 88f75ccb38
commit 177148c947
1 changed files with 68 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
uitlezen_laadpaal.py
View File

@@ -11,8 +11,8 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# ── Configuratie ──────────────────────────────────────────────
HOST = os.getenv("HOST", "https://192.168.178.184")
USERNAME = os.getenv("USERNAME", "admin")
PASSWORD = os.getenv("PASSWORD")
USERNAME = os.getenv("CHARGER_USERNAME", "admin")
PASSWORD = os.getenv("CHARGER_PASSWORD")
# ─────────────────────────────────────────────────────────────
session = requests.Session()
@@ -45,33 +45,65 @@ def fetch_page(offset):
r.raise_for_status()
return r.text
def _probe_max_offset():
"""Doe één request met een enorm offset-nummer om het hoogste record-nummer te vinden."""
raw = fetch_page(999_999_999)
offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE)
return max(int(x) for x in offsets) if offsets else None
def _print_progress(current, total, width=40):
import sys
pct = min(current / total, 1.0)
filled = int(width * pct)
bar = '#' * filled + '-' * (width - filled)
line = f" [{bar}] {pct*100:5.1f}% (record {current}/{total})"
sys.stdout.write(f"\r{line:<70}")
sys.stdout.flush()
def get_all_raw():
"""Haalt alle transactiepagina's op via paginering.
Stopt zodra de record-nummers terugvallen (circulaire buffer bereikt).
"""
import sys
print(" Bepalen totaal aantal records...", end='', flush=True)
max_record = _probe_max_offset()
if max_record:
print(f" max record = {max_record}")
else:
print(" onbekend, toon teller")
all_raw = ""
offset = 0
while True:
raw = fetch_page(offset)
print(f" Offset {offset:>6}: {len(raw):>6} bytes")
stripped = raw.strip().rstrip('}').strip()
if not stripped or stripped in ('{"version":2,', '{"version":2'):
print("Lege response, klaar.")
if max_record:
_print_progress(max_record, max_record)
print("\n Klaar.")
break
all_raw += raw
offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE)
if not offsets:
print("\n Klaar.")
break
next_offset = max(int(x) for x in offsets) + 1
if max_record:
_print_progress(next_offset, max_record)
else:
sys.stdout.write(f"\r Record {next_offset}...")
sys.stdout.flush()
# Circulaire buffer: nummers vallen terug → we hebben alles gehad
if next_offset <= offset:
print("Circulaire data gedetecteerd, klaar.")
print("\n Klaar (circulaire buffer).")
break
offset = next_offset
@@ -86,12 +118,18 @@ def parse_transactions(raw):
current_tx = None
stop_parsing = False
# Verwijder pagina-headers eerst, zodat }{"version":2,305085_ correct wordt gesplitst
raw = re.sub(r'\{"version":\d+,\s*', '', raw)
# Records worden soms op één regel samengevoegd: "...N}305085_txstop2:..."
# Splits op } gevolgd door optionele spaties en een recordnummer
raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
for line in raw.splitlines():
if stop_parsing:
break
line = line.strip().rstrip('}').strip()
if not line or line.startswith('{"version"'):
if not line:
continue
match = re.match(r'^(\d+)_(\w+):\s*(.+)$', line)
@@ -108,10 +146,16 @@ def parse_transactions(raw):
data
)
if m:
start_time = datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S')
# Cirkelbuffer: als de starttijd eerder is dan de hoogste geziene starttijd,
# zijn we in herhaalde data beland → stop met parsen
latest_start = max((tx['start_time'] for tx in transactions), default=None)
if latest_start and start_time < latest_start:
break
current_tx = {
'rfid': m.group(1),
'socket': int(m.group(2)),
'start_time': datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S'),
'start_time': start_time,
'start_kwh': float(m.group(4)),
'tag': m.group(5),
'end_time': None,
@@ -155,7 +199,8 @@ def parse_transactions(raw):
}
current_tx['measurements'].append(meting)
# Alleen bijwerken als sessie nog niet afgesloten via txstop2
if current_tx['status'] == 'lopend':
# en de meettijd na de starttijd ligt (circulaire buffer kan oude data bevatten)
if current_tx['status'] == 'lopend' and meting['time'] >= current_tx['start_time']:
current_tx['end_time'] = meting['time']
current_tx['end_kwh'] = meting['energy_wh'] / 1000
@@ -209,18 +254,28 @@ def save_as_csv(raw, device_id, filename=None):
f"# Generated, {now}",
]
for line in raw.splitlines():
# Verwijder {"version":2, aan het begin van een pagina
line = re.sub(r'\{"version":\d+,', '', line)
# Verwijder afsluitende }
line = line.rstrip('}').strip()
# Zelfde normalisatie als parse_transactions
raw = re.sub(r'\{"version":\d+,\s*', '', raw)
raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
latest_start = None
for line in raw.splitlines():
line = line.rstrip('}').strip()
if not line:
continue
# Verwijder offset-nummers: "76_mv:" → "mv:", "0_txstart2:" → "txstart2:"
line = re.sub(r'^\d+_', '', line)
# Cirkelbuffer: stop als txstart2 terug in de tijd gaat
if line.startswith('txstart2:'):
m = re.search(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line)
if m:
start_time = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')
if latest_start and start_time < latest_start:
break
latest_start = start_time
lines.append(line)
with open(filename, 'w', encoding='utf-8') as f: