laadpaal uitlezen

import_alfen/import_to_alfendb.py

@@ -0,0 +1,343 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Import charging station transaction data from CSV to existing MySQL database
+Aangepast voor bestaande 'transactions' tabel structuur
+"""
+
+import re
+import mysql.connector
+from datetime import datetime
+from decimal import Decimal
+import sys
+import io
+
+# Force UTF-8 encoding for stdout on Windows
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
+
+# Database configuration
+DB_CONFIG = {
+    'host': '192.168.178.201',
+    'port': 3307,   
+    'user': 'alfen_user',
+    'password': '5uVgr%f%s2P5GR@3q!',
+    'database': 'alfen',  # Jouw database naam
+    'charset': 'utf8mb4'
+}
+
+class ChargingDataImporter:
+    def __init__(self, config):
+        self.config = config
+        self.conn = None
+        self.cursor = None
+        self.existing_transactions = set()  # Track existing (id, start_timestamp) combinations
+        self.starts = []  # All transaction starts from CSV
+        self.stops = []   # All transaction stops from CSV
+
+    def connect(self):
+        """Connect to MySQL database"""
+        try:
+            self.conn = mysql.connector.connect(**self.config)
+            self.cursor = self.conn.cursor()
+            print("✓ Database verbinding succesvol")
+
+            # Load existing transaction IDs from database
+            self.cursor.execute("SELECT transaction_id FROM transactions")
+            existing_ids = set()
+            for (tx_id,) in self.cursor.fetchall():
+                existing_ids.add(tx_id)
+
+            self.existing_transaction_ids = existing_ids
+            print(f"✓ {len(self.existing_transaction_ids)} bestaande transacties geladen")
+
+        except mysql.connector.Error as err:
+            print(f"✗ Fout bij verbinden met database: {err}")
+            sys.exit(1)
+
+    def transaction_exists(self, unique_tx_id):
+        """Check if transaction with this unique ID already exists"""
+        return unique_tx_id in self.existing_transaction_ids
+    
+    def close(self):
+        """Close database connection"""
+        if self.cursor:
+            self.cursor.close()
+        if self.conn:
+            self.conn.close()
+    
+    def parse_txstart(self, line):
+        """Parse transaction start line and collect it"""
+        # txstart2: id 0x0000000000000001, socket 1, 2025-10-28 18:27:42 5518.267kWh 04BB29EAFD0F94 3 2 Y
+        pattern = r'txstart2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh (\w+)'
+        match = re.match(pattern, line)
+
+        if match:
+            tx_id = match.group(1)
+            socket_num = int(match.group(2))
+            timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
+            kwh = Decimal(match.group(4))
+            card = match.group(5)
+
+            self.starts.append({
+                'transaction_id': tx_id,
+                'socket': socket_num,
+                'timestamp': timestamp,
+                'kwh': kwh,
+                'card': card
+            })
+            return True
+        return False
+    
+    def parse_txstop(self, line):
+        """Parse transaction stop line and collect it"""
+        # txstop2: id 0x0000000000000001, socket 1, 2025-10-31 15:59:29 5540.316kWh 04BB29EAFD0F94 6 5 Y
+        pattern = r'txstop2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh'
+        match = re.match(pattern, line)
+
+        if match:
+            tx_id = match.group(1)
+            socket_num = int(match.group(2))
+            timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
+            kwh = Decimal(match.group(4))
+
+            self.stops.append({
+                'transaction_id': tx_id,
+                'socket': socket_num,
+                'timestamp': timestamp,
+                'kwh': kwh
+            })
+            return True
+        return False
+
+    def generate_unique_transaction_id(self, original_id, start_timestamp):
+        """Generate a unique transaction ID by combining original ID with timestamp"""
+        # Format: originalID_YYYYMMDDHHMMSS
+        timestamp_str = start_timestamp.strftime('%Y%m%d%H%M%S')
+        return f"{original_id}_{timestamp_str}"
+
+    def match_transactions(self):
+        """Match each start with its corresponding stop (closest timestamp after start)"""
+        matched_transactions = []
+        unmatched_starts = []
+        unmatched_stops = []
+
+        print(f"\n=== Transacties matchen ===")
+        print(f"Gevonden: {len(self.starts)} starts, {len(self.stops)} stops")
+
+        # For each start, find the corresponding stop
+        for start in self.starts:
+            # Find all stops with same transaction_id that come AFTER this start
+            potential_stops = [
+                stop for stop in self.stops
+                if stop['transaction_id'] == start['transaction_id']
+                and stop['timestamp'] > start['timestamp']
+            ]
+
+            if not potential_stops:
+                unmatched_starts.append(start)
+                continue
+
+            # Get the stop with the closest timestamp to this start
+            closest_stop = min(potential_stops, key=lambda s: s['timestamp'] - start['timestamp'])
+
+            # Generate unique transaction ID
+            unique_id = self.generate_unique_transaction_id(start['transaction_id'], start['timestamp'])
+
+            # Create matched transaction
+            matched_transactions.append({
+                'transaction_id': unique_id,
+                'original_transaction_id': start['transaction_id'],
+                'socket': start['socket'],
+                'start_timestamp': start['timestamp'],
+                'start_kWh': start['kwh'],
+                'stop_timestamp': closest_stop['timestamp'],
+                'stop_kWh': closest_stop['kwh'],
+                'total_kWh': closest_stop['kwh'] - start['kwh'],
+                'card': start['card']
+            })
+
+            # Remove this stop from the list so it won't be matched again
+            self.stops.remove(closest_stop)
+
+        # Remaining stops are unmatched
+        unmatched_stops = self.stops.copy()
+
+        print(f"✓ {len(matched_transactions)} transacties gematched")
+        if unmatched_starts:
+            print(f"⚠ {len(unmatched_starts)} starts zonder stop")
+        if unmatched_stops:
+            print(f"⚠ {len(unmatched_stops)} stops zonder start")
+
+        return matched_transactions
+
+    def save_transactions(self, transactions):
+        """Save matched transactions to database, skipping duplicates"""
+        print(f"\n=== Transacties opslaan ===")
+
+        saved_count = 0
+        skipped_count = 0
+        error_count = 0
+
+        for tx in transactions:
+            unique_id = tx['transaction_id']
+            original_id = tx['original_transaction_id']
+
+            # Check if this transaction already exists
+            if self.transaction_exists(unique_id):
+                print(f"  ⊘ {original_id} -> {unique_id} bestaat al")
+                skipped_count += 1
+                continue
+
+            try:
+                self.cursor.execute("""
+                    INSERT INTO transactions
+                    (transaction_id, socket, start_timestamp, start_kWh,
+                     stop_timestamp, stop_kWh, total_kWh, card)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                """, (
+                    unique_id,
+                    tx['socket'],
+                    tx['start_timestamp'],
+                    tx['start_kWh'],
+                    tx['stop_timestamp'],
+                    tx['stop_kWh'],
+                    tx['total_kWh'],
+                    tx['card']
+                ))
+
+                self.conn.commit()
+
+                # Add to existing transactions
+                self.existing_transaction_ids.add(unique_id)
+
+                print(f"  ✓ {original_id} -> {unique_id} opgeslagen ({tx['total_kWh']:.3f} kWh)")
+                saved_count += 1
+
+            except mysql.connector.Error as err:
+                print(f"  ✗ Fout bij opslaan {unique_id}: {err}")
+                error_count += 1
+
+        print(f"\n✓ Opgeslagen: {saved_count}")
+        print(f"⊘ Overgeslagen (duplicaat): {skipped_count}")
+        if error_count:
+            print(f"✗ Fouten: {error_count}")
+
+        return saved_count
+    
+    def import_file(self, filepath):
+        """Import CSV file into database"""
+        print(f"\n=== Import gestart: {filepath} ===")
+
+        line_count = 0
+
+        try:
+            # Phase 1: Scan file and collect all starts and stops
+            print("\nFase 1: CSV scannen...")
+            with open(filepath, 'r') as file:
+                for line_num, line in enumerate(file, 1):
+                    line = line.strip()
+
+                    # Skip empty lines and comments
+                    if not line or line.startswith('#'):
+                        continue
+
+                    line_count += 1
+
+                    # Parse transaction start
+                    if line.startswith('txstart2:'):
+                        self.parse_txstart(line)
+
+                    # Parse transaction stop
+                    elif line.startswith('txstop2:'):
+                        self.parse_txstop(line)
+
+                    # Skip meter values (mv:)
+                    elif line.startswith('mv:'):
+                        continue
+
+            print(f"✓ {line_count} regels gescand")
+            print(f"✓ {len(self.starts)} starts gevonden")
+            print(f"✓ {len(self.stops)} stops gevonden")
+
+            # Phase 2: Match starts with stops
+            matched_transactions = self.match_transactions()
+
+            # Phase 3: Save to database
+            saved_count = self.save_transactions(matched_transactions)
+
+            # Show summary statistics
+            self.show_statistics()
+
+        except FileNotFoundError:
+            print(f"✗ Bestand niet gevonden: {filepath}")
+            sys.exit(1)
+        except Exception as err:
+            print(f"✗ Onverwachte fout: {err}")
+            import traceback
+            traceback.print_exc()
+            self.conn.rollback()
+            sys.exit(1)
+    
+    def show_statistics(self):
+        """Show database statistics after import"""
+        print("\n=== Database Statistieken ===")
+        
+        # Total transactions
+        self.cursor.execute("SELECT COUNT(*) FROM transactions")
+        total_tx = self.cursor.fetchone()[0]
+        print(f"Totaal transacties in database: {total_tx}")
+        
+        # Total consumption
+        self.cursor.execute("SELECT SUM(total_kWh) FROM transactions")
+        result = self.cursor.fetchone()
+        total_consumption = result[0] if result[0] else 0
+        print(f"Totaal verbruik: {total_consumption:.3f} kWh")
+        
+        # Latest transaction
+        self.cursor.execute("""
+            SELECT transaction_id, start_timestamp, stop_timestamp, total_kWh 
+            FROM transactions 
+            ORDER BY stop_timestamp DESC 
+            LIMIT 1
+        """)
+        latest = self.cursor.fetchone()
+        if latest:
+            print(f"\nLaatste transactie:")
+            print(f"  ID: {latest[0]}")
+            print(f"  Periode: {latest[1]} - {latest[2]}")
+            print(f"  Verbruik: {latest[3]:.3f} kWh")
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Gebruik: python3 import_to_existing_db.py <csv_bestand>")
+        print("\nVoorbeeld: python3 import_to_existing_db.py VAN_01971_Transactions.csv")
+        sys.exit(1)
+    
+    csv_file = sys.argv[1]
+    
+    print("=" * 60)
+    print("Charging Station Data Importer")
+    print("Import naar bestaande 'transactions' tabel")
+    print("=" * 60)
+    
+    # Create importer instance
+    importer = ChargingDataImporter(DB_CONFIG)
+    
+    try:
+        # Connect to database
+        importer.connect()
+        
+        # Import the file
+        importer.import_file(csv_file)
+        
+    finally:
+        # Close connection
+        importer.close()
+        print("\n✓ Database verbinding gesloten")
+
+
+if __name__ == "__main__":
+    main()

import_alfen/uitlezen_laadpaal.py

@@ -0,0 +1,282 @@
+import requests
+import urllib3
+from datetime import datetime
+import re
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+# ── Configuratie ──────────────────────────────────────────────
+HOST       = os.getenv("HOST", "https://192.168.178.184")
+USERNAME   = os.getenv("CHARGER_USERNAME", "admin")
+PASSWORD   = os.getenv("CHARGER_PASSWORD")
+# ─────────────────────────────────────────────────────────────
+
+session = requests.Session()
+session.verify = False
+
+
+# ── Communicatie ──────────────────────────────────────────────
+
+def login():
+    session.post(f"{HOST}/api/login", json={
+        "username": USERNAME,
+        "password": PASSWORD
+    }).raise_for_status()
+    print("Ingelogd")
+
+def logout():
+    session.post(f"{HOST}/api/logout", json={
+        "username": USERNAME,
+        "password": PASSWORD
+    })
+    print("Uitgelogd")
+
+def get_info():
+    r = session.get(f"{HOST}/api/info")
+    r.raise_for_status()
+    return r.json()
+
+def fetch_page(offset):
+    r = session.get(f"{HOST}/api/transactions", params={"offset": offset})
+    r.raise_for_status()
+    return r.text
+
+def get_all_raw():
+    """Haalt alle transactiepagina's op via paginering.
+    Stopt zodra de record-nummers terugvallen (circulaire buffer bereikt).
+    """
+    import sys, itertools
+    spin = itertools.cycle('|/-\\')
+
+    all_raw = ""
+    offset = 0
+
+    while True:
+        sys.stdout.write(next(spin) + '\b')
+        sys.stdout.flush()
+        raw = fetch_page(offset)
+
+        stripped = raw.strip().rstrip('}').strip()
+        if not stripped or stripped in ('{"version":2,', '{"version":2'):
+            break
+
+        all_raw += raw
+
+        offsets = re.findall(r'^(\d+)_\w+:', raw, re.MULTILINE)
+        if not offsets:
+            break
+
+        next_offset = max(int(x) for x in offsets) + 1
+
+        if next_offset <= offset:
+            break
+
+        offset = next_offset
+
+    return all_raw
+
+
+# ── Parser ────────────────────────────────────────────────────
+
+def parse_transactions(raw):
+    transactions = []
+    current_tx = None
+    stop_parsing = False
+
+    # Verwijder pagina-headers eerst, zodat }{"version":2,305085_ correct wordt gesplitst
+    raw = re.sub(r'\{"version":\d+,\s*', '', raw)
+    # Records worden soms op één regel samengevoegd: "...N}305085_txstop2:..."
+    # Splits op } gevolgd door optionele spaties en een recordnummer
+    raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
+
+    for line in raw.splitlines():
+        if stop_parsing:
+            break
+
+        line = line.strip().rstrip('}').strip()
+        if not line:
+            continue
+
+        match = re.match(r'^(\d+)_(\w+):\s*(.+)$', line)
+        if not match:
+            continue
+
+        _, record_type, data = match.groups()
+
+        if record_type == 'txstart2':
+            m = re.match(
+                r'id (0x[0-9a-fA-F]+), socket (\d+), '
+                r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) '
+                r'([\d.]+)kWh (\S+)',
+                data
+            )
+            if m:
+                start_time = datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S')
+                # Cirkelbuffer: als de starttijd eerder is dan de hoogste geziene starttijd,
+                # zijn we in herhaalde data beland → stop met parsen
+                latest_start = max((tx['start_time'] for tx in transactions), default=None)
+                if latest_start and start_time < latest_start:
+                    break
+                current_tx = {
+                    'rfid':         m.group(1),
+                    'socket':       int(m.group(2)),
+                    'start_time':   start_time,
+                    'start_kwh':    float(m.group(4)),
+                    'tag':          m.group(5),
+                    'end_time':     None,
+                    'end_kwh':      None,
+                    'charged_kwh':  None,
+                    'status':       'lopend',
+                    'measurements': []
+                }
+                transactions.append(current_tx)
+
+        elif record_type == 'txstop2' and current_tx:
+            # Correct record type — was eerder fout als 'txend'
+            m = re.match(
+                r'id (0x[0-9a-fA-F]+), socket (\d+), '
+                r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) '
+                r'([\d.]+)kWh',
+                data
+            )
+            if m:
+                current_tx['end_time'] = datetime.strptime(m.group(3), '%Y-%m-%d %H:%M:%S')
+                current_tx['end_kwh']  = float(m.group(4))
+                current_tx['status']   = 'afgesloten'
+            current_tx = None  # Sessie klaar
+
+        elif record_type == 'mv' and current_tx:
+            # Sla korte mv-regels over (geen volledige meetwaarden)
+            m = re.match(
+                r'socket \d+, (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) '
+                r'([\d.]+) ([\d.]+) ([\d.]+) ([\d.]+) ([\d.]+) ([\d.]+)',
+                data
+            )
+            if m:
+                meting = {
+                    'time':         datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S'),
+                    'current_a':    float(m.group(2)),
+                    'energy_wh':    float(m.group(3)),
+                    'freq_hz':      float(m.group(4)),
+                    'power_w':      float(m.group(5)),
+                    'power_factor': float(m.group(6)),
+                    'temp_c':       float(m.group(7)),
+                }
+                current_tx['measurements'].append(meting)
+                # Alleen bijwerken als sessie nog niet afgesloten via txstop2
+                # en de meettijd na de starttijd ligt (circulaire buffer kan oude data bevatten)
+                if current_tx['status'] == 'lopend' and meting['time'] >= current_tx['start_time']:
+                    current_tx['end_time'] = meting['time']
+                    current_tx['end_kwh']  = meting['energy_wh'] / 1000
+
+    # Geladen kWh berekenen
+    for tx in transactions:
+        if tx['start_kwh'] is not None and tx['end_kwh'] is not None:
+            tx['charged_kwh'] = round(tx['end_kwh'] - tx['start_kwh'], 3)
+
+    return transactions
+
+
+# ── Rapportage ────────────────────────────────────────────────
+
+def print_summary(transactions):
+    print(f"\n{'='*68}")
+    print(f"  Totaal sessies gevonden: {len(transactions)}")
+    print(f"{'='*68}")
+    for i, tx in enumerate(transactions, 1):
+        duur = "?"
+        if tx['start_time'] and tx['end_time']:
+            delta = tx['end_time'] - tx['start_time']
+            uur, rest = divmod(int(delta.total_seconds()), 3600)
+            min_ = rest // 60
+            duur = f"{uur}u {min_:02d}m"
+
+        kwh_str = f"{tx['charged_kwh']} kWh" if tx['charged_kwh'] is not None else "?"
+        eind_str = tx['end_time'].strftime('%Y-%m-%d %H:%M') if tx['end_time'] else '?'
+
+        print(
+            f"  #{i:<3} {tx['start_time'].strftime('%Y-%m-%d %H:%M')}  "
+            f"→  {eind_str}  "
+            f"| {kwh_str:>10}  "
+            f"| {duur:>7}  "
+            f"| {tx['status']}"
+        )
+    print(f"{'='*68}")
+    totaal = sum(tx['charged_kwh'] for tx in transactions if tx['charged_kwh'])
+    print(f"  Totaal geladen: {round(totaal, 3)} kWh")
+    print(f"{'='*68}\n")
+
+def save_as_csv(raw, device_id, filename=None):
+    """Schrijft de ruwe API-data naar een CSV in ACE Service Installer formaat."""
+    from datetime import datetime
+
+    if filename is None:
+        filename = f"{device_id}_Transactions.csv"
+
+    now = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
+    lines = [
+        f"# Device, {device_id}",
+        f"# Generated, {now}",
+    ]
+
+    # Zelfde normalisatie als parse_transactions
+    raw = re.sub(r'\{"version":\d+,\s*', '', raw)
+    raw = re.sub(r'\}\s*(\d+_)', r'\n\1', raw)
+
+    latest_start = None
+    for line in raw.splitlines():
+        line = line.rstrip('}').strip()
+        if not line:
+            continue
+
+        # Verwijder offset-nummers: "76_mv:" → "mv:", "0_txstart2:" → "txstart2:"
+        line = re.sub(r'^\d+_', '', line)
+
+        # Cirkelbuffer: stop als txstart2 terug in de tijd gaat
+        if line.startswith('txstart2:'):
+            m = re.search(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})', line)
+            if m:
+                start_time = datetime.strptime(m.group(1), '%Y-%m-%d %H:%M:%S')
+                if latest_start and start_time < latest_start:
+                    break
+                latest_start = start_time
+
+        lines.append(line)
+
+    with open(filename, 'w', encoding='utf-8') as f:
+        f.write('\n'.join(lines) + '\n')
+
+    print(f"Opgeslagen als: {filename}")
+    return filename
+
+# ── Main ──────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    info = get_info()
+    device_id = info.get('Identity', 'UNKNOWN')
+    print(f"Paal: {info.get('Model')} | FW: {info.get('FWVersion')} | ID: {info.get('ObjectId')}")
+
+    login()
+    try:
+        print("\nTransacties ophalen...", end=' ', flush=True)
+        raw = get_all_raw()
+        print("Verwerken...", end=' ', flush=True)
+        transactions = parse_transactions(raw)
+        print("Klaar.\n")
+        print_summary(transactions)
+        print("Opslaan als CSV...", end=' ', flush=True)
+        start_dates = [tx['start_time'] for tx in transactions if tx['start_time']]
+        end_dates   = [tx['end_time']   for tx in transactions if tx['end_time']]
+        if start_dates and end_dates:
+            date_from = min(start_dates).strftime('%Y%m%d')
+            date_to   = max(end_dates).strftime('%Y%m%d')
+            filename  = f"{device_id}_Transactions_{date_from}_{date_to}.csv"
+        else:
+            filename  = f"{device_id}_Transactions.csv"
+        save_as_csv(raw, device_id, filename)
+    finally:
+        logout()