vibe coding update import script

import_to_alfendb.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 """
 Import charging station transaction data from CSV to existing MySQL database
 Aangepast voor bestaande 'transactions' tabel structuur
@@ -9,6 +10,12 @@ import mysql.connector
 from datetime import datetime
 from decimal import Decimal
 import sys
+import io
+
+# Force UTF-8 encoding for stdout on Windows
+if sys.platform == 'win32':
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
+    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
 
 # Database configuration
 DB_CONFIG = {
@@ -25,17 +32,33 @@ class ChargingDataImporter:
         self.config = config
         self.conn = None
         self.cursor = None
-        self.pending_transactions = {}  # Track transactions waiting for stop
+        self.existing_transactions = set()  # Track existing (id, start_timestamp) combinations
-        
+        self.starts = []  # All transaction starts from CSV
+        self.stops = []   # All transaction stops from CSV
+
     def connect(self):
         """Connect to MySQL database"""
         try:
             self.conn = mysql.connector.connect(**self.config)
             self.cursor = self.conn.cursor()
             print("✓ Database verbinding succesvol")
+
+            # Load existing transaction IDs from database
+            self.cursor.execute("SELECT transaction_id FROM transactions")
+            existing_ids = set()
+            for (tx_id,) in self.cursor.fetchall():
+                existing_ids.add(tx_id)
+
+            self.existing_transaction_ids = existing_ids
+            print(f"✓ {len(self.existing_transaction_ids)} bestaande transacties geladen")
+
         except mysql.connector.Error as err:
             print(f"✗ Fout bij verbinden met database: {err}")
             sys.exit(1)
+
+    def transaction_exists(self, unique_tx_id):
+        """Check if transaction with this unique ID already exists"""
+        return unique_tx_id in self.existing_transaction_ids
     
     def close(self):
         """Close database connection"""
@@ -45,147 +68,208 @@ class ChargingDataImporter:
             self.conn.close()
     
     def parse_txstart(self, line):
-        """Parse transaction start line and store temporarily"""
+        """Parse transaction start line and collect it"""
         # txstart2: id 0x0000000000000001, socket 1, 2025-10-28 18:27:42 5518.267kWh 04BB29EAFD0F94 3 2 Y
         pattern = r'txstart2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh (\w+)'
         match = re.match(pattern, line)
-        
+
-        if match:
-            tx_id = match.group(1)  # Keep as hex string
-            socket_num = int(match.group(2))
-            timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
-            kwh = Decimal(match.group(4))
-            card = match.group(5)
-            
-            # Store temporarily until we get the stop
-            self.pending_transactions[tx_id] = {
-                'transaction_id': tx_id,
-                'socket': socket_num,
-                'start_timestamp': timestamp,
-                'start_kWh': kwh,
-                'card': card
-            }
-            
-            print(f"  → Transactie {tx_id} gestart (wacht op stop...)")
-            return True
-        return False
-    
-    def parse_txstop(self, line):
-        """Parse transaction stop line and insert complete transaction"""
-        # txstop2: id 0x0000000000000001, socket 1, 2025-10-31 15:59:29 5540.316kWh 04BB29EAFD0F94 6 5 Y
-        pattern = r'txstop2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh'
-        match = re.match(pattern, line)
-        
         if match:
             tx_id = match.group(1)
             socket_num = int(match.group(2))
             timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
             kwh = Decimal(match.group(4))
-            
+            card = match.group(5)
-            # Check if we have the start for this transaction
+
-            if tx_id not in self.pending_transactions:
+            self.starts.append({
-                print(f"  ⚠ Stop gevonden voor {tx_id} maar geen start - overgeslagen")
+                'transaction_id': tx_id,
-                return False
+                'socket': socket_num,
-            
+                'timestamp': timestamp,
-            tx_data = self.pending_transactions[tx_id]
+                'kwh': kwh,
-            
+                'card': card
-            # Calculate total consumption
+            })
-            total_kwh = kwh - tx_data['start_kWh']
+            return True
-            
+        return False
+    
+    def parse_txstop(self, line):
+        """Parse transaction stop line and collect it"""
+        # txstop2: id 0x0000000000000001, socket 1, 2025-10-31 15:59:29 5540.316kWh 04BB29EAFD0F94 6 5 Y
+        pattern = r'txstop2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh'
+        match = re.match(pattern, line)
+
+        if match:
+            tx_id = match.group(1)
+            socket_num = int(match.group(2))
+            timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
+            kwh = Decimal(match.group(4))
+
+            self.stops.append({
+                'transaction_id': tx_id,
+                'socket': socket_num,
+                'timestamp': timestamp,
+                'kwh': kwh
+            })
+            return True
+        return False
+
+    def generate_unique_transaction_id(self, original_id, start_timestamp):
+        """Generate a unique transaction ID by combining original ID with timestamp"""
+        # Format: originalID_YYYYMMDDHHMMSS
+        timestamp_str = start_timestamp.strftime('%Y%m%d%H%M%S')
+        return f"{original_id}_{timestamp_str}"
+
+    def match_transactions(self):
+        """Match each start with its corresponding stop (closest timestamp after start)"""
+        matched_transactions = []
+        unmatched_starts = []
+        unmatched_stops = []
+
+        print(f"\n=== Transacties matchen ===")
+        print(f"Gevonden: {len(self.starts)} starts, {len(self.stops)} stops")
+
+        # For each start, find the corresponding stop
+        for start in self.starts:
+            # Find all stops with same transaction_id that come AFTER this start
+            potential_stops = [
+                stop for stop in self.stops
+                if stop['transaction_id'] == start['transaction_id']
+                and stop['timestamp'] > start['timestamp']
+            ]
+
+            if not potential_stops:
+                unmatched_starts.append(start)
+                continue
+
+            # Get the stop with the closest timestamp to this start
+            closest_stop = min(potential_stops, key=lambda s: s['timestamp'] - start['timestamp'])
+
+            # Generate unique transaction ID
+            unique_id = self.generate_unique_transaction_id(start['transaction_id'], start['timestamp'])
+
+            # Create matched transaction
+            matched_transactions.append({
+                'transaction_id': unique_id,
+                'original_transaction_id': start['transaction_id'],
+                'socket': start['socket'],
+                'start_timestamp': start['timestamp'],
+                'start_kWh': start['kwh'],
+                'stop_timestamp': closest_stop['timestamp'],
+                'stop_kWh': closest_stop['kwh'],
+                'total_kWh': closest_stop['kwh'] - start['kwh'],
+                'card': start['card']
+            })
+
+            # Remove this stop from the list so it won't be matched again
+            self.stops.remove(closest_stop)
+
+        # Remaining stops are unmatched
+        unmatched_stops = self.stops.copy()
+
+        print(f"✓ {len(matched_transactions)} transacties gematched")
+        if unmatched_starts:
+            print(f"⚠ {len(unmatched_starts)} starts zonder stop")
+        if unmatched_stops:
+            print(f"⚠ {len(unmatched_stops)} stops zonder start")
+
+        return matched_transactions
+
+    def save_transactions(self, transactions):
+        """Save matched transactions to database, skipping duplicates"""
+        print(f"\n=== Transacties opslaan ===")
+
+        saved_count = 0
+        skipped_count = 0
+        error_count = 0
+
+        for tx in transactions:
+            unique_id = tx['transaction_id']
+            original_id = tx['original_transaction_id']
+
+            # Check if this transaction already exists
+            if self.transaction_exists(unique_id):
+                print(f"  ⊘ {original_id} -> {unique_id} bestaat al")
+                skipped_count += 1
+                continue
+
             try:
-                # Check if transaction already exists
-                self.cursor.execute(
-                    "SELECT id FROM transactions WHERE transaction_id = %s",
-                    (tx_id,)
-                )
-                existing = self.cursor.fetchone()
-                
-                if existing:
-                    print(f"  ⚠ Transactie {tx_id} bestaat al - overgeslagen")
-                    del self.pending_transactions[tx_id]
-                    return False
-                
-                # Insert complete transaction
                 self.cursor.execute("""
-                    INSERT INTO transactions 
+                    INSERT INTO transactions
-                    (transaction_id, socket, start_timestamp, start_kWh, 
+                    (transaction_id, socket, start_timestamp, start_kWh,
                      stop_timestamp, stop_kWh, total_kWh, card)
                     VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
                 """, (
-                    tx_id,
+                    unique_id,
-                    tx_data['socket'],
+                    tx['socket'],
-                    tx_data['start_timestamp'],
+                    tx['start_timestamp'],
-                    tx_data['start_kWh'],
+                    tx['start_kWh'],
-                    timestamp,
+                    tx['stop_timestamp'],
-                    kwh,
+                    tx['stop_kWh'],
-                    total_kwh,
+                    tx['total_kWh'],
-                    tx_data['card']
+                    tx['card']
                 ))
-                
+
                 self.conn.commit()
-                
+
-                # Remove from pending
+                # Add to existing transactions
-                del self.pending_transactions[tx_id]
+                self.existing_transaction_ids.add(unique_id)
-                
+
-                print(f"  ✓ Transactie {tx_id} opgeslagen ({total_kwh:.3f} kWh)")
+                print(f"  ✓ {original_id} -> {unique_id} opgeslagen ({tx['total_kWh']:.3f} kWh)")
-                return True
+                saved_count += 1
-                
+
             except mysql.connector.Error as err:
-                print(f"  ✗ Fout bij opslaan transactie {tx_id}: {err}")
+                print(f"  ✗ Fout bij opslaan {unique_id}: {err}")
-                return False
+                error_count += 1
-        
+
-        return False
+        print(f"\n✓ Opgeslagen: {saved_count}")
+        print(f"⊘ Overgeslagen (duplicaat): {skipped_count}")
+        if error_count:
+            print(f"✗ Fouten: {error_count}")
+
+        return saved_count
     
     def import_file(self, filepath):
         """Import CSV file into database"""
-        print(f"\n=== Import gestart: {filepath} ===\n")
+        print(f"\n=== Import gestart: {filepath} ===")
-        
+
         line_count = 0
-        tx_start_count = 0
+
-        tx_stop_count = 0
-        tx_saved_count = 0
-        
         try:
+            # Phase 1: Scan file and collect all starts and stops
+            print("\nFase 1: CSV scannen...")
             with open(filepath, 'r') as file:
                 for line_num, line in enumerate(file, 1):
                     line = line.strip()
-                    
+
                     # Skip empty lines and comments
-                    if not line or line.startswith('# Generated'):
+                    if not line or line.startswith('#'):
                         continue
-                    
+
                     line_count += 1
-                    
+
                     # Parse transaction start
                     if line.startswith('txstart2:'):
-                        if self.parse_txstart(line):
+                        self.parse_txstart(line)
-                            tx_start_count += 1
+
-                    
                     # Parse transaction stop
                     elif line.startswith('txstop2:'):
-                        if self.parse_txstop(line):
+                        self.parse_txstop(line)
-                            tx_stop_count += 1
+
-                            tx_saved_count += 1
+                    # Skip meter values (mv:)
-                    
-                    # Skip meter values (mv:) - niet opgeslagen in deze tabel
                     elif line.startswith('mv:'):
                         continue
-            
+
-            print(f"\n=== Import voltooid ===")
+            print(f"✓ {line_count} regels gescand")
-            print(f"✓ Totaal regels verwerkt: {line_count}")
+            print(f"✓ {len(self.starts)} starts gevonden")
-            print(f"✓ Transacties gestart: {tx_start_count}")
+            print(f"✓ {len(self.stops)} stops gevonden")
-            print(f"✓ Transacties gestopt: {tx_stop_count}")
+
-            print(f"✓ Transacties opgeslagen: {tx_saved_count}")
+            # Phase 2: Match starts with stops
-            
+            matched_transactions = self.match_transactions()
-            # Check for incomplete transactions
+
-            if self.pending_transactions:
+            # Phase 3: Save to database
-                print(f"\n⚠ Let op: {len(self.pending_transactions)} transactie(s) nog actief (geen stop gevonden):")
+            saved_count = self.save_transactions(matched_transactions)
-                for tx_id in self.pending_transactions:
+
-                    print(f"  - {tx_id}")
-            
             # Show summary statistics
             self.show_statistics()
-            
+
         except FileNotFoundError:
             print(f"✗ Bestand niet gevonden: {filepath}")
             sys.exit(1)