vibe coding update import script
This commit is contained in:
Mark Kors
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Import charging station transaction data from CSV to existing MySQL database
|
||||
Aangepast voor bestaande 'transactions' tabel structuur
|
||||
@@ -9,6 +10,12 @@ import mysql.connector
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import sys
|
||||
import io
|
||||
|
||||
# Force UTF-8 encoding for stdout on Windows
|
||||
if sys.platform == 'win32':
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
# Database configuration
|
||||
DB_CONFIG = {
|
||||
@@ -25,17 +32,33 @@ class ChargingDataImporter:
|
||||
self.config = config
|
||||
self.conn = None
|
||||
self.cursor = None
|
||||
self.pending_transactions = {} # Track transactions waiting for stop
|
||||
|
||||
self.existing_transactions = set() # Track existing (id, start_timestamp) combinations
|
||||
self.starts = [] # All transaction starts from CSV
|
||||
self.stops = [] # All transaction stops from CSV
|
||||
|
||||
def connect(self):
|
||||
"""Connect to MySQL database"""
|
||||
try:
|
||||
self.conn = mysql.connector.connect(**self.config)
|
||||
self.cursor = self.conn.cursor()
|
||||
print("✓ Database verbinding succesvol")
|
||||
|
||||
# Load existing transaction IDs from database
|
||||
self.cursor.execute("SELECT transaction_id FROM transactions")
|
||||
existing_ids = set()
|
||||
for (tx_id,) in self.cursor.fetchall():
|
||||
existing_ids.add(tx_id)
|
||||
|
||||
self.existing_transaction_ids = existing_ids
|
||||
print(f"✓ {len(self.existing_transaction_ids)} bestaande transacties geladen")
|
||||
|
||||
except mysql.connector.Error as err:
|
||||
print(f"✗ Fout bij verbinden met database: {err}")
|
||||
sys.exit(1)
|
||||
|
||||
def transaction_exists(self, unique_tx_id):
|
||||
"""Check if transaction with this unique ID already exists"""
|
||||
return unique_tx_id in self.existing_transaction_ids
|
||||
|
||||
def close(self):
|
||||
"""Close database connection"""
|
||||
@@ -45,147 +68,208 @@ class ChargingDataImporter:
|
||||
self.conn.close()
|
||||
|
||||
def parse_txstart(self, line):
|
||||
"""Parse transaction start line and store temporarily"""
|
||||
"""Parse transaction start line and collect it"""
|
||||
# txstart2: id 0x0000000000000001, socket 1, 2025-10-28 18:27:42 5518.267kWh 04BB29EAFD0F94 3 2 Y
|
||||
pattern = r'txstart2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh (\w+)'
|
||||
match = re.match(pattern, line)
|
||||
|
||||
if match:
|
||||
tx_id = match.group(1) # Keep as hex string
|
||||
socket_num = int(match.group(2))
|
||||
timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
|
||||
kwh = Decimal(match.group(4))
|
||||
card = match.group(5)
|
||||
|
||||
# Store temporarily until we get the stop
|
||||
self.pending_transactions[tx_id] = {
|
||||
'transaction_id': tx_id,
|
||||
'socket': socket_num,
|
||||
'start_timestamp': timestamp,
|
||||
'start_kWh': kwh,
|
||||
'card': card
|
||||
}
|
||||
|
||||
print(f" → Transactie {tx_id} gestart (wacht op stop...)")
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_txstop(self, line):
|
||||
"""Parse transaction stop line and insert complete transaction"""
|
||||
# txstop2: id 0x0000000000000001, socket 1, 2025-10-31 15:59:29 5540.316kWh 04BB29EAFD0F94 6 5 Y
|
||||
pattern = r'txstop2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh'
|
||||
match = re.match(pattern, line)
|
||||
|
||||
|
||||
if match:
|
||||
tx_id = match.group(1)
|
||||
socket_num = int(match.group(2))
|
||||
timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
|
||||
kwh = Decimal(match.group(4))
|
||||
|
||||
# Check if we have the start for this transaction
|
||||
if tx_id not in self.pending_transactions:
|
||||
print(f" ⚠ Stop gevonden voor {tx_id} maar geen start - overgeslagen")
|
||||
return False
|
||||
|
||||
tx_data = self.pending_transactions[tx_id]
|
||||
|
||||
# Calculate total consumption
|
||||
total_kwh = kwh - tx_data['start_kWh']
|
||||
|
||||
card = match.group(5)
|
||||
|
||||
self.starts.append({
|
||||
'transaction_id': tx_id,
|
||||
'socket': socket_num,
|
||||
'timestamp': timestamp,
|
||||
'kwh': kwh,
|
||||
'card': card
|
||||
})
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_txstop(self, line):
|
||||
"""Parse transaction stop line and collect it"""
|
||||
# txstop2: id 0x0000000000000001, socket 1, 2025-10-31 15:59:29 5540.316kWh 04BB29EAFD0F94 6 5 Y
|
||||
pattern = r'txstop2: id (0x[0-9a-fA-F]+), socket (\d+), ([\d-]+ [\d:]+) ([\d.]+)kWh'
|
||||
match = re.match(pattern, line)
|
||||
|
||||
if match:
|
||||
tx_id = match.group(1)
|
||||
socket_num = int(match.group(2))
|
||||
timestamp = datetime.strptime(match.group(3), '%Y-%m-%d %H:%M:%S')
|
||||
kwh = Decimal(match.group(4))
|
||||
|
||||
self.stops.append({
|
||||
'transaction_id': tx_id,
|
||||
'socket': socket_num,
|
||||
'timestamp': timestamp,
|
||||
'kwh': kwh
|
||||
})
|
||||
return True
|
||||
return False
|
||||
|
||||
def generate_unique_transaction_id(self, original_id, start_timestamp):
|
||||
"""Generate a unique transaction ID by combining original ID with timestamp"""
|
||||
# Format: originalID_YYYYMMDDHHMMSS
|
||||
timestamp_str = start_timestamp.strftime('%Y%m%d%H%M%S')
|
||||
return f"{original_id}_{timestamp_str}"
|
||||
|
||||
def match_transactions(self):
|
||||
"""Match each start with its corresponding stop (closest timestamp after start)"""
|
||||
matched_transactions = []
|
||||
unmatched_starts = []
|
||||
unmatched_stops = []
|
||||
|
||||
print(f"\n=== Transacties matchen ===")
|
||||
print(f"Gevonden: {len(self.starts)} starts, {len(self.stops)} stops")
|
||||
|
||||
# For each start, find the corresponding stop
|
||||
for start in self.starts:
|
||||
# Find all stops with same transaction_id that come AFTER this start
|
||||
potential_stops = [
|
||||
stop for stop in self.stops
|
||||
if stop['transaction_id'] == start['transaction_id']
|
||||
and stop['timestamp'] > start['timestamp']
|
||||
]
|
||||
|
||||
if not potential_stops:
|
||||
unmatched_starts.append(start)
|
||||
continue
|
||||
|
||||
# Get the stop with the closest timestamp to this start
|
||||
closest_stop = min(potential_stops, key=lambda s: s['timestamp'] - start['timestamp'])
|
||||
|
||||
# Generate unique transaction ID
|
||||
unique_id = self.generate_unique_transaction_id(start['transaction_id'], start['timestamp'])
|
||||
|
||||
# Create matched transaction
|
||||
matched_transactions.append({
|
||||
'transaction_id': unique_id,
|
||||
'original_transaction_id': start['transaction_id'],
|
||||
'socket': start['socket'],
|
||||
'start_timestamp': start['timestamp'],
|
||||
'start_kWh': start['kwh'],
|
||||
'stop_timestamp': closest_stop['timestamp'],
|
||||
'stop_kWh': closest_stop['kwh'],
|
||||
'total_kWh': closest_stop['kwh'] - start['kwh'],
|
||||
'card': start['card']
|
||||
})
|
||||
|
||||
# Remove this stop from the list so it won't be matched again
|
||||
self.stops.remove(closest_stop)
|
||||
|
||||
# Remaining stops are unmatched
|
||||
unmatched_stops = self.stops.copy()
|
||||
|
||||
print(f"✓ {len(matched_transactions)} transacties gematched")
|
||||
if unmatched_starts:
|
||||
print(f"⚠ {len(unmatched_starts)} starts zonder stop")
|
||||
if unmatched_stops:
|
||||
print(f"⚠ {len(unmatched_stops)} stops zonder start")
|
||||
|
||||
return matched_transactions
|
||||
|
||||
def save_transactions(self, transactions):
|
||||
"""Save matched transactions to database, skipping duplicates"""
|
||||
print(f"\n=== Transacties opslaan ===")
|
||||
|
||||
saved_count = 0
|
||||
skipped_count = 0
|
||||
error_count = 0
|
||||
|
||||
for tx in transactions:
|
||||
unique_id = tx['transaction_id']
|
||||
original_id = tx['original_transaction_id']
|
||||
|
||||
# Check if this transaction already exists
|
||||
if self.transaction_exists(unique_id):
|
||||
print(f" ⊘ {original_id} -> {unique_id} bestaat al")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
# Check if transaction already exists
|
||||
self.cursor.execute(
|
||||
"SELECT id FROM transactions WHERE transaction_id = %s",
|
||||
(tx_id,)
|
||||
)
|
||||
existing = self.cursor.fetchone()
|
||||
|
||||
if existing:
|
||||
print(f" ⚠ Transactie {tx_id} bestaat al - overgeslagen")
|
||||
del self.pending_transactions[tx_id]
|
||||
return False
|
||||
|
||||
# Insert complete transaction
|
||||
self.cursor.execute("""
|
||||
INSERT INTO transactions
|
||||
(transaction_id, socket, start_timestamp, start_kWh,
|
||||
INSERT INTO transactions
|
||||
(transaction_id, socket, start_timestamp, start_kWh,
|
||||
stop_timestamp, stop_kWh, total_kWh, card)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
tx_id,
|
||||
tx_data['socket'],
|
||||
tx_data['start_timestamp'],
|
||||
tx_data['start_kWh'],
|
||||
timestamp,
|
||||
kwh,
|
||||
total_kwh,
|
||||
tx_data['card']
|
||||
unique_id,
|
||||
tx['socket'],
|
||||
tx['start_timestamp'],
|
||||
tx['start_kWh'],
|
||||
tx['stop_timestamp'],
|
||||
tx['stop_kWh'],
|
||||
tx['total_kWh'],
|
||||
tx['card']
|
||||
))
|
||||
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
# Remove from pending
|
||||
del self.pending_transactions[tx_id]
|
||||
|
||||
print(f" ✓ Transactie {tx_id} opgeslagen ({total_kwh:.3f} kWh)")
|
||||
return True
|
||||
|
||||
|
||||
# Add to existing transactions
|
||||
self.existing_transaction_ids.add(unique_id)
|
||||
|
||||
print(f" ✓ {original_id} -> {unique_id} opgeslagen ({tx['total_kWh']:.3f} kWh)")
|
||||
saved_count += 1
|
||||
|
||||
except mysql.connector.Error as err:
|
||||
print(f" ✗ Fout bij opslaan transactie {tx_id}: {err}")
|
||||
return False
|
||||
|
||||
return False
|
||||
print(f" ✗ Fout bij opslaan {unique_id}: {err}")
|
||||
error_count += 1
|
||||
|
||||
print(f"\n✓ Opgeslagen: {saved_count}")
|
||||
print(f"⊘ Overgeslagen (duplicaat): {skipped_count}")
|
||||
if error_count:
|
||||
print(f"✗ Fouten: {error_count}")
|
||||
|
||||
return saved_count
|
||||
|
||||
def import_file(self, filepath):
|
||||
"""Import CSV file into database"""
|
||||
print(f"\n=== Import gestart: {filepath} ===\n")
|
||||
|
||||
print(f"\n=== Import gestart: {filepath} ===")
|
||||
|
||||
line_count = 0
|
||||
tx_start_count = 0
|
||||
tx_stop_count = 0
|
||||
tx_saved_count = 0
|
||||
|
||||
|
||||
try:
|
||||
# Phase 1: Scan file and collect all starts and stops
|
||||
print("\nFase 1: CSV scannen...")
|
||||
with open(filepath, 'r') as file:
|
||||
for line_num, line in enumerate(file, 1):
|
||||
line = line.strip()
|
||||
|
||||
|
||||
# Skip empty lines and comments
|
||||
if not line or line.startswith('# Generated'):
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
|
||||
|
||||
line_count += 1
|
||||
|
||||
|
||||
# Parse transaction start
|
||||
if line.startswith('txstart2:'):
|
||||
if self.parse_txstart(line):
|
||||
tx_start_count += 1
|
||||
|
||||
self.parse_txstart(line)
|
||||
|
||||
# Parse transaction stop
|
||||
elif line.startswith('txstop2:'):
|
||||
if self.parse_txstop(line):
|
||||
tx_stop_count += 1
|
||||
tx_saved_count += 1
|
||||
|
||||
# Skip meter values (mv:) - niet opgeslagen in deze tabel
|
||||
self.parse_txstop(line)
|
||||
|
||||
# Skip meter values (mv:)
|
||||
elif line.startswith('mv:'):
|
||||
continue
|
||||
|
||||
print(f"\n=== Import voltooid ===")
|
||||
print(f"✓ Totaal regels verwerkt: {line_count}")
|
||||
print(f"✓ Transacties gestart: {tx_start_count}")
|
||||
print(f"✓ Transacties gestopt: {tx_stop_count}")
|
||||
print(f"✓ Transacties opgeslagen: {tx_saved_count}")
|
||||
|
||||
# Check for incomplete transactions
|
||||
if self.pending_transactions:
|
||||
print(f"\n⚠ Let op: {len(self.pending_transactions)} transactie(s) nog actief (geen stop gevonden):")
|
||||
for tx_id in self.pending_transactions:
|
||||
print(f" - {tx_id}")
|
||||
|
||||
|
||||
print(f"✓ {line_count} regels gescand")
|
||||
print(f"✓ {len(self.starts)} starts gevonden")
|
||||
print(f"✓ {len(self.stops)} stops gevonden")
|
||||
|
||||
# Phase 2: Match starts with stops
|
||||
matched_transactions = self.match_transactions()
|
||||
|
||||
# Phase 3: Save to database
|
||||
saved_count = self.save_transactions(matched_transactions)
|
||||
|
||||
# Show summary statistics
|
||||
self.show_statistics()
|
||||
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"✗ Bestand niet gevonden: {filepath}")
|
||||
sys.exit(1)
|
||||
|
||||
Reference in New Issue
Block a user