diff --git a/application/IpNetwork/IpNetworkModel.php b/application/IpNetwork/IpNetworkModel.php index 6987d57bb..7e8aa84bf 100644 --- a/application/IpNetwork/IpNetworkModel.php +++ b/application/IpNetwork/IpNetworkModel.php @@ -33,10 +33,18 @@ class IpNetworkModel { $sqlConditions[] = " (CONCAT(INET_NTOA(network_address), '/', cidr) LIKE '%{$searchTerm}%' OR `name` LIKE '%{$searchTerm}%' OR `description` LIKE '%{$searchTerm}%') "; } - if (isset($filters['name'])) $sqlConditions[] = Helper::generateFilterCondition($filters['name'], 'name'); - if (isset($filters['description'])) $sqlConditions[] = Helper::generateFilterCondition($filters['description'], 'description'); - if (isset($filters['location'])) $sqlConditions[] = Helper::generateFilterCondition($filters['location'], 'location'); - if (isset($filters['status'])) $sqlConditions[] = Helper::generateFilterCondition($filters['status'], 'status'); + if (!empty($filters['name'])) $sqlConditions[] = Helper::generateFilterCondition($filters['name'], 'name'); + if (!empty($filters['description'])) $sqlConditions[] = Helper::generateFilterCondition($filters['description'], 'description'); + if (!empty($filters['location'])) $sqlConditions[] = Helper::generateFilterCondition($filters['location'], 'location'); + if (!empty($filters['status'])) $sqlConditions[] = Helper::generateFilterCondition($filters['status'], 'status'); + if (isset($filters['children']) && is_array($filters['children'])) { + if (isset($filters['children']['from'])) { + $sqlConditions[] = " (SELECT COUNT(*) FROM `IpNetwork` WHERE `parent_network_id` = main.id) >= " . intval($filters['children']['from']); + } + if (isset($filters['children']['to'])) { + $sqlConditions[] = " (SELECT COUNT(*) FROM `IpNetwork` WHERE `parent_network_id` = main.id) <= " . intval($filters['children']['to']); + } + } if (empty($filters['parent_network_id'])) { $sqlConditions[] = " `parent_network_id` IS NULL "; @@ -44,6 +52,12 @@ class IpNetworkModel { $sqlConditions[] = " `parent_network_id` = " . intval($filters['parent_network_id']) . " "; } + foreach ($sqlConditions as $key => $condition) { + if (strpos($condition, ' AND ') === 0 || strpos($condition, 'AND ') === 0) { + $sqlConditions[$key] = substr($condition, 4); + } + } + return empty($sqlConditions) ? "" : " WHERE " . implode(" AND ", $sqlConditions); } @@ -70,7 +84,6 @@ class IpNetworkModel { " . $orderClause . " " . $limitClause; - $result = $db->query($sql); $rows = []; while ($row = $result->fetch_assoc()) { @@ -101,11 +114,12 @@ class IpNetworkModel { public static function countIpNetworks($filters): int { $db = FronkDB::singleton()->link; - $sql = "SELECT COUNT(*) as `total_rows` FROM `IpNetwork`" . self::getSqlFilter($filters); + $sql = "SELECT COUNT(*) as `total_rows` FROM `IpNetwork` main" . self::getSqlFilter($filters); $result = $db->query($sql); return (int)$result->fetch_assoc()['total_rows']; } + public static function findSuggestions(string $query, int $limit = 10): array { $db = FronkDB::singleton()->link; $query = $db->real_escape_string($query); diff --git a/lib/TTCrudBaseModel/TTCrudBaseModel.php b/lib/TTCrudBaseModel/TTCrudBaseModel.php index 906d5d0c8..c2a97b024 100644 --- a/lib/TTCrudBaseModel/TTCrudBaseModel.php +++ b/lib/TTCrudBaseModel/TTCrudBaseModel.php @@ -96,7 +96,7 @@ class TTCrudBaseModel { } - public static function get($id): TTCrudBaseModel { + public static function get($id) { $db = self::getDB(); $id = $db->real_escape_string($id); $table = self::getFullyQualifiedTable(); @@ -105,6 +105,8 @@ class TTCrudBaseModel { $result = $db->query($sql); // as TTCRudBaseModel is abstract, we need to get the class name of the child class $class = get_called_class(); + // return null if no result is found + if ($result->num_rows === 0) return null; return new $class($result->fetch_assoc()); } diff --git a/scripts/ipnetwork/initial-data.php b/scripts/ipnetwork/initial-data.php index b66e7e77d..f103d06fb 100644 --- a/scripts/ipnetwork/initial-data.php +++ b/scripts/ipnetwork/initial-data.php @@ -4,147 +4,258 @@ * This script parses a wiki text file containing network information and generates an SQL script * to populate the IpNetwork table with a complete, hierarchical representation of the network. * - * @version 1.0 + * It includes on-the-fly data cleaning, hierarchical network detection from headers, robust parent-child logic, + * duplicate prevention, IP address validation, and error logging to 'initial_data_errors.txt'. + * + * @version 7.0 * @author Gemini */ // --- Configuration --- $inputFile = 'combined_wiki.txt'; -$outputFile = 'initial_data.sql'; +$outputFile = 'initial_data.sql'; // Changed: Standardized output filename $dbTableName = 'IpNetwork'; +$errorFile = 'initial_data_errors.txt'; // --- Main Execution --- +$parsing_errors = []; // Initialize error log array + +// Delete old error log if it exists +if (file_exists($errorFile)) { + unlink($errorFile); +} + // Read and parse the wiki file into a structured PHP array -$networkData = parseWikiFile($inputFile); +$networkData = parseWikiFile($inputFile, $parsing_errors); // Generate the SQL script from the parsed data -$sqlScript = generateSqlScript($networkData, $dbTableName); +$sqlScript = generateSqlScript($networkData, $dbTableName, $parsing_errors); // Save the generated SQL script to the output file file_put_contents($outputFile, $sqlScript); -echo "SQL-Skript wurde erfolgreich in '$outputFile' generiert.\n"; +echo "Final SQL script was successfully generated in '$outputFile'\n"; - -/** - * Parses the entire wiki text file and organizes the network data hierarchically. - * - * @param string $filename The path to the wiki text file. - * @return array A structured array containing all network information. - */ -function parseWikiFile(string $filename): array -{ - if (!file_exists($filename)) { - die("Fehler: Eingabedatei '$filename' nicht gefunden.\n"); - } - - $lines = file($filename, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES); - $networks = []; - $currentNetwork = null; - $currentSubnet = null; - - foreach ($lines as $line) { - // Ignore section headers - if (preg_match('/^=====.*=====$/', $line)) { - continue; - } - - // Match network/subnet definitions (e.g., "10.0.0.0/29 stko-tauka" or "10.0.0.1/32 Gateway") - if (preg_match('/^([\d\.]+)\/(\d+)\s*(.*)$/', trim($line), $matches)) { - $ip = $matches[1]; - $cidr = (int)$matches[2]; - $description = trim($matches[3]); - - // Determine if it's a main network or a subnet based on CIDR - if ($cidr < 29) { // Heuristic: Treat larger blocks as potential parent subnets - $currentSubnet = "$ip/$cidr"; - $networks[$currentSubnet] = [ - 'ip' => $ip, - 'cidr' => $cidr, - 'name' => $description, - 'description' => $description, - 'children' => [] - ]; - } else { // Treat smaller blocks and hosts as children - if ($currentSubnet) { - $networks[$currentSubnet]['children'][] = [ - 'ip' => $ip, - 'cidr' => $cidr, - 'name' => $description, - 'description' => $description, - ]; - } - } - } - // Match host definitions (e.g., "10.0.0.2 stko-tauk:stko") - elseif (preg_match('/^([\d\.]+)\s+(.*)$/', trim($line), $matches)) { - $ip = $matches[1]; - $description = trim($matches[2]); - - if ($currentSubnet) { - $networks[$currentSubnet]['children'][] = [ - 'ip' => $ip, - 'cidr' => 32, // Assume /32 for host entries - 'name' => $description, - 'description' => $description, - ]; - } - } - } - - return $networks; +// Save errors if any were found +if (!empty($parsing_errors)) { + $error_log_content = "Parsing process found the following issues:\n\n"; + $error_log_content .= implode("\n", $parsing_errors); + file_put_contents($errorFile, $error_log_content); + echo "Found " . count($parsing_errors) . " issues during processing. See $errorFile for details.\n"; } /** - * Generates the complete SQL script string. + * Parses the wiki text file, cleans data, validates IPs, checks for duplicates, and organizes the network data. + * It now also creates parent network blocks based on file headers. * - * @param array $data The structured network data. - * @param string $tableName The name of the database table. - * @return string The complete SQL script. + * @param string $filename The path to the wiki text file. + * @param array &$errors Array to store logging information. + * @return array A flat list of unique, valid network/host entries. */ -function generateSqlScript(array $data, string $tableName): string +function parseWikiFile(string $filename, array &$errors): array { - // --- SQL Header --- - $sql = "-- SQL-Daten für die Tabelle '$tableName'\n"; - $sql .= "-- Automatisch generiert aus der Wiki-Dokumentation\n\n"; - $sql .= "TRUNCATE TABLE `$tableName`;\n\n"; - $sql .= "-- Eltern-Netzwerke (Top-Level)\n"; + if (!file_exists($filename)) { + die("Error: Input file '$filename' not found.\n"); + } - // --- Insert Top-Level Parent Networks --- + $lines = file($filename, FILE_IGNORE_NEW_LINES); + $entries = []; + $seen_networks = []; // Tracker for duplicates + $pending_description = ''; + $ignore_section = false; + + foreach ($lines as $line_number => $line) { + // 1. Clean up the line from multiple inconsistencies + $cleaned_line = html_entity_decode($line, ENT_QUOTES | ENT_HTML5); + $cleaned_line = str_replace(['\\', ' '], ['', ' '], $cleaned_line); + $cleaned_line = preg_replace('/\[([^\]]+)\]\(mailto:[^\)]+\)/', '$1', $cleaned_line); + $cleaned_line = trim(preg_replace('/\s+/', ' ', $cleaned_line)); + + // 2. Check for section headers (START, END, etc.) + if (str_starts_with($cleaned_line, '=====')) { + $pending_description = ''; // Reset for any new section + $ignore_section = (stripos($line, 'gelöschte IP Netze') !== false); + + // IMPROVEMENT: Specifically parse START headers for network definitions + if (!$ignore_section && preg_match('/^===== START\s+([0-9\.]+)(?:[\/-](\d{1,2}))?\s*(.*?)(?:\.md)?\s*=====$/', $cleaned_line, $header_matches)) { + $ip_from_header = $header_matches[1]; + $cidr_from_header = $header_matches[2] ?? null; + $desc_from_header = trim($header_matches[3]); + + // Heuristic: If CIDR is missing but IP ends in .0, assume it's a /24 network block. + if ($cidr_from_header === null && preg_match('/\.0$/', $ip_from_header)) { + $cidr_from_header = 24; + if (empty($desc_from_header) || strtolower($desc_from_header) === 'linknetze') { + $desc_from_header = "Network Block {$ip_from_header}"; + } + } + + // If a full network/CIDR is defined or inferred from the header, add it as a distinct entry. + if ($cidr_from_header !== null && filter_var($ip_from_header, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + $network_key = "$ip_from_header/$cidr_from_header"; + if (!isset($seen_networks[$network_key])) { + $entries[] = [ + 'ip' => $ip_from_header, + 'cidr' => (int)$cidr_from_header, + 'name' => $desc_from_header, + 'description' => $desc_from_header, + ]; + $seen_networks[$network_key] = true; + } + } + } + + continue; // Header processed, move to next line + } + + if ($ignore_section || empty($cleaned_line)) { + if (empty($cleaned_line)) $pending_description = ''; + continue; + } + + // 3. Handle special HTML table data + if (strpos($line, ']*>([\d\.]+)<\/td>/', $line, $ip_matches); + if (!empty($ip_matches[1])) { + foreach ($ip_matches[1] as $ip_from_table) { + if (filter_var($ip_from_table, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + $entries[] = [ 'ip' => $ip_from_table, 'cidr' => 32, 'name' => 'CGNAT Host', 'description' => 'CGNAT Host from table' ]; + } + } + } + $pending_description = ''; + continue; + } + + // 4. Try to match IP patterns + $is_ip_entry = false; + $entry_data = null; + + if (preg_match('/^([0-9\.]+)\/(\d+)\s*(.*)$/', $cleaned_line, $matches)) { + $ip = $matches[1]; + if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + $errors[] = "Line " . ($line_number + 1) . ": Invalid IPv4 address in network definition: '$ip'. Skipping."; + $is_ip_entry = true; + } else { + $entry_data = ['ip' => $ip, 'cidr' => (int)$matches[2], 'desc' => $matches[3]]; + } + } elseif (preg_match('/^([0-9\.]+)\s*(.*)$/', $cleaned_line, $matches)) { + $ip = $matches[1]; + if (!filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) { + $is_ip_entry = false; + } else { + $entry_data = ['ip' => $ip, 'cidr' => 32, 'desc' => $matches[2]]; + } + } + + if (!$entry_data && preg_match('/^([0-9a-fA-F:]+)\/\d+/', $cleaned_line, $matches)) { + if (filter_var($matches[1], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) { + $errors[] = "Line " . ($line_number + 1) . ": IPv6 address found and ignored: '$cleaned_line'."; + $is_ip_entry = true; + } + } + + if ($entry_data) { + $is_ip_entry = true; + $network_key = "{$entry_data['ip']}/{$entry_data['cidr']}"; + + if (isset($seen_networks[$network_key])) { + $errors[] = "Line " . ($line_number + 1) . ": Duplicate entry in wiki file: '$network_key'. Skipping."; + } else { + $seen_networks[$network_key] = true; + $full_description = trim($pending_description . ' ' . $entry_data['desc']); + + if ($entry_data['cidr'] === 32 && preg_match('/\b(frei|f r e i|reserve[d]?)\b/i', $full_description)) { + // It is a reserved entry, do not add it to the list. + } else { + $entries[] = [ + 'ip' => $entry_data['ip'], + 'cidr' => $entry_data['cidr'], + 'name' => $full_description, + 'description' => $full_description, + ]; + } + } + } + + if ($is_ip_entry) { + $pending_description = ''; + } else { + $pending_description .= (empty($pending_description) ? '' : "\n") . $cleaned_line; + } + } + + return $entries; +} + + +/** + * Generates the complete SQL script string, including additional parent blocks for better grouping. + */ +function generateSqlScript(array $data, string $tableName, array &$errors): string +{ + $sql = "-- SQL Data for table '$tableName'\n"; + $sql .= "-- Automatically generated from the Wiki documentation (Version 7.0)\n\n"; + $sql .= "TRUNCATE TABLE `$tableName`;\n\n"; + $sql .= "-- Parent Networks (Top-Level)\n"; + + $inserted_networks = []; + + // Added more specific parent blocks to better organize the network hierarchy. $parents = [ - ['10.0.0.0', 8, 'Privates Netzwerk Klasse A', 'RFC 1918 privater Adressbereich für große Netzwerke.'], - ['172.16.0.0', 12, 'Privates Netzwerk Klasse B', 'RFC 1918 privater Adressbereich für mittlere Netzwerke.'], - ['192.168.0.0', 16, 'Privates Netzwerk Klasse C', 'RFC 1918 privater Adressbereich für kleine Netzwerke.'], - ['100.64.0.0', 10, 'Carrier-Grade NAT (CGNAT)', 'RFC 6598 Adressbereich für Carrier-Grade NAT.'], - ['5.206.200.0', 21, 'Öffentlicher Netzblock 5.206.200.0/21', 'Öffentlicher IP-Adressbereich.'], - ['185.29.88.0', 22, 'Öffentlicher Netzblock 185.29.88.0/22', 'Öffentlicher IP-Adressbereich.'], - ['193.105.204.0', 22, 'Öffentlicher Netzblock 193.105.204.0/22', 'Öffentlicher IP-Adressbereich.'], - ['193.186.244.0', 22, 'Öffentlicher Netzblock 193.186.244.0/22', 'Öffentlicher IP-Adressbereich.'], - ['45.82.168.0', 22, 'Öffentlicher Netzblock 45.82.168.0/22', 'Öffentlicher IP-Adressbereich.'], - ['46.151.200.0', 21, 'Öffentlicher Netzblock 46.151.200.0/21', 'Öffentlicher IP-Adressbereich.'], - ['91.227.230.0', 22, 'Öffentlicher Netzblock 91.227.230.0/22', 'Öffentlicher IP-Adressbereich.'] + // Private RFC1918 Ranges + ['10.0.0.0', 8, 'Private Network Class A', 'RFC 1918 private address range for large networks.'], + ['172.16.0.0', 12, 'Private Network Class B', 'RFC 1918 private address range for medium networks.'], + ['192.168.0.0', 16, 'Private Network Class C', 'RFC 1918 private address range for small networks.'], + + // Public & Special Ranges + ['100.64.0.0', 10, 'Carrier-Grade NAT (CGNAT)', 'RFC 6598 address range for Carrier-Grade NAT.'], + ['5.206.200.0', 21, 'Public Network Block 5.206.200.0/21', 'Public IP address range.'], + ['45.82.168.0', 22, 'Public Network Block 45.82.168.0/22', 'Public IP address range.'], + ['46.151.200.0', 21, 'Public Network Block 46.151.200.0/21', 'Public IP address range.'], + ['91.227.230.0', 22, 'Public Network Block 91.227.230.0/22', 'Public IP address range.'], + ['91.227.236.0', 22, 'Public Network Block 91.227.236.0/22', 'Public IP address range.'], + ['185.29.88.0', 22, 'Public Network Block 185.29.88.0/22', 'Public IP address range.'], + ['192.254.252.0', 22, 'Public Network Block 192.254.252.0/22', 'Public IP address range.'], + ['193.105.204.0', 22, 'Public Network Block 193.105.204.0/22', 'Public IP address range.'], + ['193.186.244.0', 22, 'Public Network Block 193.186.244.0/22', 'Public IP address range.'], + ['195.69.183.0', 24, 'Public Subnet 195.69.183.0/24', 'Public IP Subnet for KOLMI.'], + ['195.191.252.0', 24, 'Public Subnet 195.191.252.0/24', 'Public IP Subnet for std Konzentrator.'] ]; foreach ($parents as $p) { $sql .= generateInsertStatement($tableName, $p[0], $p[1], 'NULL', 'active', $p[2], $p[3]); + $inserted_networks["{$p[0]}/{$p[1]}"] = true; } - // --- Insert Child Networks and Hosts --- - $sql .= "\n-- Kind-Netzwerke und Hosts aus der Wiki-Dokumentation\n"; - foreach ($data as $subnetKey => $subnet) { - // Insert the subnet itself, linking it to the correct top-level parent - $parentSelect = getParentSelect($subnet['ip']); - $sql .= generateInsertStatement($tableName, $subnet['ip'], $subnet['cidr'], "($parentSelect)", 'active', $subnet['name'], $subnet['description']); - - // Insert all children of this subnet - if (!empty($subnet['children'])) { - $childParentSelect = "SELECT id FROM `$tableName` p WHERE p.network_address = INET_ATON('{$subnet['ip']}') AND p.cidr = {$subnet['cidr']}"; - foreach ($subnet['children'] as $child) { - $sql .= generateInsertStatement($tableName, $child['ip'], $child['cidr'], "($childParentSelect)", 'active', $child['name'], $child['description']); - } + // Sort data to ensure parent networks are inserted before their children. + usort($data, function ($a, $b) { + if ($a['cidr'] != $b['cidr']) { + return $a['cidr'] <=> $b['cidr']; } + return ip2long($a['ip']) <=> ip2long($b['ip']); + }); + + $sql .= "\n-- Networks and Hosts from Wiki Documentation (Hierarchical)\n"; + foreach ($data as $network) { + $network_key = "{$network['ip']}/{$network['cidr']}"; + + if (isset($inserted_networks[$network_key])) { + $errors[] = "Duplicate network detected (already exists as a top-level parent): '$network_key'. Skipping INSERT."; + continue; + } + + // Subquery to find the immediate parent network already in the table + $parentSelect = "(SELECT id FROM `$tableName` p WHERE " . + "INET_ATON('{$network['ip']}') >= p.network_address AND " . + "INET_ATON('{$network['ip']}') < (p.network_address + POWER(2, 32 - p.cidr)) AND " . + "p.cidr < {$network['cidr']} " . + "ORDER BY p.cidr DESC LIMIT 1)"; + + $sql .= generateInsertStatement($tableName, $network['ip'], $network['cidr'], $parentSelect, 'active', $network['name'], $network['description']); + $inserted_networks[$network_key] = true; } return $sql; @@ -152,15 +263,6 @@ function generateSqlScript(array $data, string $tableName): string /** * Generates a single SQL INSERT statement. - * - * @param string $tableName - * @param string $ip - * @param int $cidr - * @param string $parentIdSql SQL string for parent ID (can be 'NULL' or a subquery). - * @param string $status - * @param string $name - * @param string $description - * @return string The generated INSERT statement. */ function generateInsertStatement(string $tableName, string $ip, int $cidr, string $parentIdSql, string $status, string $name, string $description): string { @@ -171,50 +273,4 @@ function generateInsertStatement(string $tableName, string $ip, int $cidr, strin "(INET_ATON('$ip'), $cidr, $parentIdSql, '$status', '$name', '$description', NULL, UNIX_TIMESTAMP(), UNIX_TIMESTAMP());\n"; } -/** - * Determines the correct parent network's SELECT statement based on the IP address. - * - * @param string $ip The IP address of the child network. - * @return string A SQL SELECT statement to find the parent ID. - */ -function getParentSelect(string $ip): string -{ - if (strpos($ip, '10.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('10.0.0.0') AND p.cidr = 8"; - } - if (strpos($ip, '172.16.') === 0 || strpos($ip, '172.17.') === 0 || strpos($ip, '172.18.') === 0 || strpos($ip, '172.19.') === 0 || strpos($ip, '172.2') === 0 || strpos($ip, '172.30.') === 0 || strpos($ip, '172.31.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('172.16.0.0') AND p.cidr = 12"; - } - if (strpos($ip, '192.168.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('192.168.0.0') AND p.cidr = 16"; - } - if (strpos($ip, '100.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('100.64.0.0') AND p.cidr = 10"; - } - if (strpos($ip, '5.206.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('5.206.200.0') AND p.cidr = 21"; - } - if (strpos($ip, '185.29.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('185.29.88.0') AND p.cidr = 22"; - } - if (strpos($ip, '193.105.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('193.105.204.0') AND p.cidr = 22"; - } - if (strpos($ip, '193.186.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('193.186.244.0') AND p.cidr = 22"; - } - if (strpos($ip, '45.82.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('45.82.168.0') AND p.cidr = 22"; - } - if (strpos($ip, '46.151.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('46.151.200.0') AND p.cidr = 21"; - } - if (strpos($ip, '91.227.') === 0) { - return "SELECT id FROM `IpNetwork` p WHERE p.network_address = INET_ATON('91.227.230.0') AND p.cidr = 22"; - } - - // Fallback for networks that don't match a known parent - return 'NULL'; -} - -?> +?> \ No newline at end of file