WIP script for importing BEV adressregister

.gitignore

@@ -6,10 +6,13 @@ composer.lock
 nbproject
 config/config.php
 scripts/import
+scripts/addressdb/import
 todo
 files/*
 vendor/
 
+*.sql
+
 # Packages #
 ############
 # it's better to unpack these files and commit the raw source

application/Preorder/PreorderController.php

@@ -57,6 +57,15 @@ class PreorderController extends mfBaseController {
   private function getPreparedFilter($filter) {
     $new_filter = [];
     
+    if(array_key_exists("show_deleted", $filter)) {
+      if($filter['show_deleted'] == "show") {
+        $new_filter['deleted'] = null;
+      }
+      if($filter['show_deleted'] == "only") {
+        $new_filter['deleted'] = 1;
+      }
+    }
+    
     if(is_array($filter) && count($filter)) {
         foreach($filter as $name => $value) {
         $new_filter[$name] = $value;

scripts/addressdb/addressdb_import_bev_adressregister.php

@@ -0,0 +1,240 @@
+#!/usr/bin/php
+<?php
+
+//require 'vendor/autoload.php';
+require("../../config/config.php");
+
+define('FRONKDB_SQLDEBUG',false);
+error_reporting(E_ALL & ~(E_NOTICE | E_STRICT | E_DEPRECATED));
+
+require_once(LIBDIR."/mvcfronk/mfRouter/mfRouter.php");
+require_once(LIBDIR."/mvcfronk/mfBase/mfBaseModel.php");
+require_once(LIBDIR."/mvcfronk/mfBase/mfBaseController.php");
+
+$me = new User(1);
+
+$folder = BASEDIR."/scripts/addressdb/import/bev";
+$db = FronkDB::singleton(ADDRESSDB_DBHOST, ADDRESSDB_DBUSER, ADDRESSDB_DBPASS, ADDRESSDB_DBNAME);
+$log = mfLoghandler::singleton();
+
+/*
+ * Import Gemeinden
+ */
+
+$filename = "$folder/GEMEINDE.csv";
+$input = fopen($filename, "r");
+
+$i = 0;
+while($csv = fgetcsv($input, 0, ";")) {
+  $i++;
+  if($i == 1) continue;
+  
+  if(!trim($csv[0])) {
+    continue;
+  } else {
+    $gkz = trim($csv[0]);
+    $gemeinde_name = trim($csv[1]);
+    
+    $gemeinde = getGemeinde($gkz);
+    if(!$gemeinde) {
+      // create Gemeinde
+      $gemeinde = ADBGemeindeModel::create([
+        'kennziffer' => $gkz,
+        'code' => $gkz,
+        'name' => $gemeinde_name
+      ]);
+      $gemeinde_id = $gemeinde->save();
+      if(!$gemeinde_id) {
+        die("Fehler beim Speichern der Gemeinde ($gkz, $gemeinde_name)\n");
+      }
+    }
+    
+  }
+  
+}
+
+echo ($i-1)." Gemeinden importiert\n";
+
+/*
+ * Import Ortschaften
+ */
+
+$filename = "$folder/ORTSCHAFT.csv";
+$input = fopen($filename, "r");
+
+$i = 0;
+while($csv = fgetcsv($input, 0, ";")) {
+  $i++;
+  if($i == 1) continue;
+  
+  if(!trim($csv[0])) {
+    continue;
+  } else {
+    $gkz = trim($csv[0]);
+    $okz = trim($csv[1]);
+    $ortschaft_name = trim($csv[2]);
+    
+    $gemeinde = getGemeinde($gkz);
+    if(!$gemeinde) {
+      die("Kann Ortschaft nicht importieren: Gemeinde nicht gefunden ($gkz)\n");
+    }
+    
+    $ortschaft = ADBOrtschaftModel::getFirst(["kennziffer" => $okz]);
+    if(!$ortschaft) {
+      $ortschaft = ADBOrtschaftModel::create([
+        'gemeinde_id' => $gemeinde->id,
+        'kennziffer' => $okz,
+        'name' => $ortschaft_name
+      ]);
+      $ortschaft_id = $ortschaft->save();
+      if(!$ortschaft_id) {
+        die("Fehler beim Speichern der Ortschaft ($okz, $ortschaft_name)\n");
+      }
+    }
+  }
+  
+}
+
+echo ($i-1)." Ortschaften importiert\n";
+
+/*
+ * Import Strassen
+ */
+
+$filename = "$folder/STRASSE.csv";
+$input = fopen($filename, "r");
+
+$i = 0;
+while($csv = fgetcsv($input, 0, ";")) {
+  $i++;
+  if($i == 1) continue;
+  
+  if($i % 1000 == 0) {
+    echo ".";
+  }
+  
+  if(!trim($csv[0])) {
+    continue;
+  } else {
+    $skz = trim($csv[0]);
+    $strasse_name = trim($csv[1]);
+    $gkz = trim($csv[4]);
+
+    if(!$skz) continue;
+    
+    $gemeinde = getGemeinde($gkz);
+    if(!$gemeinde) {
+      die("Kann Strasse nicht importieren: Gemeinde nicht gefunden ($gkz)\n");
+    }
+    
+    $strasse = ADBStrasseModel::getFirst(["kennziffer" => $skz]);
+    if(!$strasse) {
+      if($gemeinde->id == 1) {
+        // Adresse in Premstaetten
+        // try to find strasse by fuzzy name
+        $strasse_search = [$db->escape($strasse_name)];
+
+        if(strpos($strasse_name, ' ') !== false) $strasse_search[] = $db->escape(str_replace(' ', '-', $strasse_name));
+        if(strpos($strasse_name, '-') !== false) $strasse_search[] = $db->escape(str_replace('-', ' ', $strasse_name));
+        if(strpos($strasse_name, '.') !== false) $strasse_search[] = $db->escape(str_replace('.', '. ', $strasse_name));
+        if(strpos($strasse_name, '.') !== false) $strasse_search[] = $db->escape(str_replace('.', '.-', $strasse_name));
+        if(strpos($strasse_name, '. ') !== false) $strasse_search[] = $db->escape(str_replace('. ', '.', $strasse_name));
+        if(strpos($strasse_name, '. ') !== false) $strasse_search[] = $db->escape(str_replace('. ', '.-', $strasse_name));
+        if(strpos($strasse_name, '.') !== false) $strasse_search[] = $db->escape(str_replace('.-', '.', $strasse_name));
+        if(strpos($strasse_name, '.') !== false) $strasse_search[] = $db->escape(str_replace('.-', '. ', $strasse_name));
+
+        foreach($strasse_search as $search) {
+          if(strpos($search, 'ß') !== false) $strasse_search[] = $db->escape(str_replace('ß', 'ss', $search));
+          if(strpos($search, 'ä') !== false) $strasse_search[] = $db->escape(str_replace('ä', 'ae', $search));
+          if(strpos($search, 'ö') !== false) $strasse_search[] = $db->escape(str_replace('ö', 'oe', $search));
+          if(strpos($search, 'ü') !== false) $strasse_search[] = $db->escape(str_replace('ü', 'ue', $search));
+
+          if(strpos($search, 'ss') !== false) $strasse_search[] = $db->escape(str_replace('ss', 'ß', $search));
+          if(strpos($search, 'ae') !== false) $strasse_search[] = $db->escape(str_replace('ae', 'ä', $search));
+          if(strpos($search, 'oe') !== false) $strasse_search[] = $db->escape(str_replace('oe', 'ö', $search));
+          if(strpos($search, 'ue') !== false) $strasse_search[] = $db->escape(str_replace('ue', 'ü', $search));
+        }
+
+
+        $strasse_ids = [];
+
+        $sql = "SELECT * FROM Strasse WHERE name IN ('". implode("', '", $strasse_search)."') AND gemeinde_id=".$gemeinde->id;
+        //echo $sql."\n";
+        $res = $db->query($sql);
+        if($db->num_rows($res) > 1) {
+          echo "Found multiple streets for '$strasse_name' in Gemeinde ".$gemeinde->name." (".$gemeinde->id.")\n";
+          while($data = $db->fetch_object($res)) {
+            $strasse_ids[] = $data->id;
+          }
+        } elseif($db->num_rows($res)) {
+          $data = $db->fetch_object($res);
+          $strasse_ids[] = $data->id;
+          //echo "Found matching street for '$strasse_name': ".$data->name."\n";
+        }
+
+        if(count($strasse_ids)) {
+          // update skz for eisting streets
+          foreach($strasse_ids as $strasse_id) {
+            $strasse = new ADBStrasse($strasse_id);
+            $strasse->kennziffer = $skz;
+
+            if($strasse_name != $strasse->name) {
+              echo "Updating strasse name from '".$strasse->name."' to '$strasse_name' (".$strasse->id.")\n";
+              $strasse->name = $strasse_name;
+            }
+
+            if(!$strasse->save()) {
+              echo "Fehler beim Speichern der Strase ($skz, $strasse_name, ".$gemeinde->name.")\n";
+            }
+          }
+        } else {
+          // create strasse
+
+          $strasse = ADBStrasseModel::create([
+            'gemeinde_id' => $gemeinde->id,
+            'kennziffer' => $skz,
+            'name' => $strasse_name
+          ]);
+          $strasse_id = $strasse->save();
+          if(!$strasse_id) {
+            die("Fehler beim Speichern der Ortschaft ($okz, $ortschaft_name)\n");
+          }
+        }
+      } else {
+        // nicht in premstätten
+        // create strasse
+        
+        $strasse = ADBStrasseModel::create([
+          'gemeinde_id' => $gemeinde->id,
+          'kennziffer' => $skz,
+          'name' => $strasse_name
+        ]);
+        $strasse_id = $strasse->save();
+        if(!$strasse_id) {
+          die("Fehler beim Speichern der Ortschaft ($okz, $ortschaft_name)\n");
+        }
+      }
+      
+    }
+  }
+  
+}
+
+echo "\n".($i-1)." Straßen importiert\n";
+
+
+$gemeinden_cache = [];
+function getGemeinde($kz) {
+  global $gemeinden_cache;
+  
+  if(array_key_exists($kz, $gemeinden_cache)) {
+    return $gemeinden_cache[$kz];
+  }
+  
+  $gemeinde = ADBGemeindeModel::getFirst(['kennziffer' => $kz]);
+  if(!$gemeinde) {
+    return false;
+  }
+  $gemeinden_cache[$kz] = $gemeinde;
+  return $gemeinde;
+}