ZP-230 Script to migrate file states to database states. Changes to

sqlstatemachine.php to work better with the migration script.

Released under the Affero GNU General Public License (AGPL) version 3.

src/backend/sqlstatemachine/sqlstatemachine.php

@@ -55,7 +55,7 @@ class SqlStateMachine implements IStateMachine {
     const UNKNOWNDATABASE = 1049;
     const CREATETABLE_ZPUSH_SETTINGS = "CREATE TABLE IF NOT EXISTS zpush_settings (key_name VARCHAR(50) NOT NULL, key_value VARCHAR(50) NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY (key_name));";
     const CREATETABLE_ZPUSH_USERS = "CREATE TABLE IF NOT EXISTS zpush_users (username VARCHAR(50) NOT NULL, device_id VARCHAR(50) NOT NULL, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY (username, device_id));";
-    const CREATETABLE_ZPUSH_STATES = "CREATE TABLE IF NOT EXISTS zpush_states (id_state INTEGER AUTO_INCREMENT, device_id VARCHAR(50) NOT NULL, uuid VARCHAR(50), state_type VARCHAR(50), counter INTEGER, state_data MEDIUMBLOB, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY (id_state));";
+    const CREATETABLE_ZPUSH_STATES = "CREATE TABLE IF NOT EXISTS zpush_states (id_state INTEGER AUTO_INCREMENT, device_id VARCHAR(50) NOT NULL, uuid VARCHAR(50) NULL, state_type VARCHAR(50), counter INTEGER, state_data MEDIUMBLOB, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL, PRIMARY KEY (id_state));";
     const CREATEINDEX_ZPUSH_STATES = "CREATE UNIQUE INDEX idx_zpush_states_unique ON zpush_states (device_id, uuid, state_type, counter);";
 
     private $dbh;
@@ -107,10 +107,10 @@ class SqlStateMachine implements IStateMachine {
      * @return string
      * @throws StateNotFoundException, StateInvalidException
      */
-    public function GetStateHash($devid, $type, $key = false, $counter = false) {
+    public function GetStateHash($devid, $type, $key = null, $counter = false) {
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("SqlStateMachine->GetStateHash(): '%s', '%s', '%s', '%s'", $devid, $type, $key, $counter));
 
-        $sql = "select updated_at from zpush_states where device_id = :devid and state_type = :type and uuid = :key and counter = :counter";
+        $sql = "select updated_at from zpush_states where device_id = :devid and state_type = :type and uuid ". (($key == null) ? " is " : " = ") . ":key and counter = :counter";
         $params = $this->getParams($devid, $type, $key, $counter);
 
         $hash = null;
@@ -161,12 +161,12 @@ class SqlStateMachine implements IStateMachine {
      * @return mixed
      * @throws StateNotFoundException, StateInvalidException
      */
-    public function GetState($devid, $type, $key = false, $counter = false, $cleanstates = true) {
+    public function GetState($devid, $type, $key = null, $counter = false, $cleanstates = true) {
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("SqlStateMachine->GetState(): '%s', '%s', '%s', '%s', '%s'", $devid, $type, $key, $counter, $cleanstates));
         if ($counter && $cleanstates)
             $this->CleanStates($devid, $type, $key, $counter);
 
-        $sql = "select state_data from zpush_states where device_id = :devid and state_type = :type and uuid = :key and counter = :counter";
+        $sql = "select state_data from zpush_states where device_id = :devid and state_type = :type and uuid ". (($key == null) ? " is " : " = ") . ":key and counter = :counter";
         $params = $this->getParams($devid, $type, $key, $counter);
 
         $data = null;
@@ -220,10 +220,10 @@ class SqlStateMachine implements IStateMachine {
      * @return boolean
      * @throws StateInvalidException
      */
-    public function SetState($state, $devid, $type, $key = false, $counter = false) {
+    public function SetState($state, $devid, $type, $key = null, $counter = false) {
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("SqlStateMachine->SetState(): '%s', '%s', '%s', '%s'", $devid, $type, $key, $counter));
 
-        $sql = "select device_id from zpush_states where device_id = :devid and state_type = :type and uuid = :key and counter = :counter";
+        $sql = "select device_id from zpush_states where device_id = :devid and state_type = :type and uuid ". (($key == null) ? " is " : " = ") . ":key and counter = :counter";
         $params = $this->getParams($devid, $type, $key, $counter);
 
         $sth = null;
@@ -255,7 +255,7 @@ class SqlStateMachine implements IStateMachine {
             $sth->bindParam(":devid", $devid, PDO::PARAM_STR);
             $sth->bindParam(":type", $type, PDO::PARAM_STR);
             $sth->bindParam(":key", $key, PDO::PARAM_STR);
-            $sth->bindValue(":counter", ($counter === false ? -1 : $counter), PDO::PARAM_INT);
+            $sth->bindValue(":counter", ($counter === false ? 0 : $counter), PDO::PARAM_INT);
             $sth->bindValue(":data", serialize($state), PDO::PARAM_LOB);
             $sth->bindValue(":updated_at", $this->getNow(), PDO::PARAM_STR);
 
@@ -328,7 +328,7 @@ class SqlStateMachine implements IStateMachine {
      * @access public
      * @return boolean     indicating if the user was added or not (existed already)
      */
-    public function LinkUserDevice($username, $devid) {
+    public function LinkUserDevice($username, $devid, $createdAt = null, $updatedAt = null) {
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("SqlStateMachine->LinkUserDevice(): '%s', '%s'", $username, $devid));
 
         $sth = null;
@@ -351,7 +351,8 @@ class SqlStateMachine implements IStateMachine {
             else {
                 $sth = null;
                 $sql = "insert into zpush_users (username, device_id, created_at, updated_at) values (:username, :devid, :created_at, :updated_at)";
-                $params[":created_at"] = $params[":updated_at"] = $this->getNow();
+                $params[":created_at"] = ($createdAt != null) ? $createdAt : $this->getNow();
+                $params[":updated_at"] = ($updatedAt != null) ? $updatedAt : $this->getNow();
                 $sth = $this->dbh->prepare($sql);
                 if ($sth->execute($params)) {
                     ZLog::Write(LOGLEVEL_DEBUG, sprintf("SqlStateMachine->LinkUserDevice(): Linked user-device: '%s' '%s'", $username, $devid));
@@ -926,7 +927,7 @@ class SqlStateMachine implements IStateMachine {
      * @access private
      */
     private function getParams($devid, $type, $key, $counter) {
-        return array(":devid" => $devid, ":type" => $type, ":key" => $key, ":counter" => ($counter === false ? -1 : $counter) );
+        return array(":devid" => $devid, ":type" => $type, ":key" => $key, ":counter" => ($counter === false ? 0 : $counter) );
     }
 
     /**
@@ -1032,4 +1033,53 @@ class SqlStateMachine implements IStateMachine {
             throw new RuntimeException(sprintf("SqlStateMachine->checkDbAndTables(): PDOException (%s): %s", $ex->getCode(), $ex->getMessage()));
         }
     }
+
+    /**
+     * Checks if state tables have data.
+     *
+     * @access public
+     * @return boolean
+     * @throws RuntimeException
+     */
+    public function checkTablesHaveData() {
+        $dsn = sprintf("%s:host=%s;port=%s;dbname=%s", STATE_SQL_ENGINE, STATE_SQL_SERVER, STATE_SQL_PORT, STATE_SQL_DATABASE);
+        try {
+            $this->dbh = new PDO($dsn, STATE_SQL_USER, STATE_SQL_PASSWORD, $this->options);
+            $sqlStmt = "SELECT key_name FROM zpush_settings LIMIT 1;";
+            $sth = $this->dbh->prepare($sqlStmt);
+            $sth->execute();
+            if ($sth->rowCount() > 0) {
+                print("There is data in zpush_settings table." . PHP_EOL);
+            }
+            else {
+                print("There is no data in zpush_settings table." . PHP_EOL);
+                return false;
+            }
+
+            $sqlStmt = "SELECT id_state FROM zpush_states LIMIT 1;";
+            $sth = $this->dbh->prepare($sqlStmt);
+            $sth->execute();
+            if ($sth->rowCount() > 0) {
+                print("There is data in zpush_states table." . PHP_EOL);
+            }
+            else {
+                print("There is no data in zpush_states table." . PHP_EOL);
+                return false;
+            }
+
+            $sqlStmt = "SELECT username FROM zpush_users LIMIT 1;";
+            $sth = $this->dbh->prepare($sqlStmt);
+            $sth->execute();
+            if ($sth->rowCount() > 0) {
+                print("There is data in zpush_users table." . PHP_EOL);
+                return true;
+            }
+            print("There is no data in zpush_users table." . PHP_EOL);
+            return false;
+        }
+        catch (PDOException $ex) {
+            throw new RuntimeException(sprintf("SqlStateMachine->checkDbAndTables(): PDOException (%s): %s", $ex->getCode(), $ex->getMessage()));
+        }
+        return false;
+    }
 }
\ No newline at end of file

tools/migrate-filestates-to-db.php

+#!/usr/bin/php
+<?php
+/**********************************************************
+* File      :   migrate-filestates-to-db.php
+* Project   :   Z-Push - tools
+* Descr     :   Copies file states to the database
+*
+* Created   :   12.02.2016
+*
+* Copyright 2007 - 2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+// Please adjust to match your z-push installation directory, usually /usr/share/z-push
+define('ZPUSH_BASE_PATH', "/usr/share/z-push");
+
+/************************************************
+ * MAIN
+ */
+print("Starting the filestate to database migration script." . PHP_EOL);
+
+try {
+    if (php_sapi_name() != "cli") {
+        die("This script can only be called from the CLI.");
+    }
+
+    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
+        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
+    }
+
+    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
+    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);
+
+    include_once('lib/core/zpushdefs.php');
+    include_once('lib/core/zpush.php');
+    include_once('lib/core/zlog.php');
+    include_once('lib/core/statemanager.php');
+    include_once('lib/core/stateobject.php');
+    include_once('lib/core/asdevice.php');
+    include_once('lib/interface/ichanges.php');
+    include_once('lib/interface/iexportchanges.php');
+    include_once('lib/interface/iimportchanges.php');
+    include_once('lib/core/hierarchycache.php');
+    include_once('lib/core/interprocessdata.php');
+    include_once('lib/core/syncparameters.php');
+    include_once('lib/core/bodypreference.php');
+    include_once('lib/core/contentparameters.php');
+    include_once('lib/core/changesmemorywrapper.php');
+    include_once('lib/core/streamer.php');
+    include_once('lib/exceptions/exceptions.php');
+    include_once('lib/utils/utils.php');
+    include_once('lib/request/request.php');
+    include_once('lib/request/requestprocessor.php');
+    include_once('lib/interface/ibackend.php');
+    include_once('lib/interface/ichanges.php');
+    include_once('lib/interface/iexportchanges.php');
+    include_once('lib/interface/iimportchanges.php');
+    include_once('lib/interface/isearchprovider.php');
+    include_once('lib/interface/istatemachine.php');
+    include_once('lib/syncobjects/syncobject.php');
+    include_once('lib/syncobjects/syncfolder.php');
+    include_once('config.php');
+    include_once('lib/default/filestatemachine.php');
+    include_once('backend/sqlstatemachine/sqlstatemachine.php');
+
+
+    if (STATE_MACHINE != 'FILE') {
+        die(sprintf("Only migration from 'FILE' StateMachine is possible. Your STATE_MACHINE setting is '%s'. Please modify your config.php.%s", STATE_MACHINE, PHP_EOL));
+    }
+
+    printf("Also check the logfile at %s for more information.%s", LOGFILE, PHP_EOL);
+
+    ZPush::CheckConfig();
+    $migrate = new StateMigratorFileToDB();
+
+    if (!$migrate->MigrationNecessary()) {
+        print("Migration script was run before and eventually no migration is necessary." . PHP_EOL);
+        exit(1);
+    }
+
+    $migrate->DoMigration();
+}
+catch (ZPushException $zpe) {
+    die(get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+}
+
+sprintf("terminated%s", PHP_EOL);
+
+class StateMigratorFileToDB {
+    private $fsm;
+    private $dbsm;
+
+
+    /**
+     * Check if the migration is necessary.
+     *
+     * @access public
+     * @throws FatalMisconfigurationException
+     * @throws FatalNotImplementedException
+     * @return boolean
+     */
+    public function MigrationNecessary() {
+        print("StateMigratorFileToDB->MigrationNecessary(): checking if migration is necessary." . PHP_EOL);
+        try {
+            $this->dbsm = new SqlStateMachine();
+            if ($this->dbsm->checkTablesHaveData()) {
+                print ("All state tables have data already. A migration is not necessary." . PHP_EOL);
+                return false;
+            }
+        }
+        catch (ZPushException $ex) {
+            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        }
+        return true;
+    }
+
+    /**
+     * Execute the migration.
+     *
+     * @access public
+     * @return true
+     */
+    public function DoMigration() {
+        print("StateMigratorFileToDB->DoMigration(): Starting migration routine." . PHP_EOL);
+        try {
+            $this->fsm = new FileStateMachine();
+
+            if (!($this->fsm  instanceof FileStateMachine)) {
+                throw new FatalNotImplementedException("This conversion script is only able to convert states from the FileStateMachine");
+            }
+
+            // get all state information for all devices
+            $alldevices = $this->fsm->GetAllDevices(false);
+            foreach ($alldevices as $devid) {
+                $lowerDevid = strtolower($devid);
+
+                $allStates = $this->fsm->GetAllStatesForDevice($lowerDevid);
+                printf("Processing device: %s with %d states\t", $devid, count($allStates));
+                foreach ($allStates as $stateInfo) {
+                    $state = $this->fsm->GetState($lowerDevid, $stateInfo['type'], $stateInfo['uuid'], (int) $stateInfo['counter'], false);
+                    $this->dbsm->SetState($state, $lowerDevid, $stateInfo['type'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
+                    printf("%s\t\tsetting state:%s-%s-%d", PHP_EOL, $stateInfo['uuid'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
+                }
+
+                // link devices to users
+                $devState = $this->fsm->GetState($lowerDevid, IStateMachine::DEVICEDATA);
+                foreach ($devState->devices as $user => $dev) {
+                    $this->dbsm->LinkUserDevice($user, $dev->deviceid, date("Y-m-d H:i:s", $dev->firstsynctime), date("Y-m-d H:i:s", $dev->lastupdatetime));
+                }
+
+                print("\tcompleted" . PHP_EOL);
+            }
+        }
+        catch (ZPushException $ex) {
+            print (PHP_EOL . "Something went wrong during the migration. The script will now exit." . PHP_EOL);
+            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        }
+        print("StateMigratorFileToDB->DoMigration(): Migration completed successfuly." . PHP_EOL);
+    }
+}
\ No newline at end of file