Merge pull request #187 in ZP/z-push from feature/ZP-230-sql-state-provider to develop

* commit 'd2cd0a1b':
  ZP-230 Remove obsolete include.
  ZP-230 Fixed @throws of GetState() and CleanStates() so it's the same as in the interface.
  ZP-230 Replaced RuntimeException with UnavailableException (newly introduced), changed IStateMachine so GetState() and SetState() can throw this exception, adjusted interface documentation in filestatemachine.
  ZP-230 Removed zpush_ prefixes of the table names and table crate constants, throw FatalExceptions on DB errors, option for getDbh() to rethrow the PDO exception (when checking for tables), removed created_at and updated_at fields from the users table, removed other methods not part of the interface (GetUserDevicePermission, GetMappedUsername, MapUsername and UnmapUsername - code will be attached to the ticket), renamed and fixed CheckTablesHaveData so it indicates if there is data in any table, included sql stuff to composer autoload, tweaked the output of the migration script, don't log GetStateHash() as it logs a lot.
  ZP-230 Deleted unnecessary mysql folder and mysql.sql file.
  ZP-230 Fix hierarchy folder data before starting the migration. Fixed typo in output.
  ZP-230 Removed log statement.
  ZP-230 Cache prepared SQL statement for GetStateHash calls.
  ZP-230 Create a PDO instance once and reuse it in later calls.
  ZP-230 Made dsn instance variable instead of a local variable in every function.
  ZP-230 Capitalised SQL key words in statements.
  ZP-230 Capitalised SQL key words in statements. Reviewed log statements. Removed unnecessary sprintf calls. Fixed typos.
  ZP-230 Fixed update statement in SetState. Null values weren't handled properly.
  ZP-230 Script to migrate file states to database states. Changes to sqlstatemachine.php to work better with the migration script.
  ZP-230 Changed config to use separate parameters for the engine, server, port and database. Changed the code to match that. Added checks for the database and table creation if they not exist. Fixed comments.
  ZP-230 Use __construct instead of PHP 4 style constructor for PHP 7 compatibility.
  ZP-230 move sqlstatemachine.php to backend/sqlstatemachine folder, move SQL configuration into config.php of sqlstatemachine, update references and copyright year.
  ZP-230 Create SQL state provider. Released under the Affero GNU General Public License (AGPL) version 3.

src/backend/sqlstatemachine/config.php

+<?php
+/***********************************************
+* File      :   sqlstatemachine/config.php
+* Project   :   Z-Push
+* Descr     :   configuration file for the
+*               SqlStateMachine backend.
+*
+* Created   :   19.01.2016
+*
+* Copyright 2007-2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+/**
+ * More information about the configuration on https://wiki.z-hub.io/x/xIAa
+ *
+ * STATE_SQL_ENGINE:    the DB engine
+ * STATE_SQL_SERVER:    the DB server URI or IP
+ * STATE_SQL_PORT:      the DB server port
+ * STATE_SQL_DATABASE:  the DB name
+ * STATE_SQL_USER:      username to DB
+ * STATE_SQL_PASSWORD:  STATE_SQL_USER's password to DB
+ * STATE_SQL_OPTIONS:   array with options needed
+ */
+define('STATE_SQL_ENGINE', 'mysql');
+define('STATE_SQL_SERVER', 'localhost');
+define('STATE_SQL_PORT', '3306');
+define('STATE_SQL_DATABASE', 'zpush');
+define('STATE_SQL_USER', 'root');
+define('STATE_SQL_PASSWORD', '');
+define('STATE_SQL_OPTIONS', serialize(array(PDO::ATTR_PERSISTENT => true)));
+

src/config.php

@@ -71,8 +71,14 @@
     define('USE_FULLEMAIL_FOR_LOGIN', true);
 
 /**********************************************************************************
- *  Default State settings
+ * StateMachine setting
+ *
+ * These StateMachines can be used:
+ *   FILE  - FileStateMachine (default). Needs STATE_DIR set as well.
+ *   SQL   - SqlStateMachine has own configuration file. STATE_DIR is ignored.
+ *           State migration script is available, more informations: https://wiki.z-hub.io/x/xIAa
  */
+    define('STATE_MACHINE', 'FILE');
     define('STATE_DIR', '/var/lib/z-push/');
 
 /**********************************************************************************

src/lib/core/zpush.php

@@ -410,7 +410,12 @@ class ZPush {
             }
             else {
                 // Initialize the default StateMachine
-                ZPush::$stateMachine = new FileStateMachine();
+                if (defined('STATE_MACHINE') && STATE_MACHINE == 'SQL') {
+                    ZPush::$stateMachine = new SqlStateMachine();
+                }
+                else {
+                    ZPush::$stateMachine = new FileStateMachine();
+                }
             }
 
             if (ZPush::$stateMachine->GetStateVersion() !== ZPush::GetLatestStateVersion()) {

src/lib/default/filestatemachine.php

@@ -117,7 +117,7 @@ class FileStateMachine implements IStateMachine {
      *
      * @access public
      * @return mixed
-     * @throws StateNotFoundException, StateInvalidException
+     * @throws StateNotFoundException, StateInvalidException, UnavailableException
      */
     public function GetState($devid, $type, $key = false, $counter = false, $cleanstates = true) {
         if ($counter && $cleanstates)
@@ -148,7 +148,7 @@ class FileStateMachine implements IStateMachine {
      *
      * @access public
      * @return boolean
-     * @throws StateInvalidException
+     * @throws StateInvalidException, UnavailableException
      */
     public function SetState($state, $devid, $type, $key = false, $counter = false) {
         $state = serialize($state);

src/lib/exceptions/unavailableexception.php

+<?php
+/***********************************************
+* File      :   unavailableexception.php
+* Project   :   Z-Push
+* Descr     :   This is a fatal exception when e.g. a subsystem is not
+*               available. The mobile should retry later.
+*
+* Created   :   11.05.2016
+*
+* Copyright 2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+class UnavailableException extends FatalException {}
\ No newline at end of file

src/lib/interface/istatemachine.php

@@ -99,7 +99,7 @@ interface IStateMachine {
      *
      * @access public
      * @return mixed
-     * @throws StateNotFoundException, StateInvalidException
+     * @throws StateNotFoundException, StateInvalidException, UnavailableException
      */
     public function GetState($devid, $type, $key = false, $counter = false, $cleanstates = true);
 
@@ -114,7 +114,7 @@ interface IStateMachine {
      *
      * @access public
      * @return boolean
-     * @throws StateInvalidException
+     * @throws StateInvalidException, UnavailableException
      */
     public function SetState($state, $devid, $type, $key = false, $counter = false);
 

src/vendor/composer/autoload_classmap.php

@@ -94,6 +94,7 @@ return array(
     'SendMail' => $baseDir . '/lib/request/sendmail.php',
     'Settings' => $baseDir . '/lib/request/settings.php',
     'SimpleMutex' => $baseDir . '/lib/default/simplemutex.php',
+    'SqlStateMachine' => $baseDir . '/backend/sqlstatemachine/sqlstatemachine.php',
     'StateInvalidException' => $baseDir . '/lib/exceptions/stateinvalidexception.php',
     'StateManager' => $baseDir . '/lib/core/statemanager.php',
     'StateNotFoundException' => $baseDir . '/lib/exceptions/statenotfoundexception.php',
@@ -146,6 +147,7 @@ return array(
     'TaskRequest' => $baseDir . '/backend/zarafa/mapi/class.taskrequest.php',
     'TimezoneUtil' => $baseDir . '/lib/utils/timezoneutil.php',
     'TopCollector' => $baseDir . '/lib/core/topcollector.php',
+    'UnavailableException' => $baseDir . '/lib/exceptions/unavailableexception.php',
     'Utils' => $baseDir . '/lib/utils/utils.php',
     'ValidateCert' => $baseDir . '/lib/request/validatecert.php',
     'WBXMLDecoder' => $baseDir . '/lib/wbxml/wbxmldecoder.php',

src/vendor/composer/autoload_static.php

@@ -101,6 +101,7 @@ class ComposerStaticInitd6749fc2fb9944bbe86b2b7d79a7852f
         'SendMail' => __DIR__ . '/../..' . '/lib/request/sendmail.php',
         'Settings' => __DIR__ . '/../..' . '/lib/request/settings.php',
         'SimpleMutex' => __DIR__ . '/../..' . '/lib/default/simplemutex.php',
+        'SqlStateMachine' => __DIR__ . '/../..' . '/backend/sqlstatemachine/sqlstatemachine.php',
         'StateInvalidException' => __DIR__ . '/../..' . '/lib/exceptions/stateinvalidexception.php',
         'StateManager' => __DIR__ . '/../..' . '/lib/core/statemanager.php',
         'StateNotFoundException' => __DIR__ . '/../..' . '/lib/exceptions/statenotfoundexception.php',
@@ -153,6 +154,7 @@ class ComposerStaticInitd6749fc2fb9944bbe86b2b7d79a7852f
         'TaskRequest' => __DIR__ . '/../..' . '/backend/zarafa/mapi/class.taskrequest.php',
         'TimezoneUtil' => __DIR__ . '/../..' . '/lib/utils/timezoneutil.php',
         'TopCollector' => __DIR__ . '/../..' . '/lib/core/topcollector.php',
+        'UnavailableException' => __DIR__ . '/../..' . '/lib/exceptions/unavailableexception.php',
         'Utils' => __DIR__ . '/../..' . '/lib/utils/utils.php',
         'ValidateCert' => __DIR__ . '/../..' . '/lib/request/validatecert.php',
         'WBXMLDecoder' => __DIR__ . '/../..' . '/lib/wbxml/wbxmldecoder.php',

tools/migrate-filestates-to-db.php

+#!/usr/bin/php
+<?php
+/**********************************************************
+* File      :   migrate-filestates-to-db.php
+* Project   :   Z-Push - tools
+* Descr     :   Copies file states to the database
+*
+* Created   :   12.02.2016
+*
+* Copyright 2007 - 2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+// Please adjust to match your z-push installation directory, usually /usr/share/z-push
+define('ZPUSH_BASE_PATH', "/usr/share/z-push");
+
+/************************************************
+ * MAIN
+ */
+print("Starting the filestate to database migration script." . PHP_EOL);
+
+try {
+    if (php_sapi_name() != "cli") {
+        die("This script can only be called from the CLI.");
+    }
+
+    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
+        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
+    }
+
+    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
+    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);
+
+    require_once 'vendor/autoload.php';
+
+    if (!defined('ZPUSH_CONFIG')) define('ZPUSH_CONFIG', 'config.php');
+    include_once(ZPUSH_CONFIG);
+
+
+    if (STATE_MACHINE != 'FILE') {
+        die(sprintf("Only migration from 'FILE' StateMachine is possible. Your STATE_MACHINE setting is '%s'. Please modify your config.php.%s", STATE_MACHINE, PHP_EOL));
+    }
+
+    printf("Also check the logfile at %s for more information.%s", LOGFILE, PHP_EOL);
+
+    ZPush::CheckConfig();
+    $migrate = new StateMigratorFileToDB();
+
+    if (!$migrate->MigrationNecessary()) {
+        exit(1);
+    }
+
+    $migrate->DoMigration();
+}
+catch (ZPushException $zpe) {
+    die(get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+}
+
+sprintf("terminated%s", PHP_EOL);
+
+class StateMigratorFileToDB {
+    private $fsm;
+    private $dbsm;
+
+
+    /**
+     * Check if the migration is necessary.
+     *
+     * @access public
+     * @throws FatalMisconfigurationException
+     * @throws FatalNotImplementedException
+     * @return boolean
+     */
+    public function MigrationNecessary() {
+        print("StateMigratorFileToDB->MigrationNecessary(): checking if migration is necessary." . PHP_EOL);
+        try {
+            $this->dbsm = new SqlStateMachine();
+            if ($this->dbsm->DoTablesHaveData()) {
+                print ("Tables already have data. Migration aborted. Drop database or truncate tables and try again." . PHP_EOL);
+                return false;
+            }
+        }
+        catch (ZPushException $ex) {
+            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        }
+        return true;
+    }
+
+    /**
+     * Execute the migration.
+     *
+     * @access public
+     * @return true
+     */
+    public function DoMigration() {
+        print("StateMigratorFileToDB->DoMigration(): Starting migration routine." . PHP_EOL);
+        $starttime = time();
+        $deviceCount = 0;
+        $stateCount = 0;
+        try {
+            // Fix hierarchy folder data before starting the migration
+            ZPushAdmin::FixStatesHierarchyFolderData();
+
+            $this->fsm = new FileStateMachine();
+
+            if (!($this->fsm  instanceof FileStateMachine)) {
+                throw new FatalNotImplementedException("This conversion script is only able to convert states from the FileStateMachine");
+            }
+
+            // get all state information for all devices
+            $alldevices = $this->fsm->GetAllDevices(false);
+            foreach ($alldevices as $devid) {
+                $deviceCount++;
+                $lowerDevid = strtolower($devid);
+
+                $allStates = $this->fsm->GetAllStatesForDevice($lowerDevid);
+                printf("Processing device: %s with %s states\t", str_pad($devid,35), str_pad(count($allStates), 4, ' ',STR_PAD_LEFT));
+                $migrated = 0;
+                foreach ($allStates as $stateInfo) {
+                    $state = $this->fsm->GetState($lowerDevid, $stateInfo['type'], $stateInfo['uuid'], (int) $stateInfo['counter'], false);
+                    $this->dbsm->SetState($state, $lowerDevid, $stateInfo['type'], (empty($stateInfo['uuid']) ? NULL : $stateInfo['uuid']), (int) $stateInfo['counter']);
+                    $migrated++;
+                }
+
+                // link devices to users
+                $devState = $this->fsm->GetState($lowerDevid, IStateMachine::DEVICEDATA);
+                foreach ($devState->devices as $user => $dev) {
+                    $this->dbsm->LinkUserDevice($user, $dev->deviceid);
+                }
+
+                print(" completed migration of $migrated states" . PHP_EOL);
+                $stateCount += $migrated;
+            }
+        }
+        catch (ZPushException $ex) {
+            print (PHP_EOL . "Something went wrong during the migration. The script will now exit." . PHP_EOL);
+            die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        }
+        $timeSpent = gmdate("H:i:s", time() - $starttime);
+        printf(PHP_EOL ."StateMigratorFileToDB->DoMigration(): Migration completed successfuly. Migrated %d devices with %d states in %s.".PHP_EOL.PHP_EOL, $deviceCount, $stateCount, $timeSpent);
+    }
+}
\ No newline at end of file