Merge pull request #394 in ZP/z-push from develop to release/2.3

* commit '016e688b':
  ZP-1073 Use exit() instead of die().
  ZP-1076 Catch StatusException in ZPushAdmin::GetDeviceDetails().
  ZP-1073 list-shared-folders script, incl. usage examples (in file).
  ZP-1073 Refactor Utils::GetFolderOriginFromId() into GetFolderOriginStringFromId() so GetFolderOriginFromId() returns a value comparable to DeviceManager::FLD_ORIGIN_*.
  ZP-1017 Replace z-push.sf.net url with z-push.org.
  ZP-1062 depend on php instead of php7.0. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-1058 autodiscover: Fix mistake: always send response
  ZP-1056 Fail earlier when no password is provided. Log used username before trying to login in the backend. The $username will always be set and doesn't need to be checked again.
  ZP-1056 Fix authorization encoding and also add it to autodiscover. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-1058 autodiscover: Rework exception handling to match z-push core.
  ZP-1058 autodiscover: Print IP to log on failed authentication for usage with e.g. fail2ban

build/deb/debian.control

@@ -8,7 +8,7 @@ Homepage: http://z-push.org
 
 Package: z-push-common
 Architecture: all
-Depends: ${misc:Depends}, php5 (>= 5.4) | php7.0, php5-cli | php7.0-cli, php-soap
+Depends: ${misc:Depends}, php5 (>= 5.4) | php (>= 5.4) , php5-cli | php-cli, php-soap
 Conflicts: d-push, z-push
 Replaces: d-push, z-push
 Description: open source implementation of the ActiveSync protocol
@@ -26,7 +26,7 @@ Description: Z-Push caldav backend
 
 Package: z-push-backend-carddav
 Architecture: all
-Depends: ${misc:Depends}, z-push-common (= ${binary:Version}), php-curl | php5-curl, php5-xsl | php7.0-xsl
+Depends: ${misc:Depends}, z-push-common (= ${binary:Version}), php-curl | php5-curl, php5-xsl | php-xml
 Description: Z-Push carddav backend
  Backend for Z-Push, that adds the ability to connect to a carddav server
 

src/autodiscover/autodiscover.php

@@ -104,25 +104,42 @@ class ZPushAutodiscover {
             }
         }
 
-        catch (AuthenticationRequiredException $ex) {
-            if (isset($incomingXml)) {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover because login failed for user with email '%s'", $incomingXml->Request->EMailAddress));
+        catch (Exception $ex) {
+            // Extract any previous exception message for logging purpose.
+            $exclass = get_class($ex);
+            $exception_message = $ex->getMessage();
+            if($ex->getPrevious()){
+                do {
+                    $current_exception = $ex->getPrevious();
+                    $exception_message .= ' -> ' . $current_exception->getMessage();
+                } while($current_exception->getPrevious());
             }
-            else {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover incorrect request: '%s'", $ex->getMessage()));
+
+            ZLog::Write(LOGLEVEL_FATAL, sprintf('Exception: (%s) - %s', $exclass, $exception_message));
+
+            if ($ex instanceof AuthenticationRequiredException) {
+                if (isset($incomingXml)) {
+                    // log the failed login attemt e.g. for fail2ban
+                    if (defined('LOGAUTHFAIL') && LOGAUTHFAIL != false)
+                        ZLog::Write(LOGLEVEL_WARN, sprintf("Unable to complete autodiscover because login failed for user with email '%s' from IP %s.", $incomingXml->Request->EMailAddress, $_SERVER["REMOTE_ADDR"]));
+                }
+                else {
+                    ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover incorrect request: '%s'", $ex->getMessage()));
+                }
+                http_response_code(401);
+                header('WWW-Authenticate: Basic realm="ZPush"');
             }
-            http_response_code(401);
-            header('WWW-Authenticate: Basic realm="ZPush"');
-        }
-        catch (ZPushException $ex) {
-            ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover because of ZPushException. Error: %s", $ex->getMessage()));
-            if(!headers_sent()) {
-                header('HTTP/1.1 '. $ex->getHTTPCodeString());
-                foreach ($ex->getHTTPHeaders() as $h) {
-                    header($h);
+            else if ($ex instanceof ZPushException) {
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover because of ZPushException. Error: %s", $ex->getMessage()));
+                if(!headers_sent()) {
+                    header('HTTP/1.1 '. $ex->getHTTPCodeString());
+                    foreach ($ex->getHTTPHeaders() as $h) {
+                        header($h);
+                    }
                 }
             }
         }
+
         $this->sendResponse($response);
     }
 
@@ -182,21 +199,32 @@ class ZPushAutodiscover {
      * @return string $username
      */
     private function login($backend, $incomingXml) {
+        // don't even try to login if there is no PW set
+        if (!isset($_SERVER['PHP_AUTH_PW'])) {
+            throw new AuthenticationRequiredException("Access denied. No password provided.");
+        }
+
         // Determine the login name depending on the configuration: complete email address or
         // the local part only.
         if (USE_FULLEMAIL_FOR_LOGIN) {
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the complete email address for login."));
             $username = $incomingXml->Request->EMailAddress;
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the complete email address for login: '%s'", $username));
         }
-        else{
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the username only for login."));
+        else {
             $username = Utils::GetLocalPartFromEmail($incomingXml->Request->EMailAddress);
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the username only for login: '%s'", $username));
         }
 
-        if($backend->Logon($username, "", $_SERVER['PHP_AUTH_PW']) == false) {
+        // Mobile devices send Authorization header using UTF-8 charset. Outlook sends it using ISO-8859-1 encoding.
+        // For the successful authentication the user and password must be UTF-8 encoded. Try to determine which
+        // charset was sent by the client and convert it to UTF-8. See https://jira.z-hub.io/browse/ZP-864.
+        $username = Utils::ConvertAuthorizationToUTF8($username);
+        $password = Utils::ConvertAuthorizationToUTF8($_SERVER['PHP_AUTH_PW']);
+        if ($backend->Logon($username, "", $password) == false) {
             throw new AuthenticationRequiredException("Access denied. Username or password incorrect.");
         }
-        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login() Using '%s' as the username.", $username));
+
+        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login() successfull with '%s' as the username.", $username));
         return $username;
     }
 

src/lib/request/request.php

@@ -254,25 +254,10 @@ class Request {
         // Mobile devices send Authorization header using UTF-8 charset. Outlook sends it using ISO-8859-1 encoding.
         // For the successful authentication the user and password must be UTF-8 encoded. Try to determine which
         // charset was sent by the client and convert it to UTF-8. See https://jira.z-hub.io/browse/ZP-864.
-        if (isset($_SERVER['PHP_AUTH_USER'])) {
-            $encoding = mb_detect_encoding(self::$authUser, "UTF-8, ISO-8859-1");
-            if (!$encoding) {
-                $encoding = mb_detect_encoding(self::$authUser, Utils::GetAvailableCharacterEncodings());
-                if ($encoding) {
-                    ZLog::Write(LOGLEVEL_WARN,
-                            sprintf("Request->ProcessHeaders(): mb_detect_encoding detected '%s' charset. This charset is not in the default detect list. Please report it to Z-Push developers.",
-                                    $encoding));
-                }
-                else {
-                    ZLog::Write(LOGLEVEL_ERROR, "Request->ProcessHeaders(): mb_detect_encoding failed to detect the Authorization header charset. It's possible that user won't be able to login.");
-                }
-            }
-            if ($encoding && strtolower($encoding) != "utf-8") {
-                ZLog::Write(LOGLEVEL_DEBUG, sprintf("Request->ProcessHeaders(): mb_detect_encoding detected '%s' charset. Authorization header will be converted to UTF-8 from it.", $encoding));
-                self::$authUser = mb_convert_encoding(self::$authUser, "UTF-8", $encoding);
-                self::$authPassword = mb_convert_encoding(self::$authPassword, "UTF-8", $encoding);
-            }
-        }
+        if (isset(self::$authUser))
+          self::$authUser = Utils::ConvertAuthorizationToUTF8(self::$authUser);
+        if (isset(self::$authPassword))
+          self::$authPassword = Utils::ConvertAuthorizationToUTF8(self::$authPassword);
     }
 
     /**

src/lib/utils/utils.php

@@ -1092,15 +1092,36 @@ class Utils {
     }
 
     /**
-     * Returns folder origin from its id.
+     * Returns folder origin identifier from its id.
      *
-     * @param string $fid
+     * @param string $folderid
+     *
+     * @access public
+     * @return string|boolean  matches values of DeviceManager::FLD_ORIGIN_*
+     */
+    public static function GetFolderOriginFromId($folderid) {
+        $origin = substr($folderid, 0, 1);
+        switch ($origin) {
+            case DeviceManager::FLD_ORIGIN_CONFIG:
+            case DeviceManager::FLD_ORIGIN_GAB:
+            case DeviceManager::FLD_ORIGIN_SHARED:
+            case DeviceManager::FLD_ORIGIN_USER:
+                return $origin;
+        }
+        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginFromId(): Unknown folder origin for folder with id '%s'", $folderid));
+        return false;
+    }
+
+    /**
+     * Returns folder origin as string from its id.
+     *
+     * @param string $folderid
      *
      * @access public
      * @return string
      */
-    public static function GetFolderOriginFromId($fid) {
-        $origin = substr($fid, 0, 1);
+    public static function GetFolderOriginStringFromId($folderid) {
+        $origin = substr($folderid, 0, 1);
         switch ($origin) {
             case DeviceManager::FLD_ORIGIN_CONFIG:
                 return 'configured';
@@ -1111,7 +1132,7 @@ class Utils {
             case DeviceManager::FLD_ORIGIN_USER:
                 return 'user';
         }
-        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginFromId(): Unknown folder origin for folder with id '%s'", $fid));
+        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginStringFromId(): Unknown folder origin for folder with id '%s'", $folderid));
         return 'unknown';
     }
 
@@ -1130,6 +1151,38 @@ class Utils {
         }
         return array(null, $id);
     }
+
+    /**
+     * Detects encoding of the input and converts it to UTF-8.
+     * This is currently only used for authorization header conversion.
+     *
+     * @param string      $data     input data
+     *
+     * @access public
+     * @return string               utf-8 encoded data
+     */
+    public static function ConvertAuthorizationToUTF8($data) {
+        $encoding = mb_detect_encoding($data, "UTF-8, ISO-8859-1");
+
+        if (!$encoding) {
+            $encoding = mb_detect_encoding($data, Utils::GetAvailableCharacterEncodings());
+            if ($encoding) {
+                ZLog::Write(LOGLEVEL_WARN,
+                        sprintf("Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding detected '%s' charset. This charset is not in the default detect list. Please report it to Z-Push developers.",
+                                $encoding));
+            }
+            else {
+                ZLog::Write(LOGLEVEL_ERROR, "Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding failed to detect the Authorization header charset. It's possible that user won't be able to login.");
+            }
+        }
+
+        if ($encoding && strtolower($encoding) != "utf-8") {
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding detected '%s' charset. Authorization header will be converted to UTF-8 from it.", $encoding));
+            return mb_convert_encoding($data, "UTF-8", $encoding);
+        }
+
+        return $data;
+    }
 }
 
 

src/lib/utils/zpushadmin.php

@@ -113,6 +113,10 @@ class ZPushAdmin {
                 ZLog::Write(LOGLEVEL_WARN, sprintf("ZPushAdmin::GetDeviceDetails(): device '%s' of user '%s' has invalid states. Please sync to solve this issue.", $devid, $user));
                 $device->SetDeviceError("Invalid states. Please force synchronization!");
             }
+            catch (StatusException $ste) {
+                ZLog::Write(LOGLEVEL_WARN, sprintf("ZPushAdmin::GetDeviceDetails(): device '%s' of user '%s' has status exceptions. Please sync to solve this issue.", $devid, $user));
+                $device->SetDeviceError("State exceptions. Please force synchronization or remove device to fix!");
+            }
 
             if ($sc) {
                 if ($sc->GetLastSyncTime())
@@ -496,7 +500,7 @@ class ZPushAdmin {
             foreach ($device->GetAdditionalFolders() as $folder) {
                 $syncfolderid = $device->GetFolderIdForBackendId($folder['folderid'], false, false, null);
                 $folder['syncfolderid'] = $syncfolderid;
-                $folder['origin'] = Utils::GetFolderOriginFromId($syncfolderid);
+                $folder['origin'] = Utils::GetFolderOriginStringFromId($syncfolderid);
                 $new_list[$folder['folderid']] = $folder;
             }
             foreach (ZPush::GetAdditionalSyncFolders() as $fid => $so) {
@@ -509,7 +513,7 @@ class ZPushAdmin {
                                         'syncfolderid' => $syncfolderid,
                                         'name' => $so->displayname,
                                         'type' => $so->type,
-                                        'origin' => Utils::GetFolderOriginFromId($syncfolderid),
+                                        'origin' => Utils::GetFolderOriginStringFromId($syncfolderid),
                                         'flags' => 0,           // static folders have no flags
                                     );
                 }

src/lib/webservice/webservice.php

@@ -62,7 +62,7 @@ class Webservice {
             ZLog::Write(LOGLEVEL_INFO, sprintf("Webservice::HandleWebservice('%s'): user '%s' executing action for user '%s'", $commandCode, Request::GetAuthUser(), Request::GetGETUser()));
 
         // initialize non-wsdl soap server
-        $this->server = new SoapServer(null, array('uri' => "http://z-push.sf.net/webservice"));
+        $this->server = new SoapServer(null, array('uri' => "http://z-push.org/webservice"));
 
         // the webservice command is handled by its class
         if ($commandCode == ZPush::COMMAND_WEBSERVICE_DEVICE) {

tools/list-shared-folders.php

+#!/usr/bin/env php
+<?php
+/**********************************************************
+* File      :   list-shared-folders.php
+* Project   :   Z-Push - tools
+* Descr     :   Lists all users and devices that have open additional folders
+*
+* Created   :   20.10.2016
+*
+* Copyright 2007 - 2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+// Please adjust to match your z-push installation directory, usually /usr/share/z-push
+define('ZPUSH_BASE_PATH', "/usr/share/z-push");
+
+/**
+ * Usage:
+ *     php list-shared-folders.php
+ *              List all shared folders for all devices
+ *
+ *     php list-shared-folders.php | awk -F',' '$6 == "13" && $7 != ""'
+ *              List all shared folders of type calendar (13) that are synchronized on a device (else SyncUUID is empty)
+ *
+ *     php list-shared-folders.php | awk -F',' '$6 == "13" && $7 != "" { system("z-push-admin -a resync -d " $1 " -u " $2 " -t " $5) }'
+ *              Resynchronizes all shared folders from type calendar (13) that are synchronized already
+ */
+
+
+/************************************************
+ * MAIN
+ */
+
+try {
+    if (php_sapi_name() != "cli") {
+        fwrite(STDERR, "This script can only be called from the CLI.\n");
+        exit(1);
+    }
+
+    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
+        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
+    }
+
+    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
+    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);
+
+    require_once 'vendor/autoload.php';
+
+    if (!defined('ZPUSH_CONFIG')) define('ZPUSH_CONFIG', 'config.php');
+    include_once(ZPUSH_CONFIG);
+
+    ZPush::CheckConfig();
+
+    $sm = ZPush::GetStateMachine();
+    $devices = $sm->GetAllDevices();
+
+    if (!empty($devices)) {
+        printf("%s,%s,%s,%s,%s,%s,%s,%s\n", "DeviceId", "User", "Store", "SyncFolderId", "FolderId", "Type", "SyncUUID", "Name");
+    }
+    foreach ($devices as $devid) {
+        $users = ZPushAdmin::ListUsers($devid);
+        foreach($users as $user) {
+            $device = ZPushAdmin::GetDeviceDetails($devid, $user);
+            foreach ($device->GetAdditionalFolders() as $folder) {
+                $syncfolderid = $device->GetFolderIdForBackendId($folder['folderid'], false, false, null);
+                if(Utils::GetFolderOriginFromId($syncfolderid) !== DeviceManager::FLD_ORIGIN_SHARED) {
+                    continue;
+                }
+                $syncfolder = $device->GetHierarchyCache()->GetFolder($syncfolderid);
+                // if there is no syncfolder then this folder was not synchronized to the client (e.g. no permissions)
+                if (!$syncfolder) {
+                    continue;
+                }
+                $folderUuid = $device->GetFolderUUID($syncfolderid);
+                printf("%s,%s,%s,%s,%s,%d,%s,%s\n", $devid, $user, $syncfolder->Store, $syncfolderid, $folder['folderid'], $syncfolder->type, $folderUuid, $syncfolder->displayname);
+            }
+        }
+    }
+}
+catch (ZPushException $zpe) {
+    fwrite(STDERR, get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+    exit(1);
+}