Merge pull request #394 in ZP/z-push from develop to release/2.3

* commit '016e688b':
  ZP-1073 Use exit() instead of die().
  ZP-1076 Catch StatusException in ZPushAdmin::GetDeviceDetails().
  ZP-1073 list-shared-folders script, incl. usage examples (in file).
  ZP-1073 Refactor Utils::GetFolderOriginFromId() into GetFolderOriginStringFromId() so GetFolderOriginFromId() returns a value comparable to DeviceManager::FLD_ORIGIN_*.
  ZP-1017 Replace z-push.sf.net url with z-push.org.
  ZP-1062 depend on php instead of php7.0. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-1058 autodiscover: Fix mistake: always send response
  ZP-1056 Fail earlier when no password is provided. Log used username before trying to login in the backend. The $username will always be set and doesn't need to be checked again.
  ZP-1056 Fix authorization encoding and also add it to autodiscover. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-1058 autodiscover: Rework exception handling to match z-push core.
  ZP-1058 autodiscover: Print IP to log on failed authentication for usage with e.g. fail2ban

build/deb/debian.control

@@ -8,7 +8,7 @@ Homepage: http://z-push.org
 
 Package: z-push-common
 Architecture: all
-Depends: ${misc:Depends}, php5 (>= 5.4) | php7.0, php5-cli | php7.0-cli, php-soap
+Depends: ${misc:Depends}, php5 (>= 5.4) | php (>= 5.4) , php5-cli | php-cli, php-soap
 Conflicts: d-push, z-push
 Replaces: d-push, z-push
 Description: open source implementation of the ActiveSync protocol
@@ -26,7 +26,7 @@ Description: Z-Push caldav backend
 
 Package: z-push-backend-carddav
 Architecture: all
-Depends: ${misc:Depends}, z-push-common (= ${binary:Version}), php-curl | php5-curl, php5-xsl | php7.0-xsl
+Depends: ${misc:Depends}, z-push-common (= ${binary:Version}), php-curl | php5-curl, php5-xsl | php-xml
 Description: Z-Push carddav backend
  Backend for Z-Push, that adds the ability to connect to a carddav server
 

src/autodiscover/INSTALL

@@ -25,7 +25,7 @@ The specification suggests several ways for client to contact the responsible
 server to receive additional information. Tests have shown, that basically all 
 mobile phones tested support only the most basic ways. These are sufficient for 
 almost all types of scenarios and are the ones implemented by Z-Push AutoDiscover. 
-Please refer to the Mobile Compatibility List (http://z-push.sf.net/compatibility) 
+Please refer to the Mobile Compatibility List (http://z-push.org/compatibility) 
 for an overview of supported and tested devices.
 The used email address is the key for the process. The client splits it up into 
 the local and domain part (before and after the @-sign). The client then tries 

src/autodiscover/autodiscover.php

@@ -104,9 +104,24 @@ class ZPushAutodiscover {
             }
         }
 
-        catch (AuthenticationRequiredException $ex) {
+        catch (Exception $ex) {
+            // Extract any previous exception message for logging purpose.
+            $exclass = get_class($ex);
+            $exception_message = $ex->getMessage();
+            if($ex->getPrevious()){
+                do {
+                    $current_exception = $ex->getPrevious();
+                    $exception_message .= ' -> ' . $current_exception->getMessage();
+                } while($current_exception->getPrevious());
+            }
+
+            ZLog::Write(LOGLEVEL_FATAL, sprintf('Exception: (%s) - %s', $exclass, $exception_message));
+
+            if ($ex instanceof AuthenticationRequiredException) {
                 if (isset($incomingXml)) {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover because login failed for user with email '%s'", $incomingXml->Request->EMailAddress));
+                    // log the failed login attemt e.g. for fail2ban
+                    if (defined('LOGAUTHFAIL') && LOGAUTHFAIL != false)
+                        ZLog::Write(LOGLEVEL_WARN, sprintf("Unable to complete autodiscover because login failed for user with email '%s' from IP %s.", $incomingXml->Request->EMailAddress, $_SERVER["REMOTE_ADDR"]));
                 }
                 else {
                     ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover incorrect request: '%s'", $ex->getMessage()));
@@ -114,7 +129,7 @@ class ZPushAutodiscover {
                 http_response_code(401);
                 header('WWW-Authenticate: Basic realm="ZPush"');
             }
-        catch (ZPushException $ex) {
+            else if ($ex instanceof ZPushException) {
                 ZLog::Write(LOGLEVEL_ERROR, sprintf("Unable to complete autodiscover because of ZPushException. Error: %s", $ex->getMessage()));
                 if(!headers_sent()) {
                     header('HTTP/1.1 '. $ex->getHTTPCodeString());
@@ -123,6 +138,8 @@ class ZPushAutodiscover {
                     }
                 }
             }
+        }
+
         $this->sendResponse($response);
     }
 
@@ -182,21 +199,32 @@ class ZPushAutodiscover {
      * @return string $username
      */
     private function login($backend, $incomingXml) {
+        // don't even try to login if there is no PW set
+        if (!isset($_SERVER['PHP_AUTH_PW'])) {
+            throw new AuthenticationRequiredException("Access denied. No password provided.");
+        }
+
         // Determine the login name depending on the configuration: complete email address or
         // the local part only.
         if (USE_FULLEMAIL_FOR_LOGIN) {
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the complete email address for login."));
             $username = $incomingXml->Request->EMailAddress;
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the complete email address for login: '%s'", $username));
         }
-        else{
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the username only for login."));
+        else {
             $username = Utils::GetLocalPartFromEmail($incomingXml->Request->EMailAddress);
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the username only for login: '%s'", $username));
         }
 
-        if($backend->Logon($username, "", $_SERVER['PHP_AUTH_PW']) == false) {
+        // Mobile devices send Authorization header using UTF-8 charset. Outlook sends it using ISO-8859-1 encoding.
+        // For the successful authentication the user and password must be UTF-8 encoded. Try to determine which
+        // charset was sent by the client and convert it to UTF-8. See https://jira.z-hub.io/browse/ZP-864.
+        $username = Utils::ConvertAuthorizationToUTF8($username);
+        $password = Utils::ConvertAuthorizationToUTF8($_SERVER['PHP_AUTH_PW']);
+        if ($backend->Logon($username, "", $password) == false) {
             throw new AuthenticationRequiredException("Access denied. Username or password incorrect.");
         }
-        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login() Using '%s' as the username.", $username));
+
+        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login() successfull with '%s' as the username.", $username));
         return $username;
     }
 

src/lib/request/request.php

@@ -254,25 +254,10 @@ class Request {
         // Mobile devices send Authorization header using UTF-8 charset. Outlook sends it using ISO-8859-1 encoding.
         // For the successful authentication the user and password must be UTF-8 encoded. Try to determine which
         // charset was sent by the client and convert it to UTF-8. See https://jira.z-hub.io/browse/ZP-864.
-        if (isset($_SERVER['PHP_AUTH_USER'])) {
-            $encoding = mb_detect_encoding(self::$authUser, "UTF-8, ISO-8859-1");
-            if (!$encoding) {
-                $encoding = mb_detect_encoding(self::$authUser, Utils::GetAvailableCharacterEncodings());
-                if ($encoding) {
-                    ZLog::Write(LOGLEVEL_WARN,
-                            sprintf("Request->ProcessHeaders(): mb_detect_encoding detected '%s' charset. This charset is not in the default detect list. Please report it to Z-Push developers.",
-                                    $encoding));
-                }
-                else {
-                    ZLog::Write(LOGLEVEL_ERROR, "Request->ProcessHeaders(): mb_detect_encoding failed to detect the Authorization header charset. It's possible that user won't be able to login.");
-                }
-            }
-            if ($encoding && strtolower($encoding) != "utf-8") {
-                ZLog::Write(LOGLEVEL_DEBUG, sprintf("Request->ProcessHeaders(): mb_detect_encoding detected '%s' charset. Authorization header will be converted to UTF-8 from it.", $encoding));
-                self::$authUser = mb_convert_encoding(self::$authUser, "UTF-8", $encoding);
-                self::$authPassword = mb_convert_encoding(self::$authPassword, "UTF-8", $encoding);
-            }
-        }
+        if (isset(self::$authUser))
+          self::$authUser = Utils::ConvertAuthorizationToUTF8(self::$authUser);
+        if (isset(self::$authPassword))
+          self::$authPassword = Utils::ConvertAuthorizationToUTF8(self::$authPassword);
     }
 
     /**

src/lib/utils/utils.php

@@ -1092,15 +1092,36 @@ class Utils {
     }
 
     /**
-     * Returns folder origin from its id.
+     * Returns folder origin identifier from its id.
      *
-     * @param string $fid
+     * @param string $folderid
+     *
+     * @access public
+     * @return string|boolean  matches values of DeviceManager::FLD_ORIGIN_*
+     */
+    public static function GetFolderOriginFromId($folderid) {
+        $origin = substr($folderid, 0, 1);
+        switch ($origin) {
+            case DeviceManager::FLD_ORIGIN_CONFIG:
+            case DeviceManager::FLD_ORIGIN_GAB:
+            case DeviceManager::FLD_ORIGIN_SHARED:
+            case DeviceManager::FLD_ORIGIN_USER:
+                return $origin;
+        }
+        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginFromId(): Unknown folder origin for folder with id '%s'", $folderid));
+        return false;
+    }
+
+    /**
+     * Returns folder origin as string from its id.
+     *
+     * @param string $folderid
      *
      * @access public
      * @return string
      */
-    public static function GetFolderOriginFromId($fid) {
-        $origin = substr($fid, 0, 1);
+    public static function GetFolderOriginStringFromId($folderid) {
+        $origin = substr($folderid, 0, 1);
         switch ($origin) {
             case DeviceManager::FLD_ORIGIN_CONFIG:
                 return 'configured';
@@ -1111,7 +1132,7 @@ class Utils {
             case DeviceManager::FLD_ORIGIN_USER:
                 return 'user';
         }
-        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginFromId(): Unknown folder origin for folder with id '%s'", $fid));
+        ZLog::Write(LOGLEVEL_WARN, sprintf("Utils->GetFolderOriginStringFromId(): Unknown folder origin for folder with id '%s'", $folderid));
         return 'unknown';
     }
 
@@ -1130,6 +1151,38 @@ class Utils {
         }
         return array(null, $id);
     }
+
+    /**
+     * Detects encoding of the input and converts it to UTF-8.
+     * This is currently only used for authorization header conversion.
+     *
+     * @param string      $data     input data
+     *
+     * @access public
+     * @return string               utf-8 encoded data
+     */
+    public static function ConvertAuthorizationToUTF8($data) {
+        $encoding = mb_detect_encoding($data, "UTF-8, ISO-8859-1");
+
+        if (!$encoding) {
+            $encoding = mb_detect_encoding($data, Utils::GetAvailableCharacterEncodings());
+            if ($encoding) {
+                ZLog::Write(LOGLEVEL_WARN,
+                        sprintf("Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding detected '%s' charset. This charset is not in the default detect list. Please report it to Z-Push developers.",
+                                $encoding));
+            }
+            else {
+                ZLog::Write(LOGLEVEL_ERROR, "Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding failed to detect the Authorization header charset. It's possible that user won't be able to login.");
+            }
+        }
+
+        if ($encoding && strtolower($encoding) != "utf-8") {
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Utils::ConvertAuthorizationToUTF8(): mb_detect_encoding detected '%s' charset. Authorization header will be converted to UTF-8 from it.", $encoding));
+            return mb_convert_encoding($data, "UTF-8", $encoding);
+        }
+
+        return $data;
+    }
 }
 
 

src/lib/utils/zpushadmin.php

@@ -113,6 +113,10 @@ class ZPushAdmin {
                 ZLog::Write(LOGLEVEL_WARN, sprintf("ZPushAdmin::GetDeviceDetails(): device '%s' of user '%s' has invalid states. Please sync to solve this issue.", $devid, $user));
                 $device->SetDeviceError("Invalid states. Please force synchronization!");
             }
+            catch (StatusException $ste) {
+                ZLog::Write(LOGLEVEL_WARN, sprintf("ZPushAdmin::GetDeviceDetails(): device '%s' of user '%s' has status exceptions. Please sync to solve this issue.", $devid, $user));
+                $device->SetDeviceError("State exceptions. Please force synchronization or remove device to fix!");
+            }
 
             if ($sc) {
                 if ($sc->GetLastSyncTime())
@@ -496,7 +500,7 @@ class ZPushAdmin {
             foreach ($device->GetAdditionalFolders() as $folder) {
                 $syncfolderid = $device->GetFolderIdForBackendId($folder['folderid'], false, false, null);
                 $folder['syncfolderid'] = $syncfolderid;
-                $folder['origin'] = Utils::GetFolderOriginFromId($syncfolderid);
+                $folder['origin'] = Utils::GetFolderOriginStringFromId($syncfolderid);
                 $new_list[$folder['folderid']] = $folder;
             }
             foreach (ZPush::GetAdditionalSyncFolders() as $fid => $so) {
@@ -509,7 +513,7 @@ class ZPushAdmin {
                                         'syncfolderid' => $syncfolderid,
                                         'name' => $so->displayname,
                                         'type' => $so->type,
-                                        'origin' => Utils::GetFolderOriginFromId($syncfolderid),
+                                        'origin' => Utils::GetFolderOriginStringFromId($syncfolderid),
                                         'flags' => 0,           // static folders have no flags
                                     );
                 }

src/lib/webservice/webservice.php

@@ -62,7 +62,7 @@ class Webservice {
             ZLog::Write(LOGLEVEL_INFO, sprintf("Webservice::HandleWebservice('%s'): user '%s' executing action for user '%s'", $commandCode, Request::GetAuthUser(), Request::GetGETUser()));
 
         // initialize non-wsdl soap server
-        $this->server = new SoapServer(null, array('uri' => "http://z-push.sf.net/webservice"));
+        $this->server = new SoapServer(null, array('uri' => "http://z-push.org/webservice"));
 
         // the webservice command is handled by its class
         if ($commandCode == ZPush::COMMAND_WEBSERVICE_DEVICE) {

tools/list-shared-folders.php

+#!/usr/bin/env php
+<?php
+/**********************************************************
+* File      :   list-shared-folders.php
+* Project   :   Z-Push - tools
+* Descr     :   Lists all users and devices that have open additional folders
+*
+* Created   :   20.10.2016
+*
+* Copyright 2007 - 2016 Zarafa Deutschland GmbH
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Affero General Public License, version 3,
+* as published by the Free Software Foundation with the following additional
+* term according to sec. 7:
+*
+* According to sec. 7 of the GNU Affero General Public License, version 3,
+* the terms of the AGPL are supplemented with the following terms:
+*
+* "Zarafa" is a registered trademark of Zarafa B.V.
+* "Z-Push" is a registered trademark of Zarafa Deutschland GmbH
+* The licensing of the Program under the AGPL does not imply a trademark license.
+* Therefore any rights, title and interest in our trademarks remain entirely with us.
+*
+* However, if you propagate an unmodified version of the Program you are
+* allowed to use the term "Z-Push" to indicate that you distribute the Program.
+* Furthermore you may use our trademarks where it is necessary to indicate
+* the intended purpose of a product or service provided you use it in accordance
+* with honest practices in industrial or commercial matters.
+* If you want to propagate modified versions of the Program under the name "Z-Push",
+* you may only do so if you have a written permission by Zarafa Deutschland GmbH
+* (to acquire a permission please contact Zarafa at trademark@zarafa.com).
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+* GNU Affero General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* Consult LICENSE file for details
+************************************************/
+
+// Please adjust to match your z-push installation directory, usually /usr/share/z-push
+define('ZPUSH_BASE_PATH', "/usr/share/z-push");
+
+/**
+ * Usage:
+ *     php list-shared-folders.php
+ *              List all shared folders for all devices
+ *
+ *     php list-shared-folders.php | awk -F',' '$6 == "13" && $7 != ""'
+ *              List all shared folders of type calendar (13) that are synchronized on a device (else SyncUUID is empty)
+ *
+ *     php list-shared-folders.php | awk -F',' '$6 == "13" && $7 != "" { system("z-push-admin -a resync -d " $1 " -u " $2 " -t " $5) }'
+ *              Resynchronizes all shared folders from type calendar (13) that are synchronized already
+ */
+
+
+/************************************************
+ * MAIN
+ */
+
+try {
+    if (php_sapi_name() != "cli") {
+        fwrite(STDERR, "This script can only be called from the CLI.\n");
+        exit(1);
+    }
+
+    if (!defined('ZPUSH_BASE_PATH') || !file_exists(ZPUSH_BASE_PATH . "/config.php")) {
+        die("ZPUSH_BASE_PATH not set correctly or no config.php file found\n");
+    }
+
+    define('BASE_PATH_CLI',  ZPUSH_BASE_PATH ."/");
+    set_include_path(get_include_path() . PATH_SEPARATOR . ZPUSH_BASE_PATH);
+
+    require_once 'vendor/autoload.php';
+
+    if (!defined('ZPUSH_CONFIG')) define('ZPUSH_CONFIG', 'config.php');
+    include_once(ZPUSH_CONFIG);
+
+    ZPush::CheckConfig();
+
+    $sm = ZPush::GetStateMachine();
+    $devices = $sm->GetAllDevices();
+
+    if (!empty($devices)) {
+        printf("%s,%s,%s,%s,%s,%s,%s,%s\n", "DeviceId", "User", "Store", "SyncFolderId", "FolderId", "Type", "SyncUUID", "Name");
+    }
+    foreach ($devices as $devid) {
+        $users = ZPushAdmin::ListUsers($devid);
+        foreach($users as $user) {
+            $device = ZPushAdmin::GetDeviceDetails($devid, $user);
+            foreach ($device->GetAdditionalFolders() as $folder) {
+                $syncfolderid = $device->GetFolderIdForBackendId($folder['folderid'], false, false, null);
+                if(Utils::GetFolderOriginFromId($syncfolderid) !== DeviceManager::FLD_ORIGIN_SHARED) {
+                    continue;
+                }
+                $syncfolder = $device->GetHierarchyCache()->GetFolder($syncfolderid);
+                // if there is no syncfolder then this folder was not synchronized to the client (e.g. no permissions)
+                if (!$syncfolder) {
+                    continue;
+                }
+                $folderUuid = $device->GetFolderUUID($syncfolderid);
+                printf("%s,%s,%s,%s,%s,%d,%s,%s\n", $devid, $user, $syncfolder->Store, $syncfolderid, $folder['folderid'], $syncfolder->type, $folderUuid, $syncfolder->displayname);
+            }
+        }
+    }
+}
+catch (ZPushException $zpe) {
+    fwrite(STDERR, get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+    exit(1);
+}