Merge branch 'develop' into bugfix/ZP-1267-change-summed-mapi-tags

src/autodiscover/INSTALL

@@ -170,7 +170,21 @@ USE_FULLEMAIL_FOR_LOGIN If this is set to "true", AutoDiscover will attempt to
                         login on the collaboration server with the full email 
                         address sent by the client. If disabled (default), the 
                         local part of the email address is used.
-                        
+
+AUTODISCOVER_LOGIN_TYPE If the local part of the email address doesn't match the
+                        user name, this parameter helps to convert it for some common cases.
+Possible values:
+    AUTODISCOVER_LOGIN_EMAIL            uses the local part of the email address
+                                        as provided when setting up the account.
+    AUTODISCOVER_LOGIN_NO_DOT           removes the '.' from email address:
+                                        email: first.last@domain.com -> resulting username: firstlast
+    AUTODISCOVER_LOGIN_F_NO_DOT_LAST    cuts the first part before '.' after the first letter
+                                        and removes the '.' from email address:
+                                        email: first.last@domain.com -> resulting username: flast
+    AUTODISCOVER_LOGIN_F_DOT_LAST       cuts the part before '.' after the first letter and
+                                        leaves the part after '.' as is:
+                                        email: first.last@domain.com -> resulting username: f.last
+
 LOGFILEDIR              The directory where logfiles are created.
 
 LOGFILE                 The default AutoDiscover log file.

src/autodiscover/autodiscover.php

@@ -190,11 +190,25 @@ class ZPushAutodiscover {
         // the local part only.
         if (USE_FULLEMAIL_FOR_LOGIN) {
             $username = $incomingXml->Request->EMailAddress;
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the complete email address for login: '%s'", $username));
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login(): Using the complete email address for login: '%s'", $username));
         }
         else {
             $username = Utils::GetLocalPartFromEmail($incomingXml->Request->EMailAddress);
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Using the username only for login: '%s'", $username));
+            if (defined('AUTODISCOVER_LOGIN_TYPE') && AUTODISCOVER_LOGIN_TYPE != AUTODISCOVER_LOGIN_EMAIL) {
+                switch (AUTODISCOVER_LOGIN_TYPE) {
+                    case AUTODISCOVER_LOGIN_NO_DOT:
+                        $username = str_replace('.', '', $username);
+                        break;
+                    case AUTODISCOVER_LOGIN_F_NO_DOT_LAST:
+                        $username = str_replace('.', '', substr_replace($username, '', 1, strpos($username, '.') - 1));
+                        break;
+                    case AUTODISCOVER_LOGIN_F_DOT_LAST:
+                        $username = substr_replace($username, '', 1, strpos($username, '.') - 1);
+                        break;
+                }
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login(): AUTODISCOVER_LOGIN_TYPE is set to %d", AUTODISCOVER_LOGIN_TYPE));
+            }
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAutodiscover->login(): Using the username only for login: '%s'", $username));
         }
 
         // Mobile devices send Authorization header using UTF-8 charset. Outlook sends it using ISO-8859-1 encoding.

src/autodiscover/config.php

@@ -45,6 +45,28 @@
      */
     define('USE_FULLEMAIL_FOR_LOGIN', false);
 
+    /*
+     * AutoDiscover requires the username to match either the email address
+     * or the local part of the email address.
+     * This is not always possible as the username might have a different
+     * schema than email address. Configure this parameter to match your
+     * username settings.
+     * @see https://wiki.z-hub.io/display/ZP/Configuring+Z-Push+Autodiscover#ConfiguringZ-PushAutodiscover-Configuration
+     * @see https://jira.z-hub.io/browse/ZP-1209
+     *
+     * Possible values:
+     * AUTODISCOVER_LOGIN_EMAIL             - uses the email address as provided when setting up the account
+     * AUTODISCOVER_LOGIN_NO_DOT            - removes the '.' from email address:
+     *                                          email: first.last@domain.com -> resulting username: firstlast
+     * AUTODISCOVER_LOGIN_F_NO_DOT_LAST     - cuts the first part before '.' after the first letter and
+     *                                          removes the '.' from email address:
+     *                                          email: first.last@domain.com -> resulting username: flast
+     * AUTODISCOVER_LOGIN_F_DOT_LAST        - cuts the part before '.' after the first letter and
+     *                                          leaves the part after '.' as is:
+     *                                          email: first.last@domain.com -> resulting username: f.last
+     */
+    define('AUTODISCOVER_LOGIN_TYPE', AUTODISCOVER_LOGIN_EMAIL);
+
 /**********************************************************************************
  *  Logging settings
  *  Possible LOGLEVEL and LOGUSERLEVEL values are:

src/backend/imap/imap.php

@@ -190,7 +190,7 @@ class BackendIMAP extends BackendDiff implements ISearchProvider {
 
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("BackendIMAP->SendMail(): We get the new message"));
         $mobj = new Mail_mimeDecode($sm->mime);
-        $message = $mobj->decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
+        $message = $mobj->decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
         unset($mobj);
 
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("BackendIMAP->SendMail(): We get the From and To"));
@@ -485,7 +485,7 @@ class BackendIMAP extends BackendDiff implements ISearchProvider {
         }
 
         $mobj = new Mail_mimeDecode($mail);
-        $message = $mobj->decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
+        $message = $mobj->decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
 
         if (!isset($message->parts)) {
             throw new StatusException(sprintf("BackendIMAP->GetAttachmentData('%s'): Error, message without parts. Requesting part key: '%d'", $attname, $part), SYNC_ITEMOPERATIONSSTATUS_INVALIDATT);
@@ -1032,7 +1032,7 @@ class BackendIMAP extends BackendDiff implements ISearchProvider {
             }
 
             $mobj = new Mail_mimeDecode($mail);
-            $message = $mobj->decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
+            $message = $mobj->decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
 
             Utils::CheckAndFixEncodingInHeaders($mail, $message);
 
@@ -1672,7 +1672,7 @@ class BackendIMAP extends BackendDiff implements ISearchProvider {
         // Get the original calendar request, so we don't need to create it from scratch
         $mobj = new Mail_mimeDecode($mail);
         unset($mail);
-        $message = $mobj->decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
+        $message = $mobj->decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'rfc_822bodies' => true, 'charset' => 'utf-8'));
         unset($mobj);
 
         $body_part = null;

src/backend/kopano/mapi/mapitags.php

@@ -1135,6 +1135,12 @@ define('PR_EC_STATS_SESSION_LOCKED'                   ,mapi_prop_tag(PT_BOOLEAN,
 define('PR_EC_STATS_SESSION_BUSYSTATES'               ,mapi_prop_tag(PT_MV_STRING8,  0x6747));
 define('PR_EC_COMPANY_NAME'                           ,mapi_prop_tag(PT_STRING8,     0x6748));
 
+/* kopano specific properties for optimization of imap functionality */
+define('PR_EC_IMAP_EMAIL'                             ,mapi_prop_tag(PT_BINARY,      PR_EC_BASE+0x8C)); // the complete rfc822 email
+define('PR_EC_IMAP_EMAIL_SIZE'                        ,mapi_prop_tag(PT_LONG,        PR_EC_BASE+0x8D));
+define('PR_EC_IMAP_BODY'                              ,mapi_prop_tag(PT_STRING8,     PR_EC_BASE+0x8E)); // simplified bodystructure (mostly unused by clients)
+define('PR_EC_IMAP_BODYSTRUCTURE'                     ,mapi_prop_tag(PT_STRING8,     PR_EC_BASE+0x8F)); // extended bodystructure (often used by clients)
+
 /* user features */
 define('PR_EC_ENABLED_FEATURES'                       ,mapi_prop_tag(PT_MV_TSTRING,  0x67B3));
 define('PR_EC_ENABLED_FEATURES_A'                     ,mapi_prop_tag(PT_MV_STRING8,  0x67B3));

src/backend/kopano/mapiprovider.php

@@ -2533,32 +2533,37 @@ class MAPIProvider {
      * @return boolean
      */
     private function imtoinet($mapimessage, &$message) {
-        if (function_exists("mapi_inetmapi_imtoinet")) {
+        $mapiEmail = mapi_getprops($mapimessage, array(PR_EC_IMAP_EMAIL));
+        $stream = false;
+        if (isset($mapiEmail[PR_EC_IMAP_EMAIL]) || MAPIUtils::GetError(PR_EC_IMAP_EMAIL, $mapiEmail) == MAPI_E_NOT_ENOUGH_MEMORY) {
+            $stream = mapi_openproperty($mapimessage, PR_EC_IMAP_EMAIL, IID_IStream, 0, 0);
+            ZLog::Write(LOGLEVEL_DEBUG, "MAPIProvider->imtoinet(): using PR_EC_IMAP_EMAIL as full RFC822 message");
+        }
+        else {
             $addrbook = $this->getAddressbook();
             $stream = mapi_inetmapi_imtoinet($this->session, $addrbook, $mapimessage, array('use_tnef' => -1));
-
-            if (is_resource($stream)) {
-                $mstreamstat = mapi_stream_stat($stream);
-                $streamsize = $mstreamstat["cb"];
-                if (isset($streamsize)) {
-                    if (Request::GetProtocolVersion() >= 12.0) {
-                        if (!isset($message->asbody))
-                            $message->asbody = new SyncBaseBody();
-                        $message->asbody->data = MAPIStreamWrapper::Open($stream);
-                        $message->asbody->estimatedDataSize = $streamsize;
-                        $message->asbody->truncated = 0;
-                    }
-                    else {
-                        $message->mimedata = MAPIStreamWrapper::Open($stream);
-                        $message->mimesize = $streamsize;
-                        $message->mimetruncated = 0;
-                    }
-                    unset($message->body, $message->bodytruncated);
-                    return true;
+        }
+        if (is_resource($stream)) {
+            $mstreamstat = mapi_stream_stat($stream);
+            $streamsize = $mstreamstat["cb"];
+            if (isset($streamsize)) {
+                if (Request::GetProtocolVersion() >= 12.0) {
+                    if (!isset($message->asbody))
+                        $message->asbody = new SyncBaseBody();
+                    $message->asbody->data = MAPIStreamWrapper::Open($stream);
+                    $message->asbody->estimatedDataSize = $streamsize;
+                    $message->asbody->truncated = 0;
+                }
+                else {
+                    $message->mimedata = MAPIStreamWrapper::Open($stream);
+                    $message->mimesize = $streamsize;
+                    $message->mimetruncated = 0;
                 }
+                unset($message->body, $message->bodytruncated);
+                return true;
             }
-            ZLog::Write(LOGLEVEL_ERROR, sprintf("MAPIProvider->imtoinet(): got no stream or content from mapi_inetmapi_imtoinet()"));
         }
+        ZLog::Write(LOGLEVEL_ERROR, "MAPIProvider->imtoinet(): got no stream or content from mapi_inetmapi_imtoinet()");
 
         return false;
     }

src/backend/maildir/maildir.php

@@ -116,7 +116,7 @@ class BackendMaildir extends BackendDiff {
         // Parse e-mail
         $rfc822 = file_get_contents($this->getPath() . "/$fn");
 
-        $message = Mail_mimeDecode::decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'input' => $rfc822, 'crlf' => "\n", 'charset' => 'utf-8'));
+        $message = Mail_mimeDecode::decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'input' => $rfc822, 'crlf' => "\n", 'charset' => 'utf-8'));
 
         $attachment = new SyncItemOperationsAttachment();
         $attachment->data = StringStreamWrapper::Open($message->parts[$part]->body);
@@ -333,7 +333,7 @@ class BackendMaildir extends BackendDiff {
         // Parse e-mail
         $rfc822 = file_get_contents($this->getPath() . "/" . $fn);
 
-        $message = Mail_mimeDecode::decode(array('decode_headers' => true, 'decode_bodies' => true, 'include_bodies' => true, 'input' => $rfc822, 'crlf' => "\n", 'charset' => 'utf-8'));
+        $message = Mail_mimeDecode::decode(array('decode_headers' => 'utf-8', 'decode_bodies' => true, 'include_bodies' => true, 'input' => $rfc822, 'crlf' => "\n", 'charset' => 'utf-8'));
 
         Utils::CheckAndFixEncodingInHeaders($mail, $message);
 

src/lib/core/zpushdefs.php

@@ -1053,3 +1053,8 @@ define("NOTEIVERB_FORWARD", 104);
 define("AS_REPLYTOSENDER", 1);
 define("AS_REPLYTOALL", 2);
 define("AS_FORWARD", 3);
+
+define('AUTODISCOVER_LOGIN_EMAIL', 0);
+define('AUTODISCOVER_LOGIN_NO_DOT', 1);
+define('AUTODISCOVER_LOGIN_F_NO_DOT_LAST', 2);
+define('AUTODISCOVER_LOGIN_F_DOT_LAST', 3);

src/lib/utils/zpushadmin.php

@@ -28,6 +28,11 @@ class ZPushAdmin {
      * //TODO resync of a foldertype for all users (e.g. Appointment)
      */
 
+    const STATUS_SUCCESS = 0;
+    const STATUS_DEVICE_SYNCED_AFTER_DAYSOLD = 1;
+
+    public static $status = self::STATUS_SUCCESS;
+
     /**
      * List devices known to Z-Push.
      * If no user is given, all devices are listed
@@ -219,11 +224,13 @@ class ZPushAdmin {
      *
      * @param string    $user           (opt) user of the device
      * @param string    $devid          (opt) device id which should be removed
+     * @param int       $daysOld        (opt) devices which haven't synced for $daysOld days
+     * @param int       $time           (opt) unix timestamp to use with $daysOld
      *
      * @return boolean
      * @access public
      */
-    static public function RemoveDevice($user = false, $devid = false) {
+    static public function RemoveDevice($user = false, $devid = false, $daysOld = false, $time = false) {
         if ($user === false && $devid === false)
             return false;
 
@@ -232,7 +239,7 @@ class ZPushAdmin {
             $devicesIds = ZPush::GetStateMachine()->GetAllDevices($user);
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::RemoveDevice(): all '%d' devices for user '%s' found to be removed", count($devicesIds), $user));
             foreach ($devicesIds as $deviceid) {
-                if (!self::RemoveDevice($user, $deviceid)) {
+                if (!self::RemoveDevice($user, $deviceid, $daysOld, $time)) {
                     ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::RemoveDevice(): removing devices failed for device '%s' of user '%s'. Aborting", $deviceid, $user));
                     return false;
                 }
@@ -243,7 +250,7 @@ class ZPushAdmin {
             $users = self::ListUsers($devid);
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::RemoveDevice(): device '%d' is used by '%d' users and will be removed", $devid, count($users)));
             foreach ($users as $aUser) {
-                if (!self::RemoveDevice($aUser, $devid)) {
+                if (!self::RemoveDevice($aUser, $devid, $daysOld, $time)) {
                     ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::RemoveDevice(): removing user '%s' from device '%s' failed. Aborting", $aUser, $devid));
                     return false;
                 }
@@ -260,6 +267,21 @@ class ZPushAdmin {
                 $device->SetData($devicedata, false);
                 if (!isset($devicedata->devices))
                     throw new StateInvalidException("No devicedata stored in ASDevice");
+
+                if ($daysOld) {
+                    if (!$time) {
+                        $time = time();
+                    }
+                    $lastSynced = floor(($time - $device->getLastupdatetime()) / 86400);
+                    if ($daysOld > $lastSynced) {
+                        ZLog::Write(LOGLEVEL_INFO,
+                                sprintf("ZPushAdmin::RemoveDevice(): device '%s' of user '%s' synced %d day(s) ago but only devices which synced more than %d days ago will be removed. Skipping.",
+                                        $devid, $user, $lastSynced, $daysOld));
+                        self::$status = self::STATUS_DEVICE_SYNCED_AFTER_DAYSOLD;
+                        return true;
+                    }
+                }
+
                 $devices = $devicedata->devices;
             }
             catch (StateNotFoundException $e) {

src/z-push-admin.php

@@ -92,6 +92,7 @@ class ZPushAdminCLI {
     static private $device = false;
     static private $type = false;
     static private $errormessage;
+    static private $daysold = false;
 
     /**
      * Returns usage instructions
@@ -102,7 +103,8 @@ class ZPushAdminCLI {
     static public function UsageInstructions() {
         return  "Usage:\n\tz-push-admin.php -a ACTION [options]\n\n" .
                 "Parameters:\n\t-a list/lastsync/wipe/remove/resync/clearloop/fixstates\n\t[-u] username\n\t[-d] deviceid\n" .
-                "\t[-t] type\tthe following types are available: '".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."', '".self::TYPE_OPTION_NOTE."', '".self::TYPE_OPTION_HIERARCHY."' of '".self::TYPE_OPTION_GAB."' (for KOE) or a folder id.\n\n" .
+                "\t[-t] type\tthe following types are available: '".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."', '".self::TYPE_OPTION_NOTE."', '".self::TYPE_OPTION_HIERARCHY."' of '".self::TYPE_OPTION_GAB."' (for KOE) or a folder id.\n" .
+                "\t[--days-old] n\tshow or remove profiles older than n days with lastsync or remove. n must be a positive integer.\n\n".
                 "Actions:\n" .
                 "\tlist\t\t\t\t\t Lists all devices and synchronized users\n" .
                 "\tlist -u USER\t\t\t\t Lists all devices of user USER\n" .
@@ -150,7 +152,7 @@ class ZPushAdminCLI {
         if (self::$errormessage)
             return;
 
-        $options = getopt("u:d:a:t:");
+        $options = getopt("u:d:a:t:", array('user:', 'device:', 'action:', 'type:', 'days-old:', 'days-ago:'));
 
         // get 'user'
         if (isset($options['u']) && !empty($options['u']))
@@ -177,6 +179,19 @@ class ZPushAdminCLI {
         elseif (isset($options['type']) && !empty($options['type']))
             self::$type = strtolower(trim($options['type']));
 
+        if (isset($options['days-ago']) && !empty($options['days-ago'])) {
+            $options['days-old'] = $options['days-ago'];
+        }
+
+        if (isset($options['days-old']) && !empty($options['days-old'])) {
+            if (!is_numeric($options['days-old']) || $options['days-old'] < 0) {
+                self::$errormessage = "--days-old parameter must be a positive integer\n";
+                self::$command = null;
+                return;
+            }
+            self::$daysold = trim($options['days-old']);
+        }
+
         // if type is set, it must be one of known types or a 44 or 48 byte long folder id
         if (self::$type !== false) {
             if (self::$type !== self::TYPE_OPTION_EMAIL &&
@@ -438,17 +453,22 @@ class ZPushAdminCLI {
             echo "\tno devices found\n";
         else {
             echo "All known devices and users and their last synchronization time\n\n";
-            echo str_pad("Device id", 36) . str_pad("Synchronized user", 31) . str_pad("Last sync time", 20) . "Short Ids\n";
-            echo "-----------------------------------------------------------------------------------------------------\n";
+            echo str_pad("Device id", 36) . str_pad("Synchronized user", 31) . str_pad("Last sync time", 33) . "Short Ids\n";
+            echo "------------------------------------------------------------------------------------------------------------------\n";
         }
 
+        $now = time();
         foreach ($devicelist as $deviceId) {
             $users = ZPushAdmin::ListUsers($deviceId);
             foreach ($users as $user) {
                 $device = ZPushAdmin::GetDeviceDetails($deviceId, $user);
-                $lastsync = $device->GetLastSyncTime() ? strftime("%Y-%m-%d %H:%M", $device->GetLastSyncTime()) : "never";
+                $daysOld = floor(($now - $device->GetLastSyncTime()) / 86400);
+                if (self::$daysold > $daysOld) {
+                    continue;
+                }
+                $lastsync = $device->GetLastSyncTime() ? strftime("%Y-%m-%d %H:%M", $device->GetLastSyncTime()) . ' (' . str_pad($daysOld, 3, ' ', STR_PAD_LEFT) . ' days ago)' : "never";
                 $hasShortFolderIds = $device->HasFolderIdMapping() ? "Yes":"No";
-                echo str_pad($deviceId, 36) . str_pad($user, 30) . " " . str_pad($lastsync, 20) . $hasShortFolderIds . "\n";
+                echo str_pad($deviceId, 36) . str_pad($user, 30) . " " . str_pad($lastsync, 33) . $hasShortFolderIds . "\n";
             }
         }
     }
@@ -505,20 +525,24 @@ class ZPushAdminCLI {
     }
 
     /**
-     * Command "Remove device"
-     * Remove a device of that user from the device list
+     * Command "Remove device".
+     * Removes a device of that user from the device list.
      *
      * @return
      * @access public
      */
     static public function CommandRemoveDevice() {
-        $stat = ZPushAdmin::RemoveDevice(self::$user, self::$device);
+        $stat = ZPushAdmin::RemoveDevice(self::$user, self::$device, self::$daysold, time());
         if (self::$user === false)
            echo sprintf("State data of device '%s' removed: %s", self::$device, ($stat)?'OK':ZLog::GetLastMessage(LOGLEVEL_ERROR)). "\n";
         elseif (self::$device === false)
            echo sprintf("State data of all devices of user '%s' removed: %s", self::$user, ($stat)?'OK':ZLog::GetLastMessage(LOGLEVEL_ERROR)). "\n";
         else
            echo sprintf("State data of device '%s' of user '%s' removed: %s", self::$device, self::$user, ($stat)?'OK':ZLog::GetLastMessage(LOGLEVEL_ERROR)). "\n";
+
+        if (ZPushAdmin::$status == ZPushAdmin::STATUS_DEVICE_SYNCED_AFTER_DAYSOLD) {
+            print("Some devices might not have been removed because of --days-old parameter. Check Z-Push log file for more details.\n");
+        }
     }
 
     /**