Merge pull request #248 in ZP/z-push from develop to release/2.3

* commit 'df6fc45e':
  ZP-891 Show folderstats log in LOGLEVEL_DEBUG.
  ZP-939 Now with file.
  ZP-939 ZPushAdmin::GetDeviceDetails() should load all states (also the hierarchy cache), mark the KOE GAB folder as gab in the overview of folders of z-push-admin, show folder name of currently synchronizing folders.
  ZP-937 Warn on mutex block only if it takes longer than 50ms.
  ZP-936 Only read 4 additional bytes when we need to truncate and are at end of stream.
  ZP-933 Now with file.
  ZP-933 Fixed wording in ZPushAdmin::ResyncFolder() if no folderid has been submitted for resynching, removed always false check, always strtolower() the searchFor as well.
  ZP-934 Use is_resource() to check the stream, check if categories are set before intersecting.
  ZP-935 Check if ASDevice changed before saving it.
  ZP-933 Added "gab" option for resync type, lookup gab folderid in configuration or device data, fixed resynching long and short folderids, to lookup folderids from backendIds stripdata should not unset backend2folderidCache.
  ZP-250 Fixed typo in Terminate output message.

src/backend/ipcmemcached/ipcmemcachedprovider.php

@@ -193,7 +193,7 @@ class IpcMemcachedProvider implements IIpcProvider {
                 return false;
             }
         }
-        if ($n) {
+        if ($n*MEMCACHED_BLOCK_WAIT > 50) {
             ZLog::Write(LOGLEVEL_WARN, sprintf("IpcMemcachedProvider->BlockMutex() mutex aquired after waiting for %sms for type: %s", ($n*MEMCACHED_BLOCK_WAIT), $this->type));
         }
         return true;

src/backend/kopano/kopano.php

@@ -1417,7 +1417,7 @@ class BackendKopano implements IBackend, ISearchProvider {
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("KopanoBackend->GetFolderStat() fetched status information of %d folders for store '%s'", count($this->folderStatCache[$user]), $user));
             // TODO remove logging
             foreach($this->folderStatCache[$user] as $fid => $stat) {
-                ZLog::Write(LOGLEVEL_INFO, sprintf("FolderStat: %s %s %s\t%s", $user, $fid, $stat, $this->nameCache[$fid]));
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("FolderStat: %s %s %s\t%s", $user, $fid, $stat, $this->nameCache[$fid]));
             }
         }
 

src/backend/kopano/mapiprovider.php

@@ -2459,7 +2459,7 @@ class MAPIProvider {
             $addrbook = $this->getAddressbook();
             $stream = mapi_inetmapi_imtoinet($this->session, $addrbook, $mapimessage, array('use_tnef' => -1));
 
-            if (isset($stream)) {
+            if (is_resource($stream)) {
                 $mstreamstat = mapi_stream_stat($stream);
                 $streamsize = $mstreamstat["cb"];
                 if (isset($streamsize)) {

src/backend/kopano/mapistreamwrapper.php

@@ -100,8 +100,9 @@ class MAPIStreamWrapper {
         $len = ($this->position + $len > $this->streamlength) ? ($this->streamlength - $this->position) : $len;
 
         // read 4 additional bytes from the stream so we can always truncate correctly
-        if ($this->toTruncate)
+        if ($this->toTruncate && $this->position+$len >= $this->streamlength) {
             $len += 4;
+        }
         if ($this->mapistream) {
             $data = mapi_stream_read($this->mapistream, $len);
         }
@@ -165,7 +166,7 @@ class MAPIStreamWrapper {
      * @param int $new_size
      * @return boolean
      */
-    public function stream_truncate ($new_size) {
+    public function stream_truncate($new_size) {
         $this->streamlength = $new_size;
         $this->toTruncate = true;
 

src/lib/core/asdevice.php

@@ -204,7 +204,6 @@ class ASDevice extends StateObject {
         unset($this->forceSave);
         unset($this->newdevice);
         unset($this->ignoredMessageIds);
-        unset($this->backend2folderidCache);
 
         if (isset($this->ignoredmessages) && is_array($this->ignoredmessages)) {
             $imessages = $this->ignoredmessages;

src/lib/syncobjects/syncnote.php

@@ -127,10 +127,12 @@ class SyncNote extends SyncObject {
         // is a color other than yellow set
         if (isset($this->Color) && $this->Color != 3 && $this->Color > -1 && $this->Color < 5) {
             // check existing categories - do not rewrite category if the category is already a supported or unsupported color
-            $insecUnsupp = array_intersect($this->categories, array_values(self::$unsupportedColors));
-            $insecColors = array_intersect($this->categories, array_values(self::$colors));
-            if (!empty($this->categories) && (!empty($insecUnsupp) || !empty($insecColors))) {
-                return false;
+            if (!empty($this->categories)) {
+                $insecUnsupp = array_intersect($this->categories, array_values(self::$unsupportedColors));
+                $insecColors = array_intersect($this->categories, array_values(self::$colors));
+                if (!empty($insecUnsupp) || !empty($insecColors)) {
+                    return false;
+                }
             }
             if(!isset($this->categories)) {
                 $this->categories = array();

src/lib/utils/zpushadmin.php

@@ -106,8 +106,8 @@ class ZPushAdmin {
                 $sc = new SyncCollections();
                 $sc->SetStateManager($stateManager);
 
-                // load all collections of device without loading states, checking permissions or loading the hierarchy
-                $sc->LoadAllCollections(true, false, false, false);
+                // load all collections of device also loading states and loading hierarchy, but not checking permissions
+                $sc->LoadAllCollections(true, true, false, true);
 
                 if ($sc->GetLastSyncTime())
                     $device->SetLastSyncTime($sc->GetLastSyncTime());
@@ -326,7 +326,7 @@ class ZPushAdmin {
             }
 
             if (!$folderid || (is_array($folderid) && empty($folderid))) {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::ResyncFolder(): no folders synchronized for user '%s' on device '%s'. Aborting.",$user, $devid));
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::ResyncFolder(): no folders requested for user '%s' on device '%s'. Aborting.",$user, $devid));
                 return false;
             }
 
@@ -342,6 +342,10 @@ class ZPushAdmin {
                 ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::ResyncFolder(): folder '%s' on device '%s' of user '%s' marked to be re-synchronized.", $folderid, $devid, $user));
             }
 
+            if ($device->GetData() === false) {
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::ResyncFolder(): nothing changed for device '%s' of user '%s'", $devid, $user));
+                return false;
+            }
             ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::ResyncFolder(): saved updated device data of device '%s' of user '%s'", $devid, $user));
         }

src/z-push-admin.php

@@ -101,6 +101,7 @@ class ZPushAdminCLI {
     const TYPE_OPTION_TASK = "task";
     const TYPE_OPTION_NOTE = "note";
     const TYPE_OPTION_HIERARCHY = "hierarchy";
+    const TYPE_OPTION_GAB = "gab";
 
     static private $command;
     static private $user = false;
@@ -115,7 +116,6 @@ class ZPushAdminCLI {
      * @access public
      */
     static public function UsageInstructions() {
-        $types = "'".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."' or '".self::TYPE_OPTION_NOTE."'";
         return  "Usage:\n\tz-push-admin.php -a ACTION [options]\n\n" .
                 "Parameters:\n\t-a list/wipe/remove/resync/clearloop\n\t[-u] username\n\t[-d] deviceid\n\n" .
                 "Actions:\n" .
@@ -130,14 +130,16 @@ class ZPushAdminCLI {
                 "\tremove -d DEVICE\t\t\t Removes all state data of all users synchronized on device DEVICE\n" .
                 "\tremove -u USER -d DEVICE\t\t Removes all related state data of device DEVICE of user USER\n" .
                 "\tresync -u USER -d DEVICE\t\t Resynchronizes all data of device DEVICE of user USER\n" .
-                "\tresync -t TYPE \t\t\t\t Resynchronizes all folders of type $types for all devices and users.\n" .
-                "\tresync -t TYPE -u USER \t\t\t Resynchronizes all folders of type $types for the user USER.\n" .
-                "\tresync -t TYPE -u USER -d DEVICE\t Resynchronizes all folders of type $types for a specified device and user.\n" .
+                "\tresync -t TYPE \t\t\t\t Resynchronizes all folders of type (possible values below) for all devices and users.\n" .
+                "\tresync -t TYPE -u USER \t\t\t Resynchronizes all folders of type (possible values below) for the user USER.\n" .
+                "\tresync -t TYPE -u USER -d DEVICE\t Resynchronizes all folders of type (possible values below) for a specified device and user.\n" .
                 "\tresync -t FOLDERID -u USER\t\t Resynchronize the specified folder id only. The USER should be specified for better performance.\n" .
                 "\tresync -t hierarchy -u USER -d DEVICE\t Resynchronize the folder hierarchy data for an optional USER and optional DEVICE.\n" .
                 "\tclearloop\t\t\t\t Clears system wide loop detection data\n" .
                 "\tclearloop -d DEVICE -u USER\t\t Clears all loop detection data of a device DEVICE and an optional user USER\n" .
                 "\tfixstates\t\t\t\t Checks the states for integrity and fixes potential issues\n" .
+                "\n" .
+                "\tPossible values for type:\n\t  '".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."', '".self::TYPE_OPTION_NOTE."', '".self::TYPE_OPTION_HIERARCHY."' of '".self::TYPE_OPTION_GAB."' (for KOE)\n" .
                 "\n";
     }
 
@@ -200,11 +202,13 @@ class ZPushAdminCLI {
                 self::$type !== self::TYPE_OPTION_TASK &&
                 self::$type !== self::TYPE_OPTION_NOTE &&
                 self::$type !== self::TYPE_OPTION_HIERARCHY &&
+                self::$type !== self::TYPE_OPTION_GAB &&
+                strlen(self::$type) !== 6 &&       // like U1f38d
                 strlen(self::$type) !== 44 &&
                 strlen(self::$type) !== 48) {
                     self::$errormessage = "Wrong 'type'. Possible values are: ".
-                        "'".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."', '".self::TYPE_OPTION_NOTE."', ".self::TYPE_OPTION_HIERARCHY."' ".
-                        "or a 44 or 48 byte long folder id (as hex).";
+                        "'".self::TYPE_OPTION_EMAIL."', '".self::TYPE_OPTION_CALENDAR."', '".self::TYPE_OPTION_CONTACT."', '".self::TYPE_OPTION_TASK."', '".self::TYPE_OPTION_NOTE."', '".self::TYPE_OPTION_HIERARCHY."', '".self::TYPE_OPTION_GAB."' ".
+                        "or a 6, 44 or 48 byte long folder id (as hex).";
                     return;
                 }
         }
@@ -632,9 +636,27 @@ class ZPushAdminCLI {
         }
 
         $folders = array();
+        $searchFor = $type;
+        // get the KOE gab folderid
+        if ($type == self::TYPE_OPTION_GAB) {
+            if (@constant('KOE_GAB_FOLDERID') !== '') {
+                $gab = KOE_GAB_FOLDERID;
+            }
+            else {
+                $gab = $device->GetKoeGabBackendFolderId();
+            }
+            if (!$gab) {
+                printf("Could not find KOE GAB folderid for device '%s' of user '%s'\n", $deviceId, $user);
+                return false;
+            }
+            $searchFor = $gab;
+        }
+        // potential long ids are converted to folderids here, incl. the gab id
+        $searchFor = strtolower($device->GetFolderIdForBackendId($searchFor, false, false, null));
+
         foreach ($device->GetAllFolderIds() as $folderid) {
             // if  submitting a folderid as type to resync a specific folder.
-            if ($folderid == $type || $device->GetFolderBackendId($folderid) === $type) {
+            if (strtolower($folderid) === $searchFor) {
                 printf("Found and resynching requested folderid '%s' on device '%s' of user '%s'\n", $folderid, $deviceId, $user);
                 $folders[] = $folderid;
                 break;
@@ -645,26 +667,26 @@ class ZPushAdminCLI {
                 switch($foldertype) {
                     case SYNC_FOLDER_TYPE_APPOINTMENT:
                     case SYNC_FOLDER_TYPE_USER_APPOINTMENT:
-                        if ($type == "calendar")
+                        if ($searchFor == "calendar")
                             $folders[] = $folderid;
                         break;
                     case SYNC_FOLDER_TYPE_CONTACT:
                     case SYNC_FOLDER_TYPE_USER_CONTACT:
-                        if ($type == "contact")
+                        if ($searchFor == "contact")
                             $folders[] = $folderid;
                         break;
                     case SYNC_FOLDER_TYPE_TASK:
                     case SYNC_FOLDER_TYPE_USER_TASK:
-                        if ($type == "task")
+                        if ($searchFor == "task")
                             $folders[] = $folderid;
                         break;
                     case SYNC_FOLDER_TYPE_NOTE:
                     case SYNC_FOLDER_TYPE_USER_NOTE:
-                        if ($type == "note")
+                        if ($searchFor == "note")
                             $folders[] = $folderid;
                         break;
                     default:
-                        if ($type == "email")
+                        if ($searchFor == "email")
                             $folders[] = $folderid;
                         break;
                 }
@@ -672,7 +694,7 @@ class ZPushAdminCLI {
         }
 
         $stat = ZPushAdmin::ResyncFolder($user, $deviceId, $folders);
-        echo sprintf("Resync of %d folders of type %s on device '%s' of user '%s': %s\n", count($folders), $type, $deviceId, $user, ($stat)?'Requested':ZLog::GetLastMessage(LOGLEVEL_ERROR));
+        echo sprintf("Resync of %d folders of type '%s' on device '%s' of user '%s': %s\n", count($folders), $type, $deviceId, $user, ($stat)?'Requested':ZLog::GetLastMessage(LOGLEVEL_ERROR));
     }
 
     /**
@@ -750,10 +772,16 @@ class ZPushAdminCLI {
             if ($device->GetFolderUUID($folderid)) {
                 $synchedFolders++;
                 $type = $device->GetFolderType($folderid);
+                $name = $device->GetHierarchyCache()->GetFolder($folderid)->displayname;
                 switch($type) {
                     case SYNC_FOLDER_TYPE_APPOINTMENT:
                     case SYNC_FOLDER_TYPE_USER_APPOINTMENT:
-                        $gentype = "Calendars";
+                        if (KOE_GAB_NAME != "" && $name == KOE_GAB_NAME) {
+                            $gentype = "GAB";
+                        }
+                        else {
+                            $gentype = "Calendars";
+                        }
                         break;
                     case SYNC_FOLDER_TYPE_CONTACT:
                     case SYNC_FOLDER_TYPE_USER_CONTACT:
@@ -778,8 +806,7 @@ class ZPushAdminCLI {
                 // set the folder name for all folders which are not fully synchronized yet
                 $fstatus = $device->GetFolderSyncStatus($folderid);
                 if ($fstatus !== false && is_array($fstatus)) {
-                    // TODO would be nice if we could see the real name of the folder, right now we use the folder type as name
-                    $fstatus['name'] = $gentype;
+                    $fstatus['name'] = $name ? $name : $gentype;
                     $device->SetFolderSyncStatus($folderid, $fstatus);
                     $syncedFoldersInProgress++;
                 }
@@ -837,7 +864,10 @@ class ZPushAdminCLI {
                         $percent = round($d['done']*100/$d['total']);
                         $status = sprintf("Status: %s%d%% (%d/%d)", ($percent < 10)?" ":"", $percent, $d['done'], $d['total']);
                     }
-                    printf("\tFolder: %s%s Sync: %s    %s\n", $d['name'], str_repeat(" ", 12-strlen($d['name'])), $d['status'], $status);
+                    if (strlen($d['name']) > 20) {
+                        $d['name'] = substr($d['name'], 0, 18) . "..";
+                    }
+                    printf("\tFolder: %s Sync: %s %s\n", str_pad($d['name'], 20), str_pad($d['status'], 13), $status);
                 }
             }
         }

tools/gab-sync/lib/kopano.php

@@ -549,7 +549,7 @@ class Kopano extends SyncWorker {
 
             $parentfolder = mapi_msgstore_openentry($this->store, $parentfentryid);
             if (!$parentfolder)
-                $this->Terminate(sprintf("Kopano->CreateHiddenPublicFolder(): Error, unable to open parent folder (open entry): 0x%08X", mapi_last_hresult()));
+                $this->Terminate(sprintf("Kopano->getRootFolder(): Error, unable to open parent folder (open entry): 0x%08X", mapi_last_hresult()));
 
             $this->folderCache[$rootId] = $parentfolder;
         }