ZP-832 Add check if stats table can be loaded for shared users - log to

INFO if that does not work, sync folders where no stat can be found only
once a hour, include PR_CONTENT_COUNT, PR_CONTENT_UNREAD and
PR_DELETED_MSG_COUNT into stat to reflect all kind of changes,
SyncCollection->CheckForChanges() compares folderstats for changes when
initiating, in case of error remove the folderstat to ensure folder is
synchronized next Sync request, remove 'savestate' functionality as it
is not used.

Released under the Affero GNU General Public License (AGPL) version 3.

src/backend/zarafa/mapi/mapitags.php

@@ -1253,4 +1253,5 @@ define('PR_ZC_CONTACT_FOLDER_NAMES'                   ,mapi_prop_tag(PT_MV_TSTRI
 //Properties defined for Z-Push
 define('PR_TODO_ITEM_FLAGS'                           ,mapi_prop_tag(PT_LONG,        0x0E2B));
 define('PR_LOCAL_COMMIT_TIME_MAX'                     ,mapi_prop_tag(PT_SYSTIME,     0x670A));
+define('PR_DELETED_MSG_COUNT'                         ,mapi_prop_tag(PT_LONG,        0x6640));
 

src/backend/zarafa/zarafa.php

@@ -1300,8 +1300,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
      * @return boolean
      */
     public function HasFolderStats() {
-        // TODO set to true
-        return false;
+        return true;
     }
 
     /**
@@ -1333,18 +1332,28 @@ class BackendZarafa implements IBackend, ISearchProvider {
         if (empty($this->folderStatCache[$user])) {
             // get the store
             $userstore = $this->openMessageStore($user);
-
             $rootfolder = mapi_msgstore_openentry($userstore);
             $hierarchy =  mapi_folder_gethierarchytable($rootfolder, CONVENIENT_DEPTH);
-            $rows = mapi_table_queryallrows($hierarchy, array(PR_SOURCE_KEY, PR_LOCAL_COMMIT_TIME_MAX, PR_DISPLAY_NAME));
-            // TODO this needs to be time()
-            $now = "not set";//time();
+            $rows = mapi_table_queryallrows($hierarchy, array(PR_SOURCE_KEY, PR_LOCAL_COMMIT_TIME_MAX, PR_CONTENT_COUNT, PR_CONTENT_UNREAD, PR_DELETED_MSG_COUNT, PR_DISPLAY_NAME));
+
+            if (count($rows) == 0) {
+                ZLog::Write(LOGLEVEL_INFO, sprintf("ZarafaBackend->GetFolderStat(): could not access folder statistics for user '%s'. Probably missing 'read' permissions on the root folder! Folders of this store will be synchronized ONCE per hour only!", $user));
+            }
+
             foreach($rows as $folder) {
-                $this->folderStatCache[$user][bin2hex($folder[PR_SOURCE_KEY])] = isset($folder[PR_LOCAL_COMMIT_TIME_MAX])? $folder[PR_LOCAL_COMMIT_TIME_MAX] : $now;
+                $commit_time = isset($folder[PR_LOCAL_COMMIT_TIME_MAX])? $folder[PR_LOCAL_COMMIT_TIME_MAX] : "0000000000";
+                $content_count = isset($folder[PR_CONTENT_COUNT])? $folder[PR_CONTENT_COUNT] : -1;
+                $content_unread = isset($folder[PR_CONTENT_UNREAD])? $folder[PR_CONTENT_UNREAD] : -1;
+                $content_deleted = isset($folder[PR_DELETED_MSG_COUNT])? $folder[PR_DELETED_MSG_COUNT] : -1;
+
+                $this->folderStatCache[$user][bin2hex($folder[PR_SOURCE_KEY])] = $commit_time ."/". $content_count ."/". $content_unread ."/". $content_deleted;
                 $this->nameCache[bin2hex($folder[PR_SOURCE_KEY])] = $folder[PR_DISPLAY_NAME];
             }
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->GetFolderStat() fetched status information of %d folders for store '%s'", count($this->folderStatCache[$user]), $user));
-            $this->folderStatCache[$user]["now"] = $now;
+            // TODO remove logging
+            foreach($this->folderStatCache[$user] as $fid => $stat) {
+                ZLog::Write(LOGLEVEL_INFO, sprintf("FolderStat: %s %s %s\t%s", $user, $fid, $stat, $this->nameCache[$fid]));
+            }
         }
 
         if (isset($this->folderStatCache[$user][$folderid])) {
@@ -1353,7 +1362,8 @@ class BackendZarafa implements IBackend, ISearchProvider {
             return $this->folderStatCache[$user][$folderid];
         }
         else {
-            return $this->folderStatCache[$user]["now"];
+            // a timestamp that changes once per hour is returned in case there is no data found for this folder. It will be synchronized only once per hour.
+            return gmdate("Y-m-d-H");
         }
     }
 

src/lib/core/synccollections.php

@@ -458,31 +458,43 @@ class SyncCollections implements Iterator {
 
         $pingTracking = new PingTracking();
         $this->changes = array();
-        $changesAvailable = false;
 
         ZPush::GetDeviceManager()->AnnounceProcessAsPush();
         ZPush::GetTopCollector()->AnnounceInformation(sprintf("lifetime %ds", $lifetime), true);
         ZLog::Write(LOGLEVEL_INFO, sprintf("SyncCollections->CheckForChanges(): Waiting for %s changes... (lifetime %d seconds)", (empty($classes))?'policy':'store', $lifetime));
 
         // use changes sink where available
-        $changesSink = false;
-
-        // do not create changessink if there are no folders
-        if (!empty($classes) && ZPush::GetBackend()->HasChangesSink()) {
-            $changesSink = true;
+        $changesSink = ZPush::GetBackend()->HasChangesSink();
 
+        // create changessink and check folder stats if there are folders to Ping
+        if (!empty($classes)) {
             // initialize all possible folders
             foreach ($this->collections as $folderid => $spa) {
                 if ($onlyPingable && $spa->GetPingableFlag() !== true)
                     continue;
 
-                // switch user store if this is a additional folder and initialize sink
-                ZPush::GetBackend()->Setup(ZPush::GetAdditionalSyncFolderStore($folderid));
-                if (! ZPush::GetBackend()->ChangesSinkInitialize($folderid))
-                    throw new StatusException(sprintf("Error initializing ChangesSink for folder id '%s'", $folderid), self::ERROR_WRONG_HIERARCHY);
+                // get the user store if this is a additional folder
+                $store = ZPush::GetAdditionalSyncFolderStore($folderid);
+
+                // initialize sink if no immediate changes were found so far
+                if ($changesSink && empty($this->changes)) {
+                    ZPush::GetBackend()->Setup($store);
+                    if (! ZPush::GetBackend()->ChangesSinkInitialize($folderid))
+                        throw new StatusException(sprintf("Error initializing ChangesSink for folder id '%s'", $folderid), self::ERROR_WRONG_HIERARCHY);
+                }
+
+                // check if the folder stat changed since the last sync, if so generate a change for it
+                if (ZPush::GetBackend()->HasFolderStats() && $spa->HasFolderStat() && ZPush::GetBackend()->GetFolderStat($store, $spa->GetFolderId()) !== $spa->GetFolderStat()) {
+                    $this->changes[$spa->GetFolderId()] = 1;
+                }
             }
         }
 
+        if (!empty($this->changes)) {
+            ZLog::Write(LOGLEVEL_DEBUG, "SyncCollections->CheckForChanges(): Using ChangesSink but found changes verifying the folder stats");
+            return true;
+        }
+
         // wait for changes
         $started = time();
         $endat = time() + $lifetime;
@@ -626,28 +638,23 @@ class SyncCollections implements Iterator {
             if ($ste->getCode() == SYNC_STATUS_FOLDERHIERARCHYCHANGED) {
                 ZLog::Write(LOGLEVEL_WARN, "SyncCollections->CountChange(): exporter can not be re-configured due to state error, emulating change in folder to force Sync.");
                 $this->changes[$folderid] = 1;
+                // make sure this folder is fully synched on next Sync request
+                if($spa->HasFolderStat()) {
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CountChange(): removing folder stat '%s' for folderid '%s'", $spa->GetFolderStat(), $spa->GetFolderId()));
+                    $spa->DelFolderStat();
+                    $this->SaveCollection($spa);
+                }
                 return true;
             }
             throw new StatusException("SyncCollections->CountChange(): exporter can not be re-configured.", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
         }
 
         // start over if exporter can not be configured atm
-        if ($changecount === false )
+        if ($changecount === false)
             ZLog::Write(LOGLEVEL_WARN, "SyncCollections->CountChange(): no changes received from Exporter.");
 
         $this->changes[$folderid] = $changecount;
 
-        if(isset($this->addparms[$folderid]['savestate'])) {
-            try {
-                // Discard any data
-                while(is_array($exporter->Synchronize()));
-                $this->addparms[$folderid]['savestate'] = $exporter->GetState();
-            }
-            catch (StatusException $ste) {
-                throw new StatusException("SyncCollections->CountChange(): could not get new state from exporter", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
-            }
-        }
-
         return ($changecount > 0);
      }
 

src/lib/default/backend.php

@@ -261,7 +261,7 @@ abstract class Backend implements IBackend {
      */
     public function GetFolderStat($store, $folderid) {
         // as this is not implemented, the value returned will be different each time, resulting in a standard exporter setup.
-        return "not implemented-".time();
+        return "not implemented-".gmdate("Y-m-d-H");
     }