ZP-832 Request folder statistics from the backend and setup the exporter

only if the stat indicate that the folder contains changes. 
Per default the statistics are disabled (backend has to enable them). If
disabled, an exporter will be setup for each folder.

Released under the Affero GNU General Public License (AGPL) version 3.

src/backend/zarafa/mapi/mapitags.php

@@ -1252,3 +1252,5 @@ define('PR_ZC_CONTACT_FOLDER_NAMES'                   ,mapi_prop_tag(PT_MV_TSTRI
 
 //Properties defined for Z-Push
 define('PR_TODO_ITEM_FLAGS'                           ,mapi_prop_tag(PT_LONG,        0x0E2B));
+define('PR_LOCAL_COMMIT_TIME_MAX'                     ,mapi_prop_tag(PT_SYSTIME,     0x670A));
+

src/backend/zarafa/zarafa.php

@@ -89,6 +89,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
     private $changesSinkStores;
     private $wastebasket;
     private $addressbook;
+    private $folderStatCache;
 
     // ZCP config parameter for PR_EC_ENABLED_FEATURES / PR_EC_DISABLED_FEATURES
     const ZPUSH_ENABLED = 'mobile';
@@ -115,6 +116,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
         $this->changesSinkStores = array();
         $this->wastebasket = false;
         $this->session = false;
+        $this->folderStatCache = array();
 
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("BackendZarafa using PHP-MAPI version: %s", phpversion("mapi")));
     }
@@ -1291,6 +1293,70 @@ class BackendZarafa implements IBackend, ISearchProvider {
         return $this->storeName;
     }
 
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats() {
+        // TODO set to true
+        return false;
+    }
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid) {
+        list($user, $domain) = Utils::SplitDomainUser($store);
+        if ($user === false) {
+            $user = $this->mainUser;
+        }
+
+        if (!isset($this->folderStatCache[$user])) {
+            $this->folderStatCache[$user] = array();
+        }
+
+        // TODO remove nameCache
+        if (!isset($this->nameCache))
+            $this->nameCache = array();
+
+        // if there is nothing in the cache for a store, load the data for all folders of it
+        if (empty($this->folderStatCache[$user])) {
+            // get the store
+            $userstore = $this->openMessageStore($user);
+
+            $rootfolder = mapi_msgstore_openentry($userstore);
+            $hierarchy =  mapi_folder_gethierarchytable($rootfolder, CONVENIENT_DEPTH);
+            $rows = mapi_table_queryallrows($hierarchy, array(PR_SOURCE_KEY, PR_LOCAL_COMMIT_TIME_MAX, PR_DISPLAY_NAME));
+            // TODO this needs to be time()
+            $now = "not set";//time();
+            foreach($rows as $folder) {
+                $this->folderStatCache[$user][bin2hex($folder[PR_SOURCE_KEY])] = isset($folder[PR_LOCAL_COMMIT_TIME_MAX])? $folder[PR_LOCAL_COMMIT_TIME_MAX] : $now;
+                $this->nameCache[bin2hex($folder[PR_SOURCE_KEY])] = $folder[PR_DISPLAY_NAME];
+            }
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->GetFolderStat() fetched status information of %d folders for store '%s'", count($this->folderStatCache[$user]), $user));
+            $this->folderStatCache[$user]["now"] = $now;
+        }
+
+        if (isset($this->folderStatCache[$user][$folderid])) {
+            // TODO remove nameCache output
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->GetFolderStat() found stat for '%s': %s", $this->nameCache[$folderid], $this->folderStatCache[$user][$folderid]));
+            return $this->folderStatCache[$user][$folderid];
+        }
+        else {
+            return $this->folderStatCache[$user]["now"];
+        }
+    }
+
     /**----------------------------------------------------------------------------------------------------------
      * Private methods
      */

src/lib/core/syncparameters.php

@@ -71,6 +71,7 @@ class SyncParameters extends StateObject {
                                     'contentparameters' => array(),
                                     'foldersynctotal' => false,
                                     'foldersyncremaining' => false,
+                                    'folderstat' => false,
                                 );
 
     /**

src/lib/default/backend.php

@@ -238,6 +238,33 @@ abstract class Backend implements IBackend {
         return $this->GetUserDetails(Request::GetAuthUser());
     }
 
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats() {
+        return false;
+    }
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid) {
+        // as this is not implemented, the value returned will be different each time, resulting in a standard exporter setup.
+        return "not implemented-".time();
+    }
+
+
     /**----------------------------------------------------------------------------------------------------------
      * Protected methods for BackendStorage
      *

src/lib/interface/ibackend.php

@@ -307,4 +307,25 @@ interface IBackend {
      * @return Array
      */
     public function GetCurrentUsername();
+
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats();
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid);
 }

src/lib/request/sync.php

@@ -695,10 +695,31 @@ class Sync extends RequestProcessor {
             $changecount = false;
             $exporter = false;
             $streamimporter = false;
+            $newFolderStat = false;
+            $setupExporter = true;
 
             // TODO we could check against $sc->GetChangedFolderIds() on heartbeat so we do not need to configure all exporter again
             if($status == SYNC_STATUS_SUCCESS && ($sc->GetParameter($spa, "getchanges") || ! $spa->HasSyncKey())) {
 
+                // compare the folder statistics if the backend supports this
+                if (self::$backend->HasFolderStats()) {
+                    // check if the folder stats changed -> if not, don't setup the exporter, there are no changes!
+                    $newFolderStat = self::$backend->GetFolderStat(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId()), $spa->GetFolderId());
+                    if ($newFolderStat === $spa->GetFolderStat()) {
+                        $changecount = 0;
+                        $setupExporter = false;
+                        ZLog::Write(LOGLEVEL_DEBUG, "Sync(): Folder stat from the backend indicates that the folder did not change. Exporter will not run.");
+                    }
+                }
+
+                // no need to run the exporter if the globalwindowsize is already full
+                if ($sc->GetGlobalWindowSize() - $this->globallyExportedItems == 0) {
+                    ZLog::Write(LOGLEVEL_DEBUG, "Sync(): no exporter setup as GlobalWindowSize is full.");
+                    $setupExporter = false;
+                }
+
+                // Do a full Exporter setup if we can't avoid it
+                if ($setupExporter) {
                     //make sure the states are loaded
                     $status = $this->loadStates($sc, $spa, $actiondata);
 
@@ -745,9 +766,10 @@ class Sync extends RequestProcessor {
                                 self::$deviceManager->SetFolderSyncStatus($folderid, DeviceManager::FLD_SYNC_INITIALIZED);
                             }
                         }
-                    else if ($status != SYNC_STATUS_SUCCESS)
+                        else if ($status != SYNC_STATUS_SUCCESS) {
                             self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
-
+                        }
+                    }
                 }
             }
 
@@ -773,13 +795,13 @@ class Sync extends RequestProcessor {
             }
 
             // Fir AS 14.0+ omit output for folder, if there were no incoming or outgoing changes and no Fetch
-            if (Request::GetProtocolVersion() >= 14.0 && ! $spa->HasNewSyncKey() && $changecount == 0 && empty($actiondata["fetchids"]) && $status == SYNC_STATUS_SUCCESS) {
+            if (Request::GetProtocolVersion() >= 14.0 && ! $spa->HasNewSyncKey() && $changecount == 0 && empty($actiondata["fetchids"]) && $status == SYNC_STATUS_SUCCESS && $spa->GetFolderStat() === $newFolderStat) {
                 ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync: No changes found for %s folder id '%s'. Omitting output.", $spa->GetContentClass(), $spa->GetFolderId()));
                 continue;
             }
 
             // there is something to send here, sync folder to output
-            $this->syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status);
+            $this->syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status, $newFolderStat);
 
             // reset status for the next folder
             $status = SYNC_STATUS_SUCCESS;
@@ -833,10 +855,12 @@ class Sync extends RequestProcessor {
      * @param int                   $changecount          Amount of changes expected
      * @param ImportChangesStream   $streamimporter       Output stream
      * @param int                   $status               current status of the folder processing
+     * @param string                $newFolderStat        the new folder stat to be set if everything was exported
+     *
      * @throws StatusException
      * @return int  sync status code
      */
-    private function syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status) {
+    private function syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status, $newFolderStat) {
         $actiondata = $sc->GetParameter($spa, "actiondata");
 
         // send the WBXML start tags (if not happened already)
@@ -983,6 +1007,7 @@ class Sync extends RequestProcessor {
             // send <MoreAvailable/> if there are more changes than fit in the folder windowsize
             if($changecount > $windowSize) {
                 self::$encoder->startTag(SYNC_MOREAVAILABLE, false, true);
+                $spa->DelFolderStat();
             }
         }
 
@@ -1048,8 +1073,10 @@ class Sync extends RequestProcessor {
                 $spa->SetFolderSyncRemaining($changecount);
             }
             // changecount is initialized with 'false', so 0 means no changes!
-            if ($changecount === 0 || ($changecount !== false && $changecount <= $windowSize))
+            if ($changecount === 0 || ($changecount !== false && $changecount <= $windowSize)) {
                 self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_COMPLETED);
+                $spa->SetFolderStat($newFolderStat);
+            }
             else
                 self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_INPROGRESS);
         }