Merging in latest from upstream (ZP/z-push:refs/heads/develop)

* commit '7cdb51e4':
  ZP-832 Run SyncCollections->CheckForChanges() stat comparing only on first run, improved comment.
  ZP-832 Add check if stats table can be loaded for shared users - log to INFO if that does not work, sync folders where no stat can be found only once a hour, include PR_CONTENT_COUNT, PR_CONTENT_UNREAD and PR_DELETED_MSG_COUNT into stat to reflect all kind of changes, SyncCollection->CheckForChanges() compares folderstats for changes when initiating, in case of error remove the folderstat to ensure folder is synchronized next Sync request, remove 'savestate' functionality as it is not used.
  ZP-833 Use http socket as default MAPI_SERVER.
  ZP-834 Remove SINK_FORCERECHECK configuration option and implementation.
  ZP-832 Request folder statistics from the backend and setup the exporter only if the stat indicate that the folder contains changes. Per default the statistics are disabled (backend has to enable them). If disabled, an exporter will be setup for each folder.
  ZP-832 Log GlobalWindow size override only once.
  ZP-818 Omit folder output on unchanged folders for AS 14.0 and higher.
  ZP-831 Combined - typo variable name. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-830 Combined; minor formatting fixes. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-829 CardDAV mappings for phone numbers. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-818 Supress collection containers if there are no changes for a folder, refactored code having the actual export and streaming of data in a private method.

src/backend/carddav/carddav.php

@@ -850,11 +850,11 @@ class BackendCardDAV extends BackendDiff implements ISearchProvider {
                         else {
                             $items[$rc][SYNC_GAL_LASTNAME] = "";
                         }
-                        if (isset($contact->business2phonenumber)) {
-                            $items[$rc][SYNC_GAL_PHONE] = $contact->business2phonenumber;
+                        if (isset($contact->businessphonenumber)) {
+                            $items[$rc][SYNC_GAL_PHONE] = $contact->businessphonenumber;
                         }
-                        if (isset($contact->home2phonenumber)) {
-                            $items[$rc][SYNC_GAL_HOMEPHONE] = $contact->home2phonenumber;
+                        if (isset($contact->homephonenumber)) {
+                            $items[$rc][SYNC_GAL_HOMEPHONE] = $contact->homephonenumber;
                         }
                         if (isset($contact->mobilephonenumber)) {
                             $items[$rc][SYNC_GAL_MOBILEPHONE] = $contact->mobilephonenumber;

src/backend/combined/combined.php

@@ -533,21 +533,9 @@ class BackendCombined extends Backend implements ISearchProvider {
         $pos = strpos($folderid, $this->config['delimiter']);
         if($pos === false)
             return false;
-        return substr($folderid,0,$pos);
+        return substr($folderid, 0, $pos);
     }
 
-    /**
-     * Returns the BackendCombined as it implements the ISearchProvider interface
-     * This could be overwritten by the global configuration
-     *
-     * @access public
-     * @return object       Implementation of ISearchProvider
-     */
-    public function GetSearchProvider() {
-        return $this;
-    }
-
-
     /**
      * Indicates which AS version is supported by the backend.
      * Return the lowest version supported by the backends used.
@@ -566,6 +554,17 @@ class BackendCombined extends Backend implements ISearchProvider {
         return $version;
     }
 
+    /**
+     * Returns the BackendCombined as it implements the ISearchProvider interface
+     * This could be overwritten by the global configuration
+     *
+     * @access public
+     * @return object       Implementation of ISearchProvider
+     */
+    public function GetSearchProvider() {
+        return $this;
+    }
+
 
     /*-----------------------------------------------------------------------------------------
     -- ISearchProvider
@@ -688,5 +687,4 @@ class BackendCombined extends Backend implements ISearchProvider {
 
         return false;
     }
-
 }

src/backend/combined/exporter.php

@@ -173,7 +173,7 @@ class ExportChangesCombined implements IExportChanges {
     public function InitializeExporter(&$importer) {
         ZLog::Write(LOGLEVEL_DEBUG, "ExportChangesCombined->InitializeExporter(...)");
         foreach ($this->exporters as $i => $e) {
-            if(!isset($this->_importwraps[$i])){
+            if(!isset($this->importwraps[$i])){
                 $this->importwraps[$i] = new ImportHierarchyChangesCombinedWrap($i, $this->backend, $importer);
             }
             $e->InitializeExporter($this->importwraps[$i]);

src/backend/combined/importer.php

@@ -169,7 +169,7 @@ class ImportChangesCombined implements IImportChanges {
     public function ImportFolderChange($folder) {
         $id = $folder->serverid;
         $parent = $folder->parentid;
-        ZLog::Write(LOGLEVEL_DEBUG, "ImportChangesCombined->ImportFolderChange() ".print_r($folder, 1));
+        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ImportChangesCombined->ImportFolderChange() id: '%s', parent: '%s'", $id, $parent));
         if($parent == '0') {
             if($id) {
                 $backendid = $this->backend->GetBackendId($id);

src/backend/zarafa/config.php

@@ -2,11 +2,11 @@
 /***********************************************
 * File      :   config.php
 * Project   :   Z-Push
-* Descr     :   VCardDir backend configuration file
+* Descr     :   Zarafa backend configuration file
 *
 * Created   :   27.11.2012
 *
-* Copyright 2007 - 2013 Zarafa Deutschland GmbH
+* Copyright 2007 - 2016 Zarafa Deutschland GmbH
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
@@ -46,4 +46,4 @@
 // ************************
 
 // Defines the server to which we want to connect
-define('MAPI_SERVER', 'file:///var/run/zarafa');
+define('MAPI_SERVER', 'http://127.0.0.1:236/zarafa');

src/backend/zarafa/mapi/mapitags.php

@@ -1252,3 +1252,6 @@ define('PR_ZC_CONTACT_FOLDER_NAMES'                   ,mapi_prop_tag(PT_MV_TSTRI
 
 //Properties defined for Z-Push
 define('PR_TODO_ITEM_FLAGS'                           ,mapi_prop_tag(PT_LONG,        0x0E2B));
+define('PR_LOCAL_COMMIT_TIME_MAX'                     ,mapi_prop_tag(PT_SYSTIME,     0x670A));
+define('PR_DELETED_MSG_COUNT'                         ,mapi_prop_tag(PT_LONG,        0x6640));
+

src/backend/zarafa/zarafa.php

@@ -89,6 +89,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
     private $changesSinkStores;
     private $wastebasket;
     private $addressbook;
+    private $folderStatCache;
 
     // ZCP config parameter for PR_EC_ENABLED_FEATURES / PR_EC_DISABLED_FEATURES
     const ZPUSH_ENABLED = 'mobile';
@@ -115,6 +116,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
         $this->changesSinkStores = array();
         $this->wastebasket = false;
         $this->session = false;
+        $this->folderStatCache = array();
 
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("BackendZarafa using PHP-MAPI version: %s", phpversion("mapi")));
     }
@@ -1316,6 +1318,80 @@ class BackendZarafa implements IBackend, ISearchProvider {
         return $this->storeName;
     }
 
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats() {
+        return true;
+    }
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid) {
+        list($user, $domain) = Utils::SplitDomainUser($store);
+        if ($user === false) {
+            $user = $this->mainUser;
+        }
+
+        if (!isset($this->folderStatCache[$user])) {
+            $this->folderStatCache[$user] = array();
+        }
+
+        // TODO remove nameCache
+        if (!isset($this->nameCache))
+            $this->nameCache = array();
+
+        // if there is nothing in the cache for a store, load the data for all folders of it
+        if (empty($this->folderStatCache[$user])) {
+            // get the store
+            $userstore = $this->openMessageStore($user);
+            $rootfolder = mapi_msgstore_openentry($userstore);
+            $hierarchy =  mapi_folder_gethierarchytable($rootfolder, CONVENIENT_DEPTH);
+            $rows = mapi_table_queryallrows($hierarchy, array(PR_SOURCE_KEY, PR_LOCAL_COMMIT_TIME_MAX, PR_CONTENT_COUNT, PR_CONTENT_UNREAD, PR_DELETED_MSG_COUNT, PR_DISPLAY_NAME));
+
+            if (count($rows) == 0) {
+                ZLog::Write(LOGLEVEL_INFO, sprintf("ZarafaBackend->GetFolderStat(): could not access folder statistics for user '%s'. Probably missing 'read' permissions on the root folder! Folders of this store will be synchronized ONCE per hour only!", $user));
+            }
+
+            foreach($rows as $folder) {
+                $commit_time = isset($folder[PR_LOCAL_COMMIT_TIME_MAX])? $folder[PR_LOCAL_COMMIT_TIME_MAX] : "0000000000";
+                $content_count = isset($folder[PR_CONTENT_COUNT])? $folder[PR_CONTENT_COUNT] : -1;
+                $content_unread = isset($folder[PR_CONTENT_UNREAD])? $folder[PR_CONTENT_UNREAD] : -1;
+                $content_deleted = isset($folder[PR_DELETED_MSG_COUNT])? $folder[PR_DELETED_MSG_COUNT] : -1;
+
+                $this->folderStatCache[$user][bin2hex($folder[PR_SOURCE_KEY])] = $commit_time ."/". $content_count ."/". $content_unread ."/". $content_deleted;
+                $this->nameCache[bin2hex($folder[PR_SOURCE_KEY])] = $folder[PR_DISPLAY_NAME];
+            }
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->GetFolderStat() fetched status information of %d folders for store '%s'", count($this->folderStatCache[$user]), $user));
+            // TODO remove logging
+            foreach($this->folderStatCache[$user] as $fid => $stat) {
+                ZLog::Write(LOGLEVEL_INFO, sprintf("FolderStat: %s %s %s\t%s", $user, $fid, $stat, $this->nameCache[$fid]));
+            }
+        }
+
+        if (isset($this->folderStatCache[$user][$folderid])) {
+            // TODO remove nameCache output
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->GetFolderStat() found stat for '%s': %s", $this->nameCache[$folderid], $this->folderStatCache[$user][$folderid]));
+            return $this->folderStatCache[$user][$folderid];
+        }
+        else {
+            // a timestamp that changes once per hour is returned in case there is no data found for this folder. It will be synchronized only once per hour.
+            return gmdate("Y-m-d-H");
+        }
+    }
+
     /**----------------------------------------------------------------------------------------------------------
      * Private methods
      */

src/config.php

@@ -165,11 +165,6 @@
     // a higher value if you have a high load on the server.
     define('PING_INTERVAL', 30);
 
-    // Interval in seconds to force a re-check of potentially missed notifications when
-    // using a changes sink. Default are 300 seconds (every 5 min).
-    // This can also be disabled by setting it to false
-    define('SINK_FORCERECHECK', 300);
-
     // Set the fileas (save as) order for contacts in the webaccess/webapp/outlook.
     // It will only affect new/modified contacts on the mobile which then are synced to the server.
     // Possible values are:

src/lib/core/synccollections.php

@@ -70,6 +70,7 @@ class SyncCollections implements Iterator {
     private $lastSyncTime;
 
     private $waitingTime = 0;
+    private $loggedGlobalWindowSizeOverwrite = false;
 
 
     /**
@@ -368,7 +369,10 @@ class SyncCollections implements Iterator {
         }
 
         if (defined("SYNC_MAX_ITEMS") && SYNC_MAX_ITEMS < $globalWindowSize) {
-            ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->GetGlobalWindowSize() overwriting requested global window size of %d by %d forced in configuration.", $globalWindowSize, SYNC_MAX_ITEMS));
+            if (!$this->loggedGlobalWindowSizeOverwrite) {
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->GetGlobalWindowSize() overwriting requested global window size of %d by %d forced in configuration.", $globalWindowSize, SYNC_MAX_ITEMS));
+                $this->loggedGlobalWindowSizeOverwrite = true;
+            }
             $globalWindowSize = SYNC_MAX_ITEMS;
         }
 
@@ -454,31 +458,43 @@ class SyncCollections implements Iterator {
 
         $pingTracking = new PingTracking();
         $this->changes = array();
-        $changesAvailable = false;
 
         ZPush::GetDeviceManager()->AnnounceProcessAsPush();
         ZPush::GetTopCollector()->AnnounceInformation(sprintf("lifetime %ds", $lifetime), true);
         ZLog::Write(LOGLEVEL_INFO, sprintf("SyncCollections->CheckForChanges(): Waiting for %s changes... (lifetime %d seconds)", (empty($classes))?'policy':'store', $lifetime));
 
         // use changes sink where available
-        $changesSink = false;
-        $forceRealExport = 0;
-        // do not create changessink if there are no folders
-        if (!empty($classes) && ZPush::GetBackend()->HasChangesSink()) {
-            $changesSink = true;
+        $changesSink = ZPush::GetBackend()->HasChangesSink();
 
+        // create changessink and check folder stats if there are folders to Ping
+        if (!empty($classes)) {
             // initialize all possible folders
             foreach ($this->collections as $folderid => $spa) {
                 if ($onlyPingable && $spa->GetPingableFlag() !== true)
                     continue;
 
-                // switch user store if this is a additional folder and initialize sink
-                ZPush::GetBackend()->Setup(ZPush::GetAdditionalSyncFolderStore($folderid));
-                if (! ZPush::GetBackend()->ChangesSinkInitialize($folderid))
-                    throw new StatusException(sprintf("Error initializing ChangesSink for folder id '%s'", $folderid), self::ERROR_WRONG_HIERARCHY);
+                // get the user store if this is a additional folder
+                $store = ZPush::GetAdditionalSyncFolderStore($folderid);
+
+                // initialize sink if no immediate changes were found so far
+                if ($changesSink && empty($this->changes)) {
+                    ZPush::GetBackend()->Setup($store);
+                    if (! ZPush::GetBackend()->ChangesSinkInitialize($folderid))
+                        throw new StatusException(sprintf("Error initializing ChangesSink for folder id '%s'", $folderid), self::ERROR_WRONG_HIERARCHY);
+                }
+
+                // check if the folder stat changed since the last sync, if so generate a change for it (only on first run)
+                if ($this->waitingTime == 0 && ZPush::GetBackend()->HasFolderStats() && $spa->HasFolderStat() && ZPush::GetBackend()->GetFolderStat($store, $spa->GetFolderId()) !== $spa->GetFolderStat()) {
+                    $this->changes[$spa->GetFolderId()] = 1;
+                }
             }
         }
 
+        if (!empty($this->changes)) {
+            ZLog::Write(LOGLEVEL_DEBUG, "SyncCollections->CheckForChanges(): Using ChangesSink but found changes verifying the folder stats");
+            return true;
+        }
+
         // wait for changes
         $started = time();
         $endat = time() + $lifetime;
@@ -523,16 +539,6 @@ class SyncCollections implements Iterator {
 
             // Use changes sink if available
             if ($changesSink) {
-                // in some occasions we do realize a full export to see if there are pending changes
-                // every 5 minutes this is also done to see if there were "missed" notifications
-                if (SINK_FORCERECHECK !== false && $forceRealExport+SINK_FORCERECHECK <= $now) {
-                    if ($this->CountChanges($onlyPingable)) {
-                        ZLog::Write(LOGLEVEL_DEBUG, "SyncCollections->CheckForChanges(): Using ChangesSink but found relevant changes on regular export");
-                        return true;
-                    }
-                    $forceRealExport = $now;
-                }
-
                 ZPush::GetTopCollector()->AnnounceInformation(sprintf("Sink %d/%ds on %s", ($now-$started), $lifetime, $checkClasses));
                 $notifications = ZPush::GetBackend()->ChangesSink($nextInterval);
 
@@ -632,28 +638,23 @@ class SyncCollections implements Iterator {
             if ($ste->getCode() == SYNC_STATUS_FOLDERHIERARCHYCHANGED) {
                 ZLog::Write(LOGLEVEL_WARN, "SyncCollections->CountChange(): exporter can not be re-configured due to state error, emulating change in folder to force Sync.");
                 $this->changes[$folderid] = 1;
+                // make sure this folder is fully synched on next Sync request
+                if($spa->HasFolderStat()) {
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CountChange(): removing folder stat '%s' for folderid '%s'", $spa->GetFolderStat(), $spa->GetFolderId()));
+                    $spa->DelFolderStat();
+                    $this->SaveCollection($spa);
+                }
                 return true;
             }
             throw new StatusException("SyncCollections->CountChange(): exporter can not be re-configured.", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
         }
 
         // start over if exporter can not be configured atm
-        if ($changecount === false )
+        if ($changecount === false)
             ZLog::Write(LOGLEVEL_WARN, "SyncCollections->CountChange(): no changes received from Exporter.");
 
         $this->changes[$folderid] = $changecount;
 
-        if(isset($this->addparms[$folderid]['savestate'])) {
-            try {
-                // Discard any data
-                while(is_array($exporter->Synchronize()));
-                $this->addparms[$folderid]['savestate'] = $exporter->GetState();
-            }
-            catch (StatusException $ste) {
-                throw new StatusException("SyncCollections->CountChange(): could not get new state from exporter", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
-            }
-        }
-
         return ($changecount > 0);
      }
 

src/lib/core/syncparameters.php

@@ -71,6 +71,7 @@ class SyncParameters extends StateObject {
                                     'contentparameters' => array(),
                                     'foldersynctotal' => false,
                                     'foldersyncremaining' => false,
+                                    'folderstat' => false,
                                 );
 
     /**

src/lib/core/zpush.php

@@ -301,12 +301,6 @@ class ZPush {
         if (!is_array($specialLogUsers))
             throw new FatalMisconfigurationException("The WBXML log users is not an array.");
 
-        if (!defined('SINK_FORCERECHECK')) {
-            define('SINK_FORCERECHECK', 300);
-        }
-        else if (SINK_FORCERECHECK !== false && (!is_int(SINK_FORCERECHECK) || SINK_FORCERECHECK < 1))
-            throw new FatalMisconfigurationException("The SINK_FORCERECHECK value must be 'false' or a number higher than 0.");
-
         if (!defined('SYNC_CONTACTS_MAXPICTURESIZE')) {
             define('SYNC_CONTACTS_MAXPICTURESIZE', 49152);
         }

src/lib/default/backend.php

@@ -238,6 +238,34 @@ abstract class Backend implements IBackend {
         return $this->GetUserDetails(Request::GetAuthUser());
     }
 
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats() {
+        return false;
+    }
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid) {
+        // As this is not implemented, the value returned will change every hour.
+        // This will only be called if HasFolderStats() returns true.
+        return "not implemented-".gmdate("Y-m-d-H");
+    }
+
+
     /**----------------------------------------------------------------------------------------------------------
      * Protected methods for BackendStorage
      *

src/lib/interface/ibackend.php

@@ -307,4 +307,25 @@ interface IBackend {
      * @return Array
      */
     public function GetCurrentUsername();
+
+    /**
+     * Indicates if the Backend supports folder statistics.
+     *
+     * @access public
+     * @return boolean
+     */
+    public function HasFolderStats();
+
+    /**
+     * Returns a status indication of the folder.
+     * If there are changes in the folder, the returned value must change.
+     * The returned values are compared with '===' to determine if a folder needs synchronization or not.
+     *
+     * @param string $store         the store where the folder resides
+     * @param string $folderid      the folder id
+     *
+     * @access public
+     * @return string
+     */
+    public function GetFolderStat($store, $folderid);
 }

src/lib/request/sync.php

@@ -45,6 +45,9 @@ class Sync extends RequestProcessor {
     // Ignored SMS identifier
     const ZPUSHIGNORESMS = "ZPISMS";
     private $importer;
+    private $globallyExportedItems;
+    private $startTagsSent = false;
+    private $startFolderTagSent = false;
 
     /**
      * Handles the Sync command
@@ -61,7 +64,7 @@ class Sync extends RequestProcessor {
         $status = SYNC_STATUS_SUCCESS;
         $wbxmlproblem = false;
         $emptysync = false;
-        $globallyExportedItems = 0;
+        $this->globallyExportedItems = 0;
 
 
         // check if the hierarchySync was fully completed
@@ -662,378 +665,456 @@ class Sync extends RequestProcessor {
             }
         }
 
-        ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Start Output"));
-
         // Start the output
-        self::$encoder->startWBXML();
-        self::$encoder->startTag(SYNC_SYNCHRONIZE);
-        {
-            // global status
-            // SYNC_COMMONSTATUS_* start with values from 101
-            if ($status != SYNC_COMMONSTATUS_SUCCESS && $status > 100) {
-                self::$encoder->startTag(SYNC_STATUS);
-                    self::$encoder->content($status);
-                self::$encoder->endTag();
-            }
-            else {
-                self::$encoder->startTag(SYNC_FOLDERS);
-                {
-                    foreach($sc as $folderid => $spa) {
-                        // get actiondata
-                        $actiondata = $sc->GetParameter($spa, "actiondata");
-
-                        if ($status == SYNC_STATUS_SUCCESS && (!$spa->GetContentClass() || !$spa->GetFolderId())) {
-                            ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): no content class or folderid found for collection."));
-                            continue;
-                        }
-
-                        if (! $sc->GetParameter($spa, "requested"))
-                            ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): partial sync for folder class '%s' with id '%s'", $spa->GetContentClass(), $spa->GetFolderId()));
+        ZLog::Write(LOGLEVEL_DEBUG, "HandleSync(): Start Output");
+
+        // global status
+        // SYNC_COMMONSTATUS_* start with values from 101
+        if ($status != SYNC_COMMONSTATUS_SUCCESS && $status > 100) {
+            $this->sendStartTags();
+            self::$encoder->startTag(SYNC_STATUS);
+                self::$encoder->content($status);
+            self::$encoder->endTag();
+            return true;
+        }
 
-                        // initialize exporter to get changecount
-                        $changecount = false;
-                        if (isset($exporter))
-                            unset($exporter);
+        // Loop through requested folders
+        foreach($sc as $folderid => $spa) {
+            // get actiondata
+            $actiondata = $sc->GetParameter($spa, "actiondata");
 
-                        // TODO we could check against $sc->GetChangedFolderIds() on heartbeat so we do not need to configure all exporter again
-                        if($status == SYNC_STATUS_SUCCESS && ($sc->GetParameter($spa, "getchanges") || ! $spa->HasSyncKey())) {
+            if ($status == SYNC_STATUS_SUCCESS && (!$spa->GetContentClass() || !$spa->GetFolderId())) {
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): no content class or folderid found for collection."));
+                continue;
+            }
 
-                            //make sure the states are loaded
-                            $status = $this->loadStates($sc, $spa, $actiondata);
+            if (! $sc->GetParameter($spa, "requested"))
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): partial sync for folder class '%s' with id '%s'", $spa->GetContentClass(), $spa->GetFolderId()));
+
+            // initialize exporter to get changecount
+            $changecount = false;
+            $exporter = false;
+            $streamimporter = false;
+            $newFolderStat = false;
+            $setupExporter = true;
+
+            // TODO we could check against $sc->GetChangedFolderIds() on heartbeat so we do not need to configure all exporter again
+            if($status == SYNC_STATUS_SUCCESS && ($sc->GetParameter($spa, "getchanges") || ! $spa->HasSyncKey())) {
+
+                // compare the folder statistics if the backend supports this
+                if (self::$backend->HasFolderStats()) {
+                    // check if the folder stats changed -> if not, don't setup the exporter, there are no changes!
+                    $newFolderStat = self::$backend->GetFolderStat(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId()), $spa->GetFolderId());
+                    if ($newFolderStat === $spa->GetFolderStat()) {
+                        $changecount = 0;
+                        $setupExporter = false;
+                        ZLog::Write(LOGLEVEL_DEBUG, "Sync(): Folder stat from the backend indicates that the folder did not change. Exporter will not run.");
+                    }
+                }
 
-                            if($status == SYNC_STATUS_SUCCESS) {
-                                try {
-                                    // if this is an additional folder the backend has to be setup correctly
-                                    if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId())))
-                                        throw new StatusException(sprintf("HandleSync() could not Setup() the backend for folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_FOLDERHIERARCHYCHANGED);
+                // no need to run the exporter if the globalwindowsize is already full
+                if ($sc->GetGlobalWindowSize() - $this->globallyExportedItems == 0) {
+                    ZLog::Write(LOGLEVEL_DEBUG, "Sync(): no exporter setup as GlobalWindowSize is full.");
+                    $setupExporter = false;
+                }
 
-                                    // Use the state from the importer, as changes may have already happened
-                                    $exporter = self::$backend->GetExporter($spa->GetFolderId());
+                // Do a full Exporter setup if we can't avoid it
+                if ($setupExporter) {
+                    //make sure the states are loaded
+                    $status = $this->loadStates($sc, $spa, $actiondata);
 
-                                    if ($exporter === false)
-                                        throw new StatusException(sprintf("HandleSync() could not get an exporter for folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_FOLDERHIERARCHYCHANGED);
-                                }
-                                catch (StatusException $stex) {
-                                   $status = $stex->getCode();
-                                }
-                                try {
-                                    // Stream the messages directly to the PDA
-                                    $streamimporter = new ImportChangesStream(self::$encoder, ZPush::getSyncObjectFromFolderClass($spa->GetContentClass()));
+                    if($status == SYNC_STATUS_SUCCESS) {
+                        try {
+                            // if this is an additional folder the backend has to be setup correctly
+                            if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId())))
+                                throw new StatusException(sprintf("HandleSync() could not Setup() the backend for folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_FOLDERHIERARCHYCHANGED);
 
-                                    if ($exporter !== false) {
-                                        $exporter->Config($sc->GetParameter($spa, "state"));
-                                        $exporter->ConfigContentParameters($spa->GetCPO());
-                                        $exporter->InitializeExporter($streamimporter);
+                            // Use the state from the importer, as changes may have already happened
+                            $exporter = self::$backend->GetExporter($spa->GetFolderId());
 
-                                        $changecount = $exporter->GetChangeCount();
-                                    }
-                                }
-                                catch (StatusException $stex) {
-                                    if ($stex->getCode() === SYNC_FSSTATUS_CODEUNKNOWN && $spa->HasSyncKey())
-                                        $status = SYNC_STATUS_INVALIDSYNCKEY;
-                                    else
-                                        $status = $stex->getCode();
-                                }
+                            if ($exporter === false)
+                                throw new StatusException(sprintf("HandleSync() could not get an exporter for folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_FOLDERHIERARCHYCHANGED);
+                        }
+                        catch (StatusException $stex) {
+                           $status = $stex->getCode();
+                        }
+                        try {
+                            // Stream the messages directly to the PDA
+                            $streamimporter = new ImportChangesStream(self::$encoder, ZPush::getSyncObjectFromFolderClass($spa->GetContentClass()));
 
-                                if (! $spa->HasSyncKey()) {
-                                    self::$topCollector->AnnounceInformation(sprintf("Exporter registered. %d objects queued.", $changecount), true);
-                                    // update folder status as initialized
-                                    $spa->SetFolderSyncTotal($changecount);
-                                    $spa->SetFolderSyncRemaining($changecount);
-                                    if ($changecount > 0) {
-                                        self::$deviceManager->SetFolderSyncStatus($folderid, DeviceManager::FLD_SYNC_INITIALIZED);
-                                    }
-                                }
-                                else if ($status != SYNC_STATUS_SUCCESS)
-                                    self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+                            if ($exporter !== false) {
+                                $exporter->Config($sc->GetParameter($spa, "state"));
+                                $exporter->ConfigContentParameters($spa->GetCPO());
+                                $exporter->InitializeExporter($streamimporter);
 
+                                $changecount = $exporter->GetChangeCount();
                             }
                         }
-
-                        if (isset($hbinterval) && $changecount == 0 && $status == SYNC_STATUS_SUCCESS) {
-                            ZLog::Write(LOGLEVEL_DEBUG, "No changes found for heartbeat folder. Omitting empty output.");
-                            continue;
-                        }
-
-                        // Get a new sync key to output to the client if any changes have been send by the mobile or a new synckey is to be sent
-                        if (!empty($actiondata["modifyids"]) ||
-                            !empty($actiondata["clientids"]) ||
-                            !empty($actiondata["removeids"]) ||
-                            (! $spa->HasSyncKey() && $status == SYNC_STATUS_SUCCESS)) {
-                                $spa->SetNewSyncKey(self::$deviceManager->GetStateManager()->GetNewSyncKey($spa->GetSyncKey()));
+                        catch (StatusException $stex) {
+                            if ($stex->getCode() === SYNC_FSSTATUS_CODEUNKNOWN && $spa->HasSyncKey())
+                                $status = SYNC_STATUS_INVALIDSYNCKEY;
+                            else
+                                $status = $stex->getCode();
                         }
-                        // get a new synckey only if we did not reach the global limit yet
-                        else {
-                            // when reaching the global limit for changes of all collections, stop processing other collections (ZP-697)
-                            if ($sc->GetGlobalWindowSize() <= $globallyExportedItems) {
-                                ZLog::Write(LOGLEVEL_DEBUG, "Global WindowSize for amount of exported changes reached, omitting output for collection.");
-                                continue;
-                            }
 
-                            // get a new synckey if there are changes are we did not reach the limit yet
+                        if (! $spa->HasSyncKey()) {
+                            self::$topCollector->AnnounceInformation(sprintf("Exporter registered. %d objects queued.", $changecount), true);
+                            // update folder status as initialized
+                            $spa->SetFolderSyncTotal($changecount);
+                            $spa->SetFolderSyncRemaining($changecount);
                             if ($changecount > 0) {
-                                $spa->SetNewSyncKey(self::$deviceManager->GetStateManager()->GetNewSyncKey($spa->GetSyncKey()));
+                                self::$deviceManager->SetFolderSyncStatus($folderid, DeviceManager::FLD_SYNC_INITIALIZED);
                             }
                         }
+                        else if ($status != SYNC_STATUS_SUCCESS) {
+                            self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+                        }
+                    }
+                }
+            }
 
-                        self::$encoder->startTag(SYNC_FOLDER);
+            // Get a new sync key to output to the client if any changes have been send by the mobile or a new synckey is to be sent
+            if (!empty($actiondata["modifyids"]) ||
+                !empty($actiondata["clientids"]) ||
+                !empty($actiondata["removeids"]) ||
+                (! $spa->HasSyncKey() && $status == SYNC_STATUS_SUCCESS)) {
+                    $spa->SetNewSyncKey(self::$deviceManager->GetStateManager()->GetNewSyncKey($spa->GetSyncKey()));
+            }
+            // get a new synckey only if we did not reach the global limit yet
+            else {
+                // when reaching the global limit for changes of all collections, stop processing other collections (ZP-697)
+                if ($sc->GetGlobalWindowSize() <= $this->globallyExportedItems) {
+                    ZLog::Write(LOGLEVEL_DEBUG, "Global WindowSize for amount of exported changes reached, omitting output for collection.");
+                    continue;
+                }
 
-                        if($spa->HasContentClass()) {
-                            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Folder type: %s", $spa->GetContentClass()));
-                            // AS 12.0 devices require content class
-                            if (Request::GetProtocolVersion() < 12.1) {
-                                self::$encoder->startTag(SYNC_FOLDERTYPE);
-                                self::$encoder->content($spa->GetContentClass());
-                                self::$encoder->endTag();
-                            }
-                        }
+                // get a new synckey if there are changes are we did not reach the limit yet
+                if ($changecount > 0) {
+                    $spa->SetNewSyncKey(self::$deviceManager->GetStateManager()->GetNewSyncKey($spa->GetSyncKey()));
+                }
+            }
 
-                        self::$encoder->startTag(SYNC_SYNCKEY);
-                        if($status == SYNC_STATUS_SUCCESS && $spa->HasNewSyncKey())
-                            self::$encoder->content($spa->GetNewSyncKey());
-                        else
-                            self::$encoder->content($spa->GetSyncKey());
-                        self::$encoder->endTag();
-
-                        self::$encoder->startTag(SYNC_FOLDERID);
-                            self::$encoder->content($spa->GetFolderId());
-                        self::$encoder->endTag();
-
-                        self::$encoder->startTag(SYNC_STATUS);
-                            self::$encoder->content($status);
-                        self::$encoder->endTag();
-
-                        // announce failing status to the process loop detection
-                        if ($status !== SYNC_STATUS_SUCCESS)
-                            self::$deviceManager->AnnounceProcessStatus($spa->GetFolderId(), $status);
-
-                        // Output IDs and status for incoming items & requests
-                        if($status == SYNC_STATUS_SUCCESS && (
-                            !empty($actiondata["clientids"]) ||
-                            !empty($actiondata["modifyids"]) ||
-                            !empty($actiondata["removeids"]) ||
-                            !empty($actiondata["fetchids"]) )) {
-
-                            self::$encoder->startTag(SYNC_REPLIES);
-                            // output result of all new incoming items
-                            foreach($actiondata["clientids"] as $clientid => $serverid) {
-                                self::$encoder->startTag(SYNC_ADD);
-                                    self::$encoder->startTag(SYNC_CLIENTENTRYID);
-                                        self::$encoder->content($clientid);
-                                    self::$encoder->endTag();
-                                    if ($serverid) {
-                                        self::$encoder->startTag(SYNC_SERVERENTRYID);
-                                            self::$encoder->content($serverid);
-                                        self::$encoder->endTag();
-                                    }
-                                    self::$encoder->startTag(SYNC_STATUS);
-                                        self::$encoder->content((isset($actiondata["statusids"][$clientid])?$actiondata["statusids"][$clientid]:SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR));
-                                    self::$encoder->endTag();
-                                self::$encoder->endTag();
-                            }
+            // Fir AS 14.0+ omit output for folder, if there were no incoming or outgoing changes and no Fetch
+            if (Request::GetProtocolVersion() >= 14.0 && ! $spa->HasNewSyncKey() && $changecount == 0 && empty($actiondata["fetchids"]) && $status == SYNC_STATUS_SUCCESS && $spa->GetFolderStat() === $newFolderStat) {
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync: No changes found for %s folder id '%s'. Omitting output.", $spa->GetContentClass(), $spa->GetFolderId()));
+                continue;
+            }
 
-                            // loop through modify operations which were not a success, send status
-                            foreach($actiondata["modifyids"] as $serverid) {
-                                if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) {
-                                    self::$encoder->startTag(SYNC_MODIFY);
-                                        self::$encoder->startTag(SYNC_SERVERENTRYID);
-                                            self::$encoder->content($serverid);
-                                        self::$encoder->endTag();
-                                        self::$encoder->startTag(SYNC_STATUS);
-                                            self::$encoder->content($actiondata["statusids"][$serverid]);
-                                        self::$encoder->endTag();
-                                    self::$encoder->endTag();
-                                }
-                            }
+            // there is something to send here, sync folder to output
+            $this->syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status, $newFolderStat);
 
-                            // loop through remove operations which were not a success, send status
-                            foreach($actiondata["removeids"] as $serverid) {
-                                if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) {
-                                    self::$encoder->startTag(SYNC_REMOVE);
-                                        self::$encoder->startTag(SYNC_SERVERENTRYID);
-                                            self::$encoder->content($serverid);
-                                        self::$encoder->endTag();
-                                        self::$encoder->startTag(SYNC_STATUS);
-                                            self::$encoder->content($actiondata["statusids"][$serverid]);
-                                        self::$encoder->endTag();
-                                    self::$encoder->endTag();
-                                }
-                            }
+            // reset status for the next folder
+            $status = SYNC_STATUS_SUCCESS;
+        } // END foreach collection
 
-                            if (!empty($actiondata["fetchids"]))
-                                self::$topCollector->AnnounceInformation(sprintf("Fetching %d objects ", count($actiondata["fetchids"])), true);
+        //SYNC_FOLDERS - only if the starttag was sent
+        if ($this->startFolderTagSent)
+            self::$encoder->endTag();
 
-                            foreach($actiondata["fetchids"] as $id) {
-                                $data = false;
-                                try {
-                                    $fetchstatus = SYNC_STATUS_SUCCESS;
+        //SYNC_SYNCHRONIZE - only if the starttag was sent
+        if ($this->startTagsSent)
+            self::$encoder->endTag();
 
-                                    // if this is an additional folder the backend has to be setup correctly
-                                    if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId())))
-                                        throw new StatusException(sprintf("HandleSync(): could not Setup() the backend to fetch in folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_OBJECTNOTFOUND);
+        return true;
+    }
 
-                                    $data = self::$backend->Fetch($spa->GetFolderId(), $id, $spa->GetCPO());
+    /**
+     * Sends the SYNC_SYNCHRONIZE once per request.
+     *
+     * @access private
+     * @return void
+     */
+    private function sendStartTags() {
+        if ($this->startTagsSent === false) {
+            self::$encoder->startWBXML();
+            self::$encoder->startTag(SYNC_SYNCHRONIZE);
+            $this->startTagsSent = true;
+        }
+    }
 
-                                    // check if the message is broken
-                                    if (ZPush::GetDeviceManager(false) && ZPush::GetDeviceManager()->DoNotStreamMessage($id, $data)) {
-                                        ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): message not to be streamed as requested by DeviceManager, id = %s", $id));
-                                        $fetchstatus = SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR;
-                                    }
-                                }
-                                catch (StatusException $stex) {
-                                   $fetchstatus = $stex->getCode();
-                                }
+    /**
+     * Sends the SYNC_FOLDERS once per request.
+     *
+     * @access private
+     * @return void
+     */
+    private function sendFolderStartTag() {
+        $this->sendStartTags();
+        if ($this->startFolderTagSent === false) {
+            self::$encoder->startTag(SYNC_FOLDERS);
+            $this->startFolderTagSent = true;
+        }
+    }
 
-                                self::$encoder->startTag(SYNC_FETCH);
-                                    self::$encoder->startTag(SYNC_SERVERENTRYID);
-                                        self::$encoder->content($id);
-                                    self::$encoder->endTag();
+    /**
+     * Synchronizes a folder to the output stream. Changes for this folders are expected.
+     *
+     * @param SyncCollections       $sc
+     * @param SyncParameters        $spa
+     * @param IExportChanges        $exporter             Fully configured exporter for this folder
+     * @param int                   $changecount          Amount of changes expected
+     * @param ImportChangesStream   $streamimporter       Output stream
+     * @param int                   $status               current status of the folder processing
+     * @param string                $newFolderStat        the new folder stat to be set if everything was exported
+     *
+     * @throws StatusException
+     * @return int  sync status code
+     */
+    private function syncFolder($sc, $spa, $exporter, $changecount, $streamimporter, $status, $newFolderStat) {
+        $actiondata = $sc->GetParameter($spa, "actiondata");
+
+        // send the WBXML start tags (if not happened already)
+        $this->sendFolderStartTag();
+        self::$encoder->startTag(SYNC_FOLDER);
+
+        if($spa->HasContentClass()) {
+            ZLog::Write(LOGLEVEL_DEBUG, sprintf("Folder type: %s", $spa->GetContentClass()));
+            // AS 12.0 devices require content class
+            if (Request::GetProtocolVersion() < 12.1) {
+                self::$encoder->startTag(SYNC_FOLDERTYPE);
+                self::$encoder->content($spa->GetContentClass());
+                self::$encoder->endTag();
+            }
+        }
 
-                                    self::$encoder->startTag(SYNC_STATUS);
-                                        self::$encoder->content($fetchstatus);
-                                    self::$encoder->endTag();
+        self::$encoder->startTag(SYNC_SYNCKEY);
+        if($status == SYNC_STATUS_SUCCESS && $spa->HasNewSyncKey())
+            self::$encoder->content($spa->GetNewSyncKey());
+        else
+            self::$encoder->content($spa->GetSyncKey());
+        self::$encoder->endTag();
+
+        self::$encoder->startTag(SYNC_FOLDERID);
+        self::$encoder->content($spa->GetFolderId());
+        self::$encoder->endTag();
+
+        self::$encoder->startTag(SYNC_STATUS);
+        self::$encoder->content($status);
+        self::$encoder->endTag();
+
+        // announce failing status to the process loop detection
+        if ($status !== SYNC_STATUS_SUCCESS)
+            self::$deviceManager->AnnounceProcessStatus($spa->GetFolderId(), $status);
+
+        // Output IDs and status for incoming items & requests
+        if($status == SYNC_STATUS_SUCCESS && (
+                !empty($actiondata["clientids"]) ||
+                !empty($actiondata["modifyids"]) ||
+                !empty($actiondata["removeids"]) ||
+                !empty($actiondata["fetchids"]) )) {
+
+            self::$encoder->startTag(SYNC_REPLIES);
+            // output result of all new incoming items
+            foreach($actiondata["clientids"] as $clientid => $serverid) {
+                self::$encoder->startTag(SYNC_ADD);
+                self::$encoder->startTag(SYNC_CLIENTENTRYID);
+                self::$encoder->content($clientid);
+                self::$encoder->endTag();
+                if ($serverid) {
+                    self::$encoder->startTag(SYNC_SERVERENTRYID);
+                    self::$encoder->content($serverid);
+                    self::$encoder->endTag();
+                }
+                self::$encoder->startTag(SYNC_STATUS);
+                self::$encoder->content((isset($actiondata["statusids"][$clientid])?$actiondata["statusids"][$clientid]:SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR));
+                self::$encoder->endTag();
+                self::$encoder->endTag();
+            }
 
-                                    if($data !== false && $status == SYNC_STATUS_SUCCESS) {
-                                        self::$encoder->startTag(SYNC_DATA);
-                                            $data->Encode(self::$encoder);
-                                        self::$encoder->endTag();
-                                    }
-                                    else
-                                        ZLog::Write(LOGLEVEL_WARN, sprintf("Unable to Fetch '%s'", $id));
-                                self::$encoder->endTag();
+            // loop through modify operations which were not a success, send status
+            foreach($actiondata["modifyids"] as $serverid) {
+                if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) {
+                    self::$encoder->startTag(SYNC_MODIFY);
+                    self::$encoder->startTag(SYNC_SERVERENTRYID);
+                    self::$encoder->content($serverid);
+                    self::$encoder->endTag();
+                    self::$encoder->startTag(SYNC_STATUS);
+                    self::$encoder->content($actiondata["statusids"][$serverid]);
+                    self::$encoder->endTag();
+                    self::$encoder->endTag();
+                }
+            }
 
-                            }
-                            self::$encoder->endTag();
-                        }
+            // loop through remove operations which were not a success, send status
+            foreach($actiondata["removeids"] as $serverid) {
+                if (isset($actiondata["statusids"][$serverid]) && $actiondata["statusids"][$serverid] !== SYNC_STATUS_SUCCESS) {
+                    self::$encoder->startTag(SYNC_REMOVE);
+                    self::$encoder->startTag(SYNC_SERVERENTRYID);
+                    self::$encoder->content($serverid);
+                    self::$encoder->endTag();
+                    self::$encoder->startTag(SYNC_STATUS);
+                    self::$encoder->content($actiondata["statusids"][$serverid]);
+                    self::$encoder->endTag();
+                    self::$encoder->endTag();
+                }
+            }
 
-                        if($sc->GetParameter($spa, "getchanges") && $spa->HasFolderId() && $spa->HasContentClass() && $spa->HasSyncKey()) {
-                            $windowSize = self::$deviceManager->GetWindowSize($spa->GetFolderId(), $spa->GetContentClass(), $spa->GetUuid(), $spa->GetUuidCounter(), $changecount);
+            if (!empty($actiondata["fetchids"]))
+                self::$topCollector->AnnounceInformation(sprintf("Fetching %d objects ", count($actiondata["fetchids"])), true);
 
-                            // limit windowSize to the max available limit of the global window size left
-                            $globallyAvailable = $sc->GetGlobalWindowSize() - $globallyExportedItems;
-                            if ($changecount > $globallyAvailable && $windowSize > $globallyAvailable) {
-                                ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Limit window size to %d as the global window size limit will be reached", $globallyAvailable));
-                                $windowSize = $globallyAvailable;
-                            }
-                            // send <MoreAvailable/> if there are more changes than fit in the folder windowsize
-                            if($changecount > $windowSize) {
-                                self::$encoder->startTag(SYNC_MOREAVAILABLE, false, true);
-                            }
-                        }
+            foreach($actiondata["fetchids"] as $id) {
+                $data = false;
+                try {
+                    $fetchstatus = SYNC_STATUS_SUCCESS;
 
-                        // Stream outgoing changes
-                        if($status == SYNC_STATUS_SUCCESS && $sc->GetParameter($spa, "getchanges") == true && $windowSize > 0) {
-                            self::$topCollector->AnnounceInformation(sprintf("Streaming data of %d objects", (($changecount > $windowSize)?$windowSize:$changecount)));
-
-                            // Output message changes per folder
-                            self::$encoder->startTag(SYNC_PERFORM);
-
-                            $n = 0;
-                            WBXMLDecoder::ResetInWhile("syncSynchronize");
-                            while(WBXMLDecoder::InWhile("syncSynchronize")) {
-                                try {
-                                    $progress = $exporter->Synchronize();
-                                    if(!is_array($progress))
-                                        break;
-                                    $n++;
-                                    if ($n % 10 == 0)
-                                        self::$topCollector->AnnounceInformation(sprintf("Streamed data of %d objects out of %d", $n, (($changecount > $windowSize)?$windowSize:$changecount)));
-                                }
-                                catch (SyncObjectBrokenException $mbe) {
-                                    $brokenSO = $mbe->GetSyncObject();
-                                    if (!$brokenSO) {
-                                        ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but broken SyncObject not available. This should be fixed in the backend."));
-                                    }
-                                    else {
-                                        if (!isset($brokenSO->id)) {
-                                            $brokenSO->id = "Unknown ID";
-                                            ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but no ID of object set. This should be fixed in the backend."));
-                                        }
-                                        self::$deviceManager->AnnounceIgnoredMessage($spa->GetFolderId(), $brokenSO->id, $brokenSO);
-                                    }
-                                }
-                                // something really bad happened while exporting changes
-                                catch (StatusException $stex) {
-                                    $status = $stex->getCode();
-                                    // during export we found out that the states should be thrown away (ZP-623)
-                                    if ($status == SYNC_STATUS_INVALIDSYNCKEY) {
-                                        self::$deviceManager->ForceFolderResync($spa->GetFolderId());
-                                        break;
-                                    }
-                                }
+                    // if this is an additional folder the backend has to be setup correctly
+                    if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId())))
+                        throw new StatusException(sprintf("HandleSync(): could not Setup() the backend to fetch in folder id '%s'", $spa->GetFolderId()), SYNC_STATUS_OBJECTNOTFOUND);
 
-                                if($n >= $windowSize) {
-                                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Exported maxItems of messages: %d / %d", $n, $changecount));
-                                    break;
-                                }
-                            }
+                    $data = self::$backend->Fetch($spa->GetFolderId(), $id, $spa->GetCPO());
 
-                            // $progress is not an array when exporting the last message
-                            // so we get the number to display from the streamimporter
-                            if (isset($streamimporter)) {
-                                $n = $streamimporter->GetImportedMessages();
-                            }
+                    // check if the message is broken
+                    if (ZPush::GetDeviceManager(false) && ZPush::GetDeviceManager()->DoNotStreamMessage($id, $data)) {
+                        ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): message not to be streamed as requested by DeviceManager, id = %s", $id));
+                        $fetchstatus = SYNC_STATUS_CLIENTSERVERCONVERSATIONERROR;
+                    }
+                }
+                catch (StatusException $stex) {
+                    $fetchstatus = $stex->getCode();
+                }
 
-                            self::$encoder->endTag();
-                            self::$topCollector->AnnounceInformation(sprintf("Outgoing %d objects%s", $n, ($n >= $windowSize)?" of ".$changecount:""), true);
-                            $globallyExportedItems += $n;
+                self::$encoder->startTag(SYNC_FETCH);
+                self::$encoder->startTag(SYNC_SERVERENTRYID);
+                self::$encoder->content($id);
+                self::$encoder->endTag();
 
-                            // update folder status, if there is something set
-                            if ($spa->GetFolderSyncRemaining() && $changecount > 0) {
-                                $spa->SetFolderSyncRemaining($changecount);
-                            }
-                            // changecount is initialized with 'false', so 0 means no changes!
-                            if ($changecount === 0 || ($changecount !== false && $changecount <= $windowSize))
-                                self::$deviceManager->SetFolderSyncStatus($folderid, DeviceManager::FLD_SYNC_COMPLETED);
-                            else
-                                self::$deviceManager->SetFolderSyncStatus($folderid, DeviceManager::FLD_SYNC_INPROGRESS);
-                        }
+                self::$encoder->startTag(SYNC_STATUS);
+                self::$encoder->content($fetchstatus);
+                self::$encoder->endTag();
 
-                        self::$encoder->endTag();
+                if($data !== false && $status == SYNC_STATUS_SUCCESS) {
+                    self::$encoder->startTag(SYNC_DATA);
+                    $data->Encode(self::$encoder);
+                    self::$encoder->endTag();
+                }
+                else
+                    ZLog::Write(LOGLEVEL_WARN, sprintf("Unable to Fetch '%s'", $id));
+                self::$encoder->endTag();
 
-                        // Save the sync state for the next time
-                        if($spa->HasNewSyncKey()) {
-                            self::$topCollector->AnnounceInformation("Saving state");
+            }
+            self::$encoder->endTag();
+        }
 
-                            try {
-                                if (isset($exporter) && $exporter)
-                                    $state = $exporter->GetState();
+        if($sc->GetParameter($spa, "getchanges") && $spa->HasFolderId() && $spa->HasContentClass() && $spa->HasSyncKey()) {
+            $windowSize = self::$deviceManager->GetWindowSize($spa->GetFolderId(), $spa->GetContentClass(), $spa->GetUuid(), $spa->GetUuidCounter(), $changecount);
 
-                                // nothing exported, but possibly imported - get the importer state
-                                else if ($sc->GetParameter($spa, "state") !== null)
-                                    $state = $sc->GetParameter($spa, "state");
+            // limit windowSize to the max available limit of the global window size left
+            $globallyAvailable = $sc->GetGlobalWindowSize() - $this->globallyExportedItems;
+            if ($changecount > $globallyAvailable && $windowSize > $globallyAvailable) {
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Limit window size to %d as the global window size limit will be reached", $globallyAvailable));
+                $windowSize = $globallyAvailable;
+            }
+            // send <MoreAvailable/> if there are more changes than fit in the folder windowsize
+            if($changecount > $windowSize) {
+                self::$encoder->startTag(SYNC_MOREAVAILABLE, false, true);
+                $spa->DelFolderStat();
+            }
+        }
 
-                                // if a new request without state information (hierarchy) save an empty state
-                                else if (! $spa->HasSyncKey())
-                                    $state = "";
-                            }
-                            catch (StatusException $stex) {
-                               $status = $stex->getCode();
-                            }
+        // Stream outgoing changes
+        if($status == SYNC_STATUS_SUCCESS && $sc->GetParameter($spa, "getchanges") == true && $windowSize > 0 && !!$exporter) {
+            self::$topCollector->AnnounceInformation(sprintf("Streaming data of %d objects", (($changecount > $windowSize)?$windowSize:$changecount)));
 
+            // Output message changes per folder
+            self::$encoder->startTag(SYNC_PERFORM);
 
-                            if (isset($state) && $status == SYNC_STATUS_SUCCESS)
-                                self::$deviceManager->GetStateManager()->SetSyncState($spa->GetNewSyncKey(), $state, $spa->GetFolderId());
-                            else
-                                ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): error saving '%s' - no state information available", $spa->GetNewSyncKey()));
+            $n = 0;
+            WBXMLDecoder::ResetInWhile("syncSynchronize");
+            while(WBXMLDecoder::InWhile("syncSynchronize")) {
+                try {
+                    $progress = $exporter->Synchronize();
+                    if(!is_array($progress))
+                        break;
+                    $n++;
+                    if ($n % 10 == 0)
+                        self::$topCollector->AnnounceInformation(sprintf("Streamed data of %d objects out of %d", $n, (($changecount > $windowSize)?$windowSize:$changecount)));
+                }
+                catch (SyncObjectBrokenException $mbe) {
+                    $brokenSO = $mbe->GetSyncObject();
+                    if (!$brokenSO) {
+                        ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but broken SyncObject not available. This should be fixed in the backend."));
+                    }
+                    else {
+                        if (!isset($brokenSO->id)) {
+                            $brokenSO->id = "Unknown ID";
+                            ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): Catched SyncObjectBrokenException but no ID of object set. This should be fixed in the backend."));
                         }
+                        self::$deviceManager->AnnounceIgnoredMessage($spa->GetFolderId(), $brokenSO->id, $brokenSO);
+                    }
+                }
+                // something really bad happened while exporting changes
+                catch (StatusException $stex) {
+                    $status = $stex->getCode();
+                    // during export we found out that the states should be thrown away (ZP-623)
+                    if ($status == SYNC_STATUS_INVALIDSYNCKEY) {
+                        self::$deviceManager->ForceFolderResync($spa->GetFolderId());
+                        break;
+                    }
+                }
 
-                        // save SyncParameters
-                        if ($status == SYNC_STATUS_SUCCESS && empty($actiondata["fetchids"]))
-                            $sc->SaveCollection($spa);
+                if($n >= $windowSize) {
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("HandleSync(): Exported maxItems of messages: %d / %d", $n, $changecount));
+                    break;
+                }
+            }
 
-                        // reset status for the next folder
-                        $status = SYNC_STATUS_SUCCESS;
+            // $progress is not an array when exporting the last message
+            // so we get the number to display from the streamimporter if it's available
+            if (!!$streamimporter) {
+                $n = $streamimporter->GetImportedMessages();
+            }
 
+            self::$encoder->endTag();
+            self::$topCollector->AnnounceInformation(sprintf("Outgoing %d objects%s", $n, ($n >= $windowSize)?" of ".$changecount:""), true);
+            $this->globallyExportedItems += $n;
 
-                    } // END foreach collection
-                }
-                self::$encoder->endTag(); //SYNC_FOLDERS
+            // update folder status, if there is something set
+            if ($spa->GetFolderSyncRemaining() && $changecount > 0) {
+                $spa->SetFolderSyncRemaining($changecount);
+            }
+            // changecount is initialized with 'false', so 0 means no changes!
+            if ($changecount === 0 || ($changecount !== false && $changecount <= $windowSize)) {
+                self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_COMPLETED);
+                $spa->SetFolderStat($newFolderStat);
             }
+            else
+                self::$deviceManager->SetFolderSyncStatus($spa->GetFolderId(), DeviceManager::FLD_SYNC_INPROGRESS);
         }
-        self::$encoder->endTag(); //SYNC_SYNCHRONIZE
 
-        return true;
+        self::$encoder->endTag();
+
+        // Save the sync state for the next time
+        if($spa->HasNewSyncKey()) {
+            self::$topCollector->AnnounceInformation("Saving state");
+
+            try {
+                if (isset($exporter) && $exporter)
+                    $state = $exporter->GetState();
+
+                // nothing exported, but possibly imported - get the importer state
+                else if ($sc->GetParameter($spa, "state") !== null)
+                    $state = $sc->GetParameter($spa, "state");
+
+                // if a new request without state information (hierarchy) save an empty state
+                else if (! $spa->HasSyncKey())
+                    $state = "";
+            }
+            catch (StatusException $stex) {
+                $status = $stex->getCode();
+            }
+
+
+            if (isset($state) && $status == SYNC_STATUS_SUCCESS)
+                self::$deviceManager->GetStateManager()->SetSyncState($spa->GetNewSyncKey(), $state, $spa->GetFolderId());
+            else
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("HandleSync(): error saving '%s' - no state information available", $spa->GetNewSyncKey()));
+        }
+
+        // save SyncParameters
+        if ($status == SYNC_STATUS_SUCCESS && empty($actiondata["fetchids"]))
+            $sc->SaveCollection($spa);
+
+        return $status;
     }
 
     /**