Merging in latest from upstream (ZP/z-push:refs/heads/develop)

* commit 'c42848ed':
  ZP-840 Rename exception variable.
  ZP-840 Renamed exception variable.
  ZP-832 Do comparing only if the folder already has a folder stat, but still get the newFolderStat even if the folder has none yet.
  ZP-840 Catch StateInvalidException when in partial mode.
  ZP-840 Do not throw StateNotFoundException as this messes up Ping. Just treat it as StateInvalid (which is true, FD indicates a state that is not available), do not catch both types in Ping as only StateInvalidException will be thrown.
  ZP-839 Get collection count from SyncCollections, show 'no changes' only if nothing was queued (e.g. new exporter registered), fixed typo showing exceptions.
  ZP-840 Invalidate the folder stats in case the exporter state can not be found, improved PingableFolders(), refactored folder stat invalidated into a private method.
  ZP-840 On StateNotFoundException of the exporter state generate a fake change so Ping indicates the synchronization of the folder.
  ZP-841 Reset folderstat when SyncKey "0" is received.
  ZP-839 One line z-push-top announcement for multi-folder sync.

src/lib/core/synccollections.php

@@ -125,7 +125,7 @@ class SyncCollections implements Iterator {
      *
      * @access public
      * @throws StatusException                  with SyncCollections::ERROR_WRONG_HIERARCHY if permission check fails
-     * @throws StateNotFoundException           if the sync state can not be found ($loadState = true)
+     * @throws StateInvalidException            if the sync state can not be found or relation between states is invalid ($loadState = true)
      * @return boolean
      */
     public function LoadAllCollections($overwriteLoaded = false, $loadState = false, $checkPermissions = false) {
@@ -161,7 +161,7 @@ class SyncCollections implements Iterator {
      *
      * @access public
      * @throws StatusException                  with SyncCollections::ERROR_WRONG_HIERARCHY if permission check fails
-     * @throws StateNotFoundException           if the sync state can not be found ($loadState = true)
+     * @throws StateInvalidException            if the sync state can not be found or relation between states is invalid ($loadState = true)
      * @return boolean
      */
     public function LoadCollection($folderid, $loadState = false, $checkPermissions = false) {
@@ -194,8 +194,21 @@ class SyncCollections implements Iterator {
         $addStatus = $this->AddCollection($spa);
 
         // load the latest known syncstate if requested
-        if ($addStatus && $loadState === true)
+        if ($addStatus && $loadState === true) {
+            try {
                 $this->addparms[$folderid]["state"] = $this->stateManager->GetSyncState($spa->GetLatestSyncKey());
+            }
+            catch (StateNotFoundException $snfe) {
+                // if we can't find the state, first we should try a sync of that folder, so
+                // we generate a fake change, so a sync on this folder is triggered
+                $this->changes[$folderid] = 1;
+
+                // make sure this folder is fully synched on next Sync request
+                $this->invalidateFolderStat($spa);
+
+                return false;
+            }
+        }
 
         return $addStatus;
     }
@@ -286,6 +299,16 @@ class SyncCollections implements Iterator {
         return ! empty($this->collections);
     }
 
+    /**
+     * Indicates the amount of collections loaded.
+     *
+     * @access public
+     * @return int
+     */
+    public function GetCollectionCount() {
+        return count($this->collections);
+    }
+
     /**
      * Add a non-permanent key/value pair for a SyncParameters object
      *
@@ -639,11 +662,8 @@ class SyncCollections implements Iterator {
                 ZLog::Write(LOGLEVEL_WARN, "SyncCollections->CountChange(): exporter can not be re-configured due to state error, emulating change in folder to force Sync.");
                 $this->changes[$folderid] = 1;
                 // make sure this folder is fully synched on next Sync request
-                if($spa->HasFolderStat()) {
-                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CountChange(): removing folder stat '%s' for folderid '%s'", $spa->GetFolderStat(), $spa->GetFolderId()));
-                    $spa->DelFolderStat();
-                    $this->SaveCollection($spa);
-                }
+                $this->invalidateFolderStat($spa);
+
                 return true;
             }
             throw new StatusException("SyncCollections->CountChange(): exporter can not be re-configured.", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
@@ -675,14 +695,13 @@ class SyncCollections implements Iterator {
      * @return boolean
      */
     public function PingableFolders() {
-        $pingable = false;
-
         foreach ($this->collections as $folderid => $spa) {
-            if ($spa->GetPingableFlag() == true)
-                $pingable = true;
+            if ($spa->GetPingableFlag() == true) {
+                return true;
+            }
         }
 
-        return $pingable;
+        return false;
     }
 
     /**
@@ -762,4 +781,22 @@ class SyncCollections implements Iterator {
          if (!isset($this->stateManager))
             $this->stateManager = ZPush::GetDeviceManager()->GetStateManager();
      }
+
+     /**
+      * Remove folder statistics from a SyncParameter object.
+      *
+      * @param SyncParameters $spa
+      *
+      * @access public
+      * @return
+      */
+     private function invalidateFolderStat($spa) {
+         if($spa->HasFolderStat()) {
+             ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->invalidateFolderStat(): removing folder stat '%s' for folderid '%s'", $spa->GetFolderStat(), $spa->GetFolderId()));
+             $spa->DelFolderStat();
+             $this->SaveCollection($spa);
+             return true;
+         }
+         return false;
+     }
 }

src/lib/request/ping.php

@@ -67,24 +67,20 @@ class Ping extends RequestProcessor {
         try {
             $sc->LoadAllCollections(true, true, true);
         }
-        catch (StateNotFoundException $snfex) {
+        catch (StateInvalidException $siex) {
             // if no params are present, indicate to send params, else do hierarchy sync
             if (!$params_present) {
                 $pingstatus = SYNC_PINGSTATUS_FAILINGPARAMS;
-                self::$topCollector->AnnounceInformation("StateNotFoundException: require PingParameters", true);
+                self::$topCollector->AnnounceInformation("StateInvalidException: require PingParameters", true);
             }
             else {
-                $pingstatus = SYNC_PINGSTATUS_FOLDERHIERSYNCREQUIRED;
-                self::$topCollector->AnnounceInformation("StateNotFoundException: require HierarchySync", true);
-            }
-        }
-        catch (StateInvalidException $snfex) {
                 // we do not have a ping status for this, but SyncCollections should have generated fake changes for the folders which are broken
                 $fakechanges = $sc->GetChangedFolderIds();
                 $foundchanges = true;
 
                 self::$topCollector->AnnounceInformation("StateInvalidException: force sync", true);
             }
+        }
         catch (StatusException $stex) {
             $pingstatus = SYNC_PINGSTATUS_FOLDERHIERSYNCREQUIRED;
             self::$topCollector->AnnounceInformation("StatusException: require HierarchySync", true);

src/lib/request/sync.php

@@ -46,6 +46,8 @@ class Sync extends RequestProcessor {
     const ZPUSHIGNORESMS = "ZPISMS";
     private $importer;
     private $globallyExportedItems;
+    private $singleFolder;
+    private $multiFolderInfo;
     private $startTagsSent = false;
     private $startFolderTagSent = false;
 
@@ -64,6 +66,8 @@ class Sync extends RequestProcessor {
         $status = SYNC_STATUS_SUCCESS;
         $wbxmlproblem = false;
         $emptysync = false;
+        $this->singleFolder = true;
+        $this->multiFolderInfo = array();
         $this->globallyExportedItems = 0;
 
 
@@ -107,6 +111,11 @@ class Sync extends RequestProcessor {
                     // read class, synckey and folderid without SyncParameters Object for now
                     $class = $synckey = $folderid = false;
 
+                    // if there are already collections in SyncCollections, this is min. the second folder
+                    if ($sc->HasCollections()) {
+                        $this->singleFolder = false;
+                    }
+
                     //for AS versions < 2.5
                     if(self::$decoder->getElementStartTag(SYNC_FOLDERTYPE)) {
                         $class = self::$decoder->getElementContent();
@@ -151,16 +160,19 @@ class Sync extends RequestProcessor {
                             throw new StateInvalidException("Saved state are not of type SyncParameters");
 
                         // new/resync requested
-                        if ($synckey == "0")
+                        if ($synckey == "0") {
                             $spa->RemoveSyncKey();
+                            $spa->DelFolderStat();
+                        }
                         else if ($synckey !== false)
                             $spa->SetSyncKey($synckey);
                     }
                     catch (StateInvalidException $stie) {
                         $spa = new SyncParameters();
                         $status = SYNC_STATUS_INVALIDSYNCKEY;
-                        self::$topCollector->AnnounceInformation("State invalid - Resync folder", true);
+                        self::$topCollector->AnnounceInformation("State invalid - Resync folder", $this->singleFolder);
                         self::$deviceManager->ForceFolderResync($folderid);
+                        $this->saveMultiFolderInfo("exception", "StateInvalidException");
                     }
 
                     // update folderid.. this might be a new object
@@ -186,7 +198,7 @@ class Sync extends RequestProcessor {
                     $sc->AddParameter($spa, "requested", true);
 
                     if ($spa->HasContentClass())
-                        self::$topCollector->AnnounceInformation(sprintf("%s request", $spa->GetContentClass()), true);
+                        self::$topCollector->AnnounceInformation(sprintf("%s request", $spa->GetContentClass()), $this->singleFolder);
                     else
                         ZLog::Write(LOGLEVEL_WARN, "Not possible to determine class of request. Request did not contain class and apparently there is an issue with the HierarchyCache.");
 
@@ -473,8 +485,10 @@ class Sync extends RequestProcessor {
 
                         if ($status == SYNC_STATUS_SUCCESS && $this->importer !== false) {
                             ZLog::Write(LOGLEVEL_INFO, sprintf("Processed '%d' incoming changes", $nchanges));
-                            if (!$actiondata["fetchids"])
-                                self::$topCollector->AnnounceInformation(sprintf("%d incoming", $nchanges), true);
+                            if (!$actiondata["fetchids"]) {
+                                self::$topCollector->AnnounceInformation(sprintf("%d incoming", $nchanges), $this->singleFolder);
+                                $this->saveMultiFolderInfo("incoming", $nchanges);
+                            }
 
                             try {
                                 // Save the updated state, which is used for the exporter later
@@ -564,13 +578,15 @@ class Sync extends RequestProcessor {
             try {
                 $sc->LoadAllCollections(false, true, true);
             }
-            catch (StateNotFoundException $snfex) {
+            catch (StateInvalidException $siex) {
                 $status = SYNC_STATUS_INVALIDSYNCKEY;
-                self::$topCollector->AnnounceInformation("StateNotFoundException", true);
+                self::$topCollector->AnnounceInformation("StateNotFoundException", $this->singleFolder);
+                $this->saveMultiFolderInfo("exeption", "StateNotFoundException");
             }
             catch (StatusException $stex) {
                $status = SYNC_STATUS_FOLDERHIERARCHYCHANGED;
-               self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+               self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), $this->singleFolder);
+               $this->saveMultiFolderInfo("exeption", "StatusException");
             }
 
             // update a few values
@@ -625,7 +641,8 @@ class Sync extends RequestProcessor {
                     }
                     else {
                         $status = SYNC_STATUS_FOLDERHIERARCHYCHANGED;
-                        self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+                        self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), $this->singleFolder);
+                        $this->saveMultiFolderInfo("exeption", "StatusException");
                     }
                 }
 
@@ -708,10 +725,10 @@ class Sync extends RequestProcessor {
                 }
 
                 // compare the folder statistics if the backend supports this
-                if ($setupExporter && self::$backend->HasFolderStats() && $spa->HasFolderStat()) {
+                if ($setupExporter && self::$backend->HasFolderStats()) {
                     // check if the folder stats changed -> if not, don't setup the exporter, there are no changes!
                     $newFolderStat = self::$backend->GetFolderStat(ZPush::GetAdditionalSyncFolderStore($spa->GetFolderId()), $spa->GetFolderId());
-                    if ($newFolderStat === $spa->GetFolderStat()) {
+                    if ($spa->HasFolderStat() && $newFolderStat === $spa->GetFolderStat()) {
                         $changecount = 0;
                         $setupExporter = false;
                         ZLog::Write(LOGLEVEL_DEBUG, "Sync(): Folder stat from the backend indicates that the folder did not change. Exporter will not run.");
@@ -758,7 +775,8 @@ class Sync extends RequestProcessor {
                         }
 
                         if (! $spa->HasSyncKey()) {
-                            self::$topCollector->AnnounceInformation(sprintf("Exporter registered. %d objects queued.", $changecount), true);
+                            self::$topCollector->AnnounceInformation(sprintf("Exporter registered. %d objects queued.", $changecount), $this->singleFolder);
+                            $this->saveMultiFolderInfo("queued", $changecount);
                             // update folder status as initialized
                             $spa->SetFolderSyncTotal($changecount);
                             $spa->SetFolderSyncRemaining($changecount);
@@ -767,7 +785,8 @@ class Sync extends RequestProcessor {
                             }
                         }
                         else if ($status != SYNC_STATUS_SUCCESS) {
-                            self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+                            self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), $this->singleFolder);
+                            $this->saveMultiFolderInfo("exception", "StatusException");
                         }
                     }
                 }
@@ -816,6 +835,11 @@ class Sync extends RequestProcessor {
         if ($this->startTagsSent)
             self::$encoder->endTag();
 
+        // final top announcement for a multi-folder sync
+        if ($sc->GetCollectionCount() > 1) {
+            self::$topCollector->AnnounceInformation($this->getMultiFolderInfoLine($sc->GetCollectionCount()), true);
+        }
+
         return true;
     }
 
@@ -950,8 +974,10 @@ class Sync extends RequestProcessor {
                 }
             }
 
-            if (!empty($actiondata["fetchids"]))
-                self::$topCollector->AnnounceInformation(sprintf("Fetching %d objects ", count($actiondata["fetchids"])), true);
+            if (!empty($actiondata["fetchids"])) {
+                self::$topCollector->AnnounceInformation(sprintf("Fetching %d objects ", count($actiondata["fetchids"])), $this->singleFolder);
+                $this->saveMultiFolderInfo("fetching", count($actiondata["fetchids"]));
+            }
 
             foreach($actiondata["fetchids"] as $id) {
                 $data = false;
@@ -1066,7 +1092,10 @@ class Sync extends RequestProcessor {
             }
 
             self::$encoder->endTag();
-            self::$topCollector->AnnounceInformation(sprintf("Outgoing %d objects%s", $n, ($n >= $windowSize)?" of ".$changecount:""), true);
+            self::$topCollector->AnnounceInformation(sprintf("Outgoing %d objects%s", $n, ($n >= $windowSize)?" of ".$changecount:""), $this->singleFolder);
+            $this->saveMultiFolderInfo("outgoing", $n);
+            $this->saveMultiFolderInfo("queued", $changecount);
+
             $this->globallyExportedItems += $n;
 
             // update folder status, if there is something set
@@ -1149,11 +1178,14 @@ class Sync extends RequestProcessor {
             }
             catch (StateNotFoundException $snfex) {
                 $status = SYNC_STATUS_INVALIDSYNCKEY;
-                self::$topCollector->AnnounceInformation("StateNotFoundException", true);
+                self::$topCollector->AnnounceInformation("StateNotFoundException", $this->singleFolder);
+                $this->saveMultiFolderInfo("exception", "StateNotFoundException");
             }
             catch (StatusException $stex) {
                $status = $stex->getCode();
-               self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), true);
+               self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), $this->singleFolder);
+               $this->saveMultiFolderInfo("exception", "StateNotFoundException");
+
             }
         }
 
@@ -1366,4 +1398,61 @@ class Sync extends RequestProcessor {
             ZLog::Write(LOGLEVEL_DEBUG, "Sync->importMessage(): message imported");
         }
     }
+
+    /**
+     * Keeps some interesting information about the sync process of several folders.
+     *
+     * @access private
+     * @return
+     */
+    private function saveMultiFolderInfo($key, $value) {
+        if ($key == "incoming" || $key == "outgoing" || $key == "queued" || $key == "fetching") {
+            if (!isset($this->multiFolderInfo[$key])) {
+                $this->multiFolderInfo[$key] = 0;
+            }
+            $this->multiFolderInfo[$key] += $value;
+        }
+        if ($key == "exception") {
+            if (!isset($this->multiFolderInfo[$key])) {
+                $this->multiFolderInfo[$key] = array();
+            }
+            $this->multiFolderInfo[$key][] = $value;
+        }
+    }
+
+    /**
+     * Returns a single string with information about the multi folder synchronization.
+     *
+     * @param int $amountOfFolders
+     *
+     * @access private
+     * @return string
+     */
+    private function getMultiFolderInfoLine($amountOfFolders) {
+        $s = $amountOfFolders . " folders";
+        if (isset($this->multiFolderInfo["incoming"])) {
+            $s .= ": ". $this->multiFolderInfo["incoming"] ." saved";
+        }
+        if (isset($this->multiFolderInfo["outgoing"]) && isset($this->multiFolderInfo["queued"]) && $this->multiFolderInfo["outgoing"] > 0) {
+            $s .= sprintf(": Streamed %d out of %d", $this->multiFolderInfo["outgoing"], $this->multiFolderInfo["queued"]);
+        }
+        else if (!isset($this->multiFolderInfo["outgoing"]) && !isset($this->multiFolderInfo["queued"])) {
+            $s .= ": no changes";
+        }
+        else {
+            if (isset($this->multiFolderInfo["outgoing"])) {
+                $s .= "/".$this->multiFolderInfo["outgoing"] ." streamed";
+            }
+            if (isset($this->multiFolderInfo["queued"])) {
+                $s .= "/".$this->multiFolderInfo["queued"] ." queued";
+            }
+        }
+        if (isset($this->multiFolderInfo["exception"])) {
+            $exceptions = array_count_values($this->multiFolderInfo["exception"]);
+            foreach ($exceptions as $name => $count) {
+                $s .= sprintf("-%s(%d)", $name, $count);
+            }
+        }
+        return $s;
+    }
 }