Merge pull request #388 in ZP/z-push from develop to release/2.3

* commit 'ba8eb4ec':
  ZP-1054 Use sprintf.
  ZP-1054 Throw UnavailableException if the hash cannot be loaded.
  ZP-1047 Count the elements to be moved only once.
  ZP-1047 Better logging when moving large amounts of messages in one request.
  ZP-1063 Items marked as private are shown in shared folders.
  ZP-1069 Ignore incoming KOE reply flag category update.
  ZP-1060 Before ignoring SyncNote changes make sure they're coming from KOE and the notes capability is enabled.
  ZP-1063 Items marked as private are shown in shared folders.
  ZP-1060 Ignore incoming KOE updates from PatchItem.
  ZP-1052 Attached eml message does not have size.
  ZP-1052 Attached eml message does not have size.
  ZP-1050 The returned data contains a "hasFolderIdMapping" flag now.

src/backend/kopano/mapiphpwrapper.php

@@ -131,6 +131,15 @@ class PHPWrapper {
         $mapimessage = mapi_msgstore_openentry($this->store, $entryid);
         try {
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("PHPWrapper->ImportMessageChange(): Getting message from MAPIProvider, sourcekey: '%s', parentsourcekey: '%s', entryid: '%s'", bin2hex($sourcekey), bin2hex($parentsourcekey), bin2hex($entryid)));
+
+            // do not send private messages from shared folders to the device
+            $sensitivity = mapi_getprops($mapimessage, array(PR_SENSITIVITY));
+            $sharedUser = ZPush::GetAdditionalSyncFolderStore(bin2hex($this->folderid));
+            if ($sharedUser != false && $sharedUser != 'SYSTEM' && isset($sensitivity[PR_SENSITIVITY]) && $sensitivity[PR_SENSITIVITY] >= SENSITIVITY_PRIVATE) {
+                ZLog::Write(LOGLEVEL_DEBUG, "PHPWrapper->ImportMessageChange(): ignoring private message from a shared folder");
+                return SYNC_E_IGNORE;
+            }
+
             $message = $this->mapiprovider->GetMessage($mapimessage, $this->contentparameters);
         }
         catch (SyncObjectBrokenException $mbe) {

src/backend/kopano/mapiprovider.php

@@ -748,6 +748,12 @@ class MAPIProvider {
                     // android devices require attachment size in order to display an attachment properly
                     if (!isset($attachprops[PR_ATTACH_SIZE])) {
                         $stream = mapi_openpropertytostream($mapiattach, PR_ATTACH_DATA_BIN);
+                        // It's not possible to open some (embedded only?) messages, so we need to open the attachment object itself to get the data
+                        if (mapi_last_hresult()) {
+                            $embMessage = mapi_attach_openobj($mapiattach);
+                            $addrbook = $this->getAddressbook();
+                            $stream = mapi_inetmapi_imtoinet($this->session, $addrbook, $embMessage, array('use_tnef' => -1));
+                        }
                         $stat = mapi_stream_stat($stream);
                         $attach->estimatedDataSize = $stat['cb'];
                     }

src/backend/kopano/replybackimexporter.php

@@ -270,6 +270,24 @@ class ReplyBackImExporter implements IImportChanges, IExportChanges {
      * @return boolean
      */
     public function ImportMessageChange($id, $message) {
+        if(ZPush::GetDeviceManager()->IsKoe()) {
+            // Ignore incoming update events of KOE caused by PatchItem - ZP-1060
+            if (KOE_CAPABILITY_NOTES && $id && $message instanceof SyncNote && !isset($message->asbody)) {
+                ZLog::Write(LOGLEVEL_DEBUG, "ReplyBackImExporter->ImportMessageChange(): KOE patch item update. Ignoring incoming update.");
+                return true;
+            }
+            // KOE ZP-990: OL updates the deleted category which causes a race condition if more than one KOE is connected to that user
+            if (KOE_CAPABILITY_RECEIVEFLAGS && $message instanceof SyncMail && !isset($message->flag) && isset($message->categories)) {
+                // check if the categories changed
+                $serverMessage = $this->getMessage($id, false);
+                if((empty($message->categories) && empty($serverMessage->categories)) ||
+                    (is_array($mapiCategories) && count(array_diff($mapiCategories, $message->categories)) == 0 && count(array_diff($message->categories, $mapiCategories)) == 0)) {
+                        ZLog::Write(LOGLEVEL_DEBUG, "ReplyBackImExporter->ImportMessageChange(): KOE update of flag categories. Ignoring incoming update.");
+                        return true;
+                }
+            }
+        }
+
         // data is going to be dropped, inform the user
         if (@constant('READ_ONLY_NOTIFY_LOST_DATA')) {
             try {

src/backend/sqlstatemachine/sqlstatemachine.php

@@ -143,7 +143,7 @@ class SqlStateMachine implements IStateMachine {
      *
      * @access public
      * @return string
-     * @throws StateNotFoundException, StateInvalidException
+     * @throws StateNotFoundException, StateInvalidException, UnavailableException
      */
     public function GetStateHash($devid, $type, $key = null, $counter = false) {
         $hash = null;
@@ -155,8 +155,12 @@ class SqlStateMachine implements IStateMachine {
 
             $record = $sth->fetch(PDO::FETCH_ASSOC);
             if (!$record) {
+                $errCode = $sth->errorCode();
                 $this->clearConnection($this->dbh, $sth, $record);
-                throw new StateNotFoundException("SqlStateMachine->GetStateHash(): Could not locate state");
+                if ($errCode == 'HY000') {
+                    throw new UnavailableException("SqlStateMachine->GetStateHash(): Database not available", $errCode, null, LOGLEVEL_WARN);
+                }
+                throw new StateNotFoundException(sprintf("SqlStateMachine->GetStateHash(): Could not locate state with error code: %s", $errCode));
             }
             else {
                 // datetime->format("U") returns EPOCH

src/lib/core/devicemanager.php

@@ -1001,6 +1001,13 @@ class DeviceManager {
         catch (StateNotFoundException $snfex) {
             $this->hierarchySyncRequired = true;
         }
+        catch (UnavailableException $uaex) {
+            // This is temporary and can be ignored e.g. in PING - see https://jira.z-hub.io/browse/ZP-1054
+            // If the hash was not available before we treat it like a StateNotFoundException.
+            if ($this->deviceHash === false) {
+                $this->hierarchySyncRequired = true;
+            }
+        }
         return true;
     }
 

src/lib/default/filestatemachine.php

@@ -92,7 +92,7 @@ class FileStateMachine implements IStateMachine {
      *
      * @access public
      * @return string
-     * @throws StateNotFoundException, StateInvalidException
+     * @throws StateNotFoundException, StateInvalidException, UnavailableException
      */
     public function GetStateHash($devid, $type, $key = false, $counter = false) {
         $filename = $this->getFullFilePath($devid, $type, $key, $counter);

src/lib/interface/istatemachine.php

@@ -82,7 +82,7 @@ interface IStateMachine {
      *
      * @access public
      * @return string
-     * @throws StateNotFoundException, StateInvalidException
+     * @throws StateNotFoundException, StateInvalidException, UnavailableException
      */
     public function GetStateHash($devid, $type, $key = false, $counter = false);
 

src/lib/request/moveitems.php

@@ -86,7 +86,11 @@ class MoveItems extends RequestProcessor {
 
         self::$encoder->startTag(SYNC_MOVE_MOVES);
 
+        $operationResults = array();
+        $operationCounter = 0;
+        $operationTotal = count($moves);
         foreach($moves as $move) {
+            $operationCounter++;
             self::$encoder->startTag(SYNC_MOVE_RESPONSE);
             self::$encoder->startTag(SYNC_MOVE_SRCMSGID);
             self::$encoder->content($move["srcmsgid"]);
@@ -141,7 +145,15 @@ class MoveItems extends RequestProcessor {
                     $status = $stex->getCode();
             }
 
-            self::$topCollector->AnnounceInformation(sprintf("Operation status: %s", $status), true);
+            if ($operationCounter % 10 == 0) {
+                self::$topCollector->AnnounceInformation(sprintf("Moved %d objects out of %d", $operationCounter, $operationTotal));
+            }
+
+            // save the operation result
+            if (!isset($operationResults[$status])) {
+                $operationResults[$status] = 0;
+            }
+            $operationResults[$status]++;
 
             self::$encoder->startTag(SYNC_MOVE_STATUS);
             self::$encoder->content($status);
@@ -153,6 +165,12 @@ class MoveItems extends RequestProcessor {
             self::$encoder->endTag();
         }
 
+        self::$topCollector->AnnounceInformation(sprintf("Moved %d - Codes", $operationTotal), true);
+        foreach ($operationResults as $status => $occurences) {
+            self::$topCollector->AnnounceInformation(sprintf("%dx%d", $occurences, $status), true);
+        }
+
+
         self::$encoder->endTag();
         return true;
     }

src/lib/utils/zpushadmin.php

@@ -121,6 +121,9 @@ class ZPushAdmin {
                 // get information about the folder synchronization status from SyncCollections
                 $folders = $device->GetAllFolderIds();
 
+                // indicate short folderids
+                $device->hasFolderIdMapping = $device->HasFolderIdMapping();
+
                 foreach ($folders as $folderid) {
                     $fstatus = $device->GetFolderSyncStatus($folderid);