Merge pull request #535 in ZP/z-push from develop to release/2.3

* commit '0e4a1a39':
  ZP-1221 Make sure authentication during OPTIONS request is done after processing all headers and converting charset encodings.
  ZP-1220 Don't switch stores when processing incoming KOE patch syncs of secondary contact folders.
  ZP-1199 For emails with a lot of HTML 3x the requested truncation on plaintext might not be enough.
  ZP-1199 Fixed variable name holding the length of the stream/string.
  ZP-1199 Utils::Utf8_truncate() needs to check that a string is valid UTF-8 even if it's shorter than the requested length.
  ZP-1219 Fixed whitespace.
  ZP-1219 printwbxml sets WBXML_DEBUGGING flag to use different internal routines.
  ZP-1215 Setup exporter even if GlobalWindowSize is full for folders without a SyncKey (new folders).
  ZP-1212 Randomize folderstat expiration time.
  ZP-1204 Transport capability to KOE.
  ZP-1210 Requre php-mapi-webapp on Fedora, CentOS and RHEL instead of php-mapi.
  ZP-1204 Implement delivery receipt requests.
  ZP-1201 REVERT: Overwrite KOE capabilities only if they are different from the already saved value.
  ZP-1208 Outlook expects GlobalObjId to be base64 encoded (as specified for AS 14).
  ZP-1196 Opening contentstable fails for hidden groups.
  ZP-1201 Overwrite KOE capabilities only if they are different from the already saved value.
  ZP-1201 Also check if arrays are equal.
  ZP-1196 Catch warnings when no GAB data can be retrieved from the server.
  ZP-1196 Catch MAPI_E_INVALID_PARAMETER when a group can not be opened from the addressbook.

build/rpm/z-push.spec

@@ -145,7 +145,11 @@ Z-Push for Kopano meta package
 %package -n %name-kopano-gabsync
 Summary:    GAB sync for Kopano
 Group:      Productivity/Networking/Email/Utilities
+%if 0%{?fedora_version} || 0%{?centos_version} || 0%{?rhel_version}
+Requires:   php-mapi-webapp
+%else
 Requires:   php-mapi
+%endif
 
 %description -n %name-kopano-gabsync
 Synchronizes a Kopano global address book
@@ -154,7 +158,11 @@ Synchronizes a Kopano global address book
 Summary:    GAB sync into a contacts folder for Kopano
 Group:      Productivity/Networking/Email/Utilities
 Requires:   %name-common = %version
+%if 0%{?fedora_version} || 0%{?centos_version} || 0%{?rhel_version}
+Requires:   php-mapi-webapp
+%else
 Requires:   php-mapi
+%endif
 
 %description -n %name-kopano-gab2contacts
 Synchronizes a Kopano global address book into a contacts folder

src/backend/kopano/kopano.php

@@ -551,6 +551,20 @@ class BackendKopano implements IBackend, ISearchProvider {
                         $sendMailProps["sentrepresentingaddt"], $sendMailProps["sentrepresentinsrchk"]));
         }
 
+        // Processing of KOE X-Push-Receipts header - delivery notification: ZP-1204
+        if (defined('KOE_CAPABILITY_RECEIPTS') && KOE_CAPABILITY_RECEIPTS) {
+            // KOE: grep for the Sender header indicating that there are TODOs
+            if (preg_match("/^X-Push-Receipts:\s(.*?)$/im", $sm->mime, $receiptsvalue)) {
+                $receipts = trim($receiptsvalue[1]);
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("KopanoBackend->SendMail(): Receipts '%s' requested by KOE", $receipts));
+
+                // delivery notification requested
+                if (stripos($receipts, 'delivery') !== false) {
+                    $mapiprops[PR_ORIGINATOR_DELIVERY_REPORT_REQUESTED] = true;
+                }
+            }
+        }
+
         if(isset($sm->source->itemid) && $sm->source->itemid) {
             // answering an email in a public/shared folder
             // TODO as the store is setup, we should actually user $this->store instead of $this->defaultstore - nevertheless we need to make sure this store is able to send mail (has an outbox)

src/backend/kopano/mapiprovider.php

@@ -2596,7 +2596,7 @@ class MAPIProvider {
                 if ($bpReturnType == SYNC_BODYPREFERENCE_PLAIN) {
                     ZLog::Write(LOGLEVEL_DEBUG, "MAPIProvider->setMessageBody(): truncated plain-text body requested, stripping all links and images");
                     // Get more data because of the filtering it's most probably going down in size. It's going to be truncated to the correct size below.
-                    $plainbody = stream_get_contents($message->asbody->data, $bpo->GetTruncationSize() * 3);
+                    $plainbody = stream_get_contents($message->asbody->data, $bpo->GetTruncationSize() * 5);
                     $message->asbody->data = StringStreamWrapper::Open(preg_replace('/<http(s){0,1}:\/\/.*?>/i', '', $plainbody));
                 }
 

src/config.php

@@ -311,6 +311,8 @@
     define('KOE_CAPABILITY_SECONDARYCONTACTS', true);
     // Copy WebApp signature into KOE
     define('KOE_CAPABILITY_SIGNATURES', true);
+    // Delivery receipt requests
+    define('KOE_CAPABILITY_RECEIPTS', true);
 
     // To synchronize the GAB KOE, the GAB store and folderid need to be specified.
     // Use the gab-sync script to generate this data. The name needs to

src/index.php

@@ -55,17 +55,17 @@ include_once(ZPUSH_CONFIG);
         if (! Request::HasAuthenticationInfo() || !Request::GetGETUser())
             throw new AuthenticationRequiredException("Access denied. Please send authorisation information");
 
+        ZPush::CheckAdvancedConfig();
+
+        // Process request headers and look for AS headers
+        Request::ProcessHeaders();
+
         // Stop here if this is an OPTIONS request
         if (Request::IsMethodOPTIONS()) {
             RequestProcessor::Authenticate();
             throw new NoPostRequestException("Options request", NoPostRequestException::OPTIONS_REQUEST);
         }
 
-        ZPush::CheckAdvancedConfig();
-
-        // Process request headers and look for AS headers
-        Request::ProcessHeaders();
-
         // Check required GET parameters
         if(Request::IsMethodPOST() && (Request::GetCommandCode() === false || !Request::GetDeviceID() || !Request::GetDeviceType()))
             throw new FatalException("Requested the Z-Push URL without the required GET parameters");

src/lib/core/stateobject.php

@@ -84,7 +84,10 @@ class StateObject implements Serializable {
      */
     public function __set($name, $value) {
         $lname = strtolower($name);
-        if (isset($this->data[$lname]) && is_scalar($value) && !is_array($value) && $this->data[$lname] === $value)
+        if (isset($this->data[$lname]) &&
+                ( (is_scalar($value) && !is_array($value) && $this->data[$lname] === $value) ||
+                  (is_array($value) && is_array($this->data[$lname]) && $this->data[$lname] === $value)
+                ))
             return false;
 
         $this->data[$lname] = $value;

src/lib/request/folderchange.php

@@ -124,12 +124,13 @@ class FolderChange extends RequestProcessor {
                 throw new StatusException("HandleFolderChange() can not proceed as there are unprocessed hierarchy changes", SYNC_FSSTATUS_SERVERERROR);
 
             // any additional folders can not be modified - with exception if they are of type SYNC_FOLDER_TYPE_UNKNOWN (ZP-907)
-            if ($serverid !== false && ZPush::GetAdditionalSyncFolderStore($backendid) && self::$deviceManager->GetFolderTypeFromCacheById($serverid) != SYNC_FOLDER_TYPE_UNKNOWN)
+            if (self::$deviceManager->GetFolderTypeFromCacheById($serverid) != SYNC_FOLDER_TYPE_UNKNOWN && $serverid !== false && ZPush::GetAdditionalSyncFolderStore($backendid))
                 throw new StatusException("HandleFolderChange() can not change additional folders which are configured", SYNC_FSSTATUS_SYSTEMFOLDER);
 
             // switch user store if this this happens inside an additional folder
             // if this is an additional folder the backend has to be setup correctly
-            if (!self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore((($parentBackendId != false) ? $parentBackendId : $backendid))))
+            // backend should also not be switched when type is SYNC_FOLDER_TYPE_UNKNOWN (ZP-1220)
+            if (self::$deviceManager->GetFolderTypeFromCacheById($serverid) != SYNC_FOLDER_TYPE_UNKNOWN && !self::$backend->Setup(ZPush::GetAdditionalSyncFolderStore((($parentBackendId != false) ? $parentBackendId : $backendid))))
                 throw new StatusException(sprintf("HandleFolderChange() could not Setup() the backend for folder id '%s'", (($parentBackendId != false) ? $parentBackendId : $backendid)), SYNC_FSSTATUS_SERVERERROR);
         }
         catch (StateNotFoundException $snfex) {

src/lib/request/request.php

@@ -636,7 +636,7 @@ class Request {
      * Checks the device type if it expects the globalobjid in meeting requests encoded as hex.
      * If it's not the case, globalobjid will be base64 encoded.
      *
-     * WindowsOutlook and iOS device since 9.3 (?) version expect globalobjid to be hex encoded.
+     * iOS device since 9.3 (?) version expect globalobjid to be hex encoded.
      * @see https://jira.z-hub.io/projects/ZP/issues/ZP-1013
      *
      * @access public
@@ -644,9 +644,6 @@ class Request {
      */
     static public function IsGlobalObjIdHexClient() {
         switch (self::GetDeviceType()) {
-            case "WindowsOutlook":
-                ZLog::Write(LOGLEVEL_DEBUG, "Request->IsGlobalObjIdHexClient(): WindowsOutlook");
-                return true;
             case "iPod":
             case "iPad":
             case "iPhone":
@@ -658,6 +655,7 @@ class Request {
         }
         return false;
     }
+
     /**----------------------------------------------------------------------------------------------------------
      * Private stuff
      */

src/lib/request/settings.php

@@ -51,6 +51,7 @@ class Settings extends RequestProcessor {
             if (defined('KOE_CAPABILITY_SENDAS') && KOE_CAPABILITY_SENDAS)                          $cap[] = "sendas";
             if (defined('KOE_CAPABILITY_SECONDARYCONTACTS') && KOE_CAPABILITY_SECONDARYCONTACTS)    $cap[] = "secondarycontacts";
             if (defined('KOE_CAPABILITY_SIGNATURES') && KOE_CAPABILITY_SIGNATURES)                  $cap[] = "signatures";
+            if (defined('KOE_CAPABILITY_RECEIPTS') && KOE_CAPABILITY_RECEIPTS)                      $cap[] = "receipts";
 
             self::$specialHeaders = array();
             self::$specialHeaders[] = "X-Push-Capabilities: ". implode(",",$cap);

src/lib/request/sync.php

@@ -768,8 +768,8 @@ class Sync extends RequestProcessor {
             // TODO we could check against $sc->GetChangedFolderIds() on heartbeat so we do not need to configure all exporter again
             if($status == SYNC_STATUS_SUCCESS && ($sc->GetParameter($spa, "getchanges") || ! $spa->HasSyncKey())) {
 
-                // no need to run the exporter if the globalwindowsize is already full
-                if ($sc->GetGlobalWindowSize() == $this->globallyExportedItems) {
+                // no need to run the exporter if the globalwindowsize is already full - if collection already has a synckey (ZP-1215)
+                if ($sc->GetGlobalWindowSize() == $this->globallyExportedItems && $spa->HasSyncKey()) {
                     ZLog::Write(LOGLEVEL_DEBUG, sprintf("Sync(): no exporter setup for '%s' as GlobalWindowSize is full.", $spa->GetFolderId()));
                     $setupExporter = false;
                 }
@@ -1583,6 +1583,8 @@ class Sync extends RequestProcessor {
 
         $interval = Utils::GetFiltertypeInterval($spa->GetFilterType());
         $timeout = time() + (($interval && $interval < $maxTimeout) ? $interval : $maxTimeout);
+        // randomize timout in 12h
+        $timeout -= rand(0, 43200);
         ZLog::Write(LOGLEVEL_DEBUG, sprintf("Sync()->setFolderStat() on %s: %s expiring %s", $spa->getFolderId(), $newFolderStat, date('Y-m-d H:i:s', $timeout)));
         $spa->SetFolderStatTimeout($timeout);
     }

src/lib/utils/stringstreamwrapper.php

@@ -136,11 +136,11 @@ class StringStreamWrapper {
     public function stream_truncate ($new_size) {
         // cut the string!
         $this->stringstream = Utils::Utf8_truncate($this->stringstream, $new_size);
-        $this->streamlength = strlen($this->stringstream);
+        $this->stringlength = strlen($this->stringstream);
 
-        if ($this->position > $this->streamlength) {
-            ZLog::Write(LOGLEVEL_WARN, sprintf("StringStreamWrapper->stream_truncate(): stream position (%d) ahead of new size of %d. Repositioning pointer to end of stream.", $this->position, $this->streamlength));
-            $this->position = $this->streamlength;
+        if ($this->position > $this->stringlength) {
+            ZLog::Write(LOGLEVEL_WARN, sprintf("StringStreamWrapper->stream_truncate(): stream position (%d) ahead of new size of %d. Repositioning pointer to end of stream.", $this->position, $this->stringlength));
+            $this->position = $this->stringlength;
         }
         return true;
     }

src/lib/utils/utils.php

@@ -388,13 +388,15 @@ class Utils {
         // make sure length is always an interger
         $length = (int)$length;
 
-        if (strlen($string) <= $length)
-            return $string;
+        // if the input string is shorter then the trunction, make sure it's valid UTF-8!
+        if (strlen($string) <= $length) {
+            $length = strlen($string) - 1;
+        }
 
         while($length >= 0) {
-            if ((ord($string[$length]) < 0x80) || (ord($string[$length]) >= 0xC0))
+            if ((ord($string[$length]) < 0x80) || (ord($string[$length]) >= 0xC0)) {
                 return substr($string, 0, $length);
-
+            }
             $length--;
         }
         return "";
@@ -1177,7 +1179,7 @@ class Utils {
     }
 
     /**
-     * Check if the UTF-8 string has ISO-2022-JP esc seq 
+     * Check if the UTF-8 string has ISO-2022-JP esc seq
      * if so, it is ISO-2022-JP, not UTF-8 and convert it into UTF-8
      * string
      *

src/lib/wbxml/wbxmldecoder.php

@@ -320,7 +320,7 @@ class WBXMLDecoder extends WBXMLDefs {
                 // when sending an email with an attachment this single log line (which is never logged in INFO)
                 // requires easily additional 20 MB of RAM. See https://jira.z-hub.io/browse/ZP-1159
                 $messagesize = strlen($el[EN_CONTENT]);
-                if ($messagesize > 10240) {
+                if ($messagesize > 10240 && !defined('WBXML_DEBUGGING')) {
                     $content = substr($el[EN_CONTENT], 0, 10240) . sprintf(" <log message with %d bytes truncated>", $messagesize);
                 }
                 else {
@@ -405,6 +405,21 @@ class WBXMLDecoder extends WBXMLDefs {
      * @return string
      */
     private function getTermStr() {
+        if (defined('WBXML_DEBUGGING') && WBXML_DEBUGGING === true) {
+            $str = "";
+            while (1) {
+                $in = $this->getByte();
+                if ($in == 0) {
+                    break;
+                }
+                else {
+                    $str .= chr($in);
+                }
+            }
+
+            return $str;
+        }
+
         // there is no unlimited "length" for stream_get_line,
         // so we use a huge value for "length" param (1Gb)
         // (0 == PHP_SOCK_CHUNK_SIZE (8192))

tools/dev/printwbxml.php

@@ -35,6 +35,9 @@ include_once('../../src/lib/wbxml/wbxmldefs.php');
 include_once('../../src/lib/wbxml/wbxmldecoder.php');
 include_once('../../src/lib/wbxml/wbxmlencoder.php');
 
+// don't truncate WBXML logs and use slow stream reader
+define('WBXML_DEBUGGING', true);
+
 // minimal definitions & log to stdout overwrite
 define('WBXML_DEBUG', true);
 define("LOGLEVEL_WBXML", "wbxml");

tools/gab-sync/lib/kopano.php

@@ -345,6 +345,10 @@ class Kopano extends SyncWorker {
                                                             PR_EC_AB_HIDDEN,
                                                             PR_DISPLAY_TYPE_EX
                                                     ));
+        if(!is_array($gabentries)) {
+            $this->Log("Kopano->GetGAB(): GAB data can not be retrieved.");
+            return $data;
+        }
         foreach ($gabentries as $entry) {
             // do not add SYSTEM user to the GAB
             if (strtoupper($entry[PR_DISPLAY_NAME]) == "SYSTEM") {
@@ -372,7 +376,12 @@ class Kopano extends SyncWorker {
             if (array_key_exists($entry[PR_ACCOUNT], $groups)) {
                 $a->type = GABEntry::GROUP;
                 $groupentry = mapi_ab_openentry($addrbook, $entry[PR_ENTRYID]);
-                $grouptable = mapi_folder_getcontentstable($groupentry, MAPI_DEFERRED_ERRORS);
+                $grouptable = @mapi_folder_getcontentstable($groupentry, MAPI_DEFERRED_ERRORS);
+                // some groups can not be listed - ZP-1196
+                if (mapi_last_hresult()) {
+                    $this->Log(sprintf("Kopano->GetGAB(): Ignoring group '%s' as members can not be listed - possibly hidden, code: 0x%08X \n", $entry[PR_ACCOUNT], mapi_last_hresult() ));
+                    continue;
+                }
                 $users = mapi_table_queryallrows($grouptable, array(PR_ENTRYID, PR_ACCOUNT, PR_SMTP_ADDRESS));
 
                 $a->members = array();

tools/gab-sync/lib/syncworker.php

@@ -128,6 +128,10 @@ abstract class SyncWorker {
             $chunks[$id][$key] = $entry;
         }
 
+        if (empty($chunks)) {
+            $this->Log("No data available");
+            return;
+        }
         $entries = 0;
         $minEntries = $minSize = 99999999999;
         $maxEntries = $maxSize = 0;