Merge pull request #365 in ZP/z-push from develop to release/2.3

* commit 'fb12c9f5': (22 commits)
  ZP-1042 Add KOE_CAPABILITY_SENDAS (sendas) to the output of the settings command.
  ZP-1033 Fixed log.
  ZP-1043 Check if folderid and name are set.
  ZP-1033 Implement send-as in Kopano->SendMail(). KOE and serverside send as are supported.
  ZP-1013 modified regexp for getting iOS user agent.
  ZP-1040 Check if ASDevice->GetData returns any data before writing it to disk.
  ZP-1037 Fixed logging.
  ZP-1037 Execute FixStatesAdditionalFolderFlags() when calling z-push-admin fixstates.
  ZP-1037 Check for upgraded profiles that don't have flags. Implement flag fix in 'z-push-admin -a fixstates'.
  ZP-1037 Added flags to OpenSharedFolderAPI in ASDevice, DeviceManager, ZPushAdmin and WebserviceDevice. Added constant DeviceManager::FLD_FLAGS_REPLYASUSER. Renamed private method DeviceManager->getAdditionalSyncFolder() to getAdditionalSyncFolderObject() for better readability, modified checks on ASDevice->EditAdditionalFolder() to allow a flag modification (maintaining the same name), added SyncFolder->Flag as ignored parameter and populate it when calling DeviceManager->GetAdditionalUserSyncFolders().
  ZP-1013 Uppercase hex globalobjid for iOS devices and Outlook.
  ZP-1039 Don't use die() to terminate in error case for gab-sync.
  ZP-1039 Don't use die() to terminate in error case for z-push-admin.
  ZP-1029 add instructions if z-push-admin failed. Released under the Affero GNU General Public License (AGPL) version 3.
  ZP-983 Remove debug log.
  ZP-983 Log dates in WBXML mode and print them in the same format as they would appear in WBXML.
  ZP-983 Retrieve start/end time from MAPI if not set.
  ZP-1013 iOS does not send meeting response.
  ZP-983 Implement all cases of the specified cases of unset start/endtime.
  ZP-983 Allow MAPIProvider->setAppointment() to ignore start+endtime if they are not set.
  ...

build/deb/debian.z-push-common.postinst

@@ -6,6 +6,13 @@
 case "$1" in
     configure)
         /usr/local/sbin/z-push-admin -a fixstates
+        if [ $? -ne 0 ]; then
+            YELLOW='\033[0;33m'
+            NC='\033[0m' # No Color
+            echo -e "${YELLOW}When upgrading from an earlier version you need to run 'z-push-admin -a fixstates'.\n
+It seems the execution on setup has failed (see error above). Please fix the issue and run the command manually.\n
+More information at https://wiki.z-hub.io/x/R4Ea${NC}\n"
+        fi
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

src/backend/kopano/kopano.php

@@ -444,6 +444,36 @@ class BackendKopano implements IBackend, ISearchProvider {
                                             Utils::PrintAsString((isset($sm->source->folderid) ? $sm->source->folderid : false)),
                                             Utils::PrintAsString(($sm->saveinsent)), Utils::PrintAsString(isset($sm->replacemime)) ));
 
+        // Send-As functionality - https://jira.z-hub.io/browse/ZP-908
+        $sendingAsSomeone = false;
+        if (defined('KOE_CAPABILITY_SENDAS') && KOE_CAPABILITY_SENDAS) {
+            $senderEmail = array();
+            // KOE: grep for the Sender header indicating we should send-as
+            // the 'X-Push-Sender-Name' header is not used
+            if (preg_match("/^X-Push-Sender:\s(.*?)$/im", $sm->mime, $senderEmail)) {
+                $sendAsEmail = trim($senderEmail[1]);
+                ZLog::Write(LOGLEVEL_DEBUG, sprintf("KopanoBackend->SendMail(): Send-As '%s' requested by KOE", $sendAsEmail));
+                $sm->mime =  preg_replace("/^From: .*?$/im", "From: ". $sendAsEmail, $sm->mime, 1);
+                $sendingAsSomeone = true;
+            }
+            // serverside Send-As - shared folder with DeviceManager::FLD_FLAGS_REPLYASUSER flag
+            elseif (isset($sm->source->folderid)) {
+                // get the owner of this folder - System is not allowed
+                $sharedUser = ZPush::GetAdditionalSyncFolderStore($sm->source->folderid);
+                if ($sharedUser != false && $sharedUser != 'SYSTEM') {
+                    $folders = ZPush::GetAdditionalSyncFolders();
+                    if (isset($folders[$sm->source->folderid]) && ($folders[$sm->source->folderid]->Flags & DeviceManager::FLD_FLAGS_REPLYASUSER)) {
+                        $sendAs = $this->resolveRecipientGAL($sharedUser, 1);
+                        if (isset($sendAs[0])) {
+                            ZLog::Write(LOGLEVEL_DEBUG, sprintf("KopanoBackend->SendMail(): Server side Send-As activated for shared folder. Sending as '%s'.", $sendAs[0]->emailaddress));
+                            $sm->mime =  preg_replace("/^From: .*?$/im", "From: ". $sendAs[0]->emailaddress, $sm->mime, 1);
+                            $sendingAsSomeone = true;
+                        }
+                    }
+                }
+            }
+        }
+
         // by splitting the message in several lines we can easily grep later
         foreach(preg_split("/((\r)?\n)/", $sm->mime) as $rfc822line)
             ZLog::Write(LOGLEVEL_WBXML, "RFC822: ". $rfc822line);
@@ -507,9 +537,12 @@ class BackendKopano implements IBackend, ISearchProvider {
         // PR_SENT_REPRESENTING_EMAIL_ADDRESS properties and "broken" PR_SENT_REPRESENTING_ENTRYID
         // which results in spooler not being able to send the message.
         // @see http://jira.zarafa.com/browse/ZP-85
-        mapi_deleteprops($mapimessage,
-            array(  $sendMailProps["sentrepresentingname"], $sendMailProps["sentrepresentingemail"], $sendMailProps["representingentryid"],
-                    $sendMailProps["sentrepresentingaddt"], $sendMailProps["sentrepresentinsrchk"]));
+        // If using KOE send-as feature, we keep this properties because they actually are the send-as
+        if (!$sendingAsSomeone) {
+            mapi_deleteprops($mapimessage,
+                array(  $sendMailProps["sentrepresentingname"], $sendMailProps["sentrepresentingemail"], $sendMailProps["representingentryid"],
+                        $sendMailProps["sentrepresentingaddt"], $sendMailProps["sentrepresentinsrchk"]));
+        }
 
         if(isset($sm->source->itemid) && $sm->source->itemid) {
             // answering an email in a public/shared folder

src/backend/kopano/mapiprovider.php

@@ -588,8 +588,15 @@ class MAPIProvider {
             $props = $this->getProps($mapimessage, $meetingrequestproperties);
 
             // Get the GOID
-            if(isset($props[$meetingrequestproperties["goidtag"]]))
-                $message->meetingrequest->globalobjid = base64_encode($props[$meetingrequestproperties["goidtag"]]);
+            if(isset($props[$meetingrequestproperties["goidtag"]])) {
+                // GlobalObjId support was removed in AS 16.0
+                if (Request::IsGlobalObjIdHexClient()) {
+                    $message->meetingrequest->globalobjid = strtoupper(bin2hex($props[$meetingrequestproperties["goidtag"]]));
+                }
+                else {
+                    $message->meetingrequest->globalobjid = base64_encode($props[$meetingrequestproperties["goidtag"]]);
+                }
+            }
 
             // Set Timezone
             if(isset($props[$meetingrequestproperties["timezonetag"]]))
@@ -1185,6 +1192,26 @@ class MAPIProvider {
         else
             $tz = false;
 
+        // start and end time may not be set - try to get them from the existing appointment for further calculation - see https://jira.z-hub.io/browse/ZP-983
+        if (!isset($appointment->starttime) || !isset($appointment->endtime)) {
+            $amapping = MAPIMapping::GetAppointmentMapping();
+            $amapping = $this->getPropIdsFromStrings($amapping);
+            $existingstartendpropsmap = array($amapping["starttime"], $amapping["endtime"]);
+            $existingstartendprops = $this->getProps($mapimessage, $existingstartendpropsmap);
+
+            if (isset($existingstartendprops[$amapping["starttime"]]) && !isset($appointment->starttime)) {
+                $appointment->starttime = $existingstartendprops[$amapping["starttime"]];
+                ZLog::Write(LOGLEVEL_WBXML, sprintf("MAPIProvider->setAppointment(): Parameter 'starttime' was not set, using value from MAPI %d (%s).", $appointment->starttime, gmstrftime("%Y%m%dT%H%M%SZ", $appointment->starttime)));
+            }
+            if (isset($existingstartendprops[$amapping["endtime"]]) && !isset($appointment->endtime)) {
+                $appointment->endtime = $existingstartendprops[$amapping["endtime"]];
+                ZLog::Write(LOGLEVEL_WBXML, sprintf("MAPIProvider->setAppointment(): Parameter 'endtime' was not set, using value from MAPI %d (%s).", $appointment->endtime, gmstrftime("%Y%m%dT%H%M%SZ", $appointment->endtime)));
+            }
+        }
+        if (!isset($appointment->starttime) || !isset($appointment->endtime)) {
+            throw new StatusException("MAPIProvider->setAppointment(): Error, start and/or end time not set and can not be retrieved from MAPI.", SYNC_STATUS_SYNCCANNOTBECOMPLETED);
+        }
+
         //calculate duration because without it some webaccess views are broken. duration is in min
         $localstart = $this->getLocaltimeByTZ($appointment->starttime, $tz);
         $localend = $this->getLocaltimeByTZ($appointment->endtime, $tz);

src/lib/core/asdevice.php

@@ -854,11 +854,18 @@ class ASDevice extends StateObject {
      * @param string    $folderid   the folder id of the additional folder.
      * @param string    $name       the name of the additional folder (has to be unique for all folders on the device).
      * @param string    $type       AS foldertype of SYNC_FOLDER_TYPE_USER_*
+     * @param int       $flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    public function AddAdditionalFolder($store, $folderid, $name, $type) {
+    public function AddAdditionalFolder($store, $folderid, $name, $type, $flags) {
+        // check if a folderid and name were sent
+        if (!$folderid || !$name) {
+            ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->AddAdditionalFolder(): No valid folderid ('%s') or name ('%s') sent. Aborting. ", $folderid, $name));
+            return false;
+        }
+
         // check if type is of a additional user type
         if (!in_array($type, array(SYNC_FOLDER_TYPE_USER_CONTACT, SYNC_FOLDER_TYPE_USER_APPOINTMENT, SYNC_FOLDER_TYPE_USER_TASK, SYNC_FOLDER_TYPE_USER_MAIL, SYNC_FOLDER_TYPE_USER_NOTE, SYNC_FOLDER_TYPE_USER_JOURNAL))) {
             ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->AddAdditionalFolder(): folder can not be added because the specified type '%s' is not a permitted user type.", $type));
@@ -900,6 +907,7 @@ class ASDevice extends StateObject {
                             'folderid'  => $folderid,
                             'name'      => $name,
                             'type'      => $type,
+                            'flags'     => $flags,
                          );
         $this->additionalfolders = $af;
 
@@ -914,11 +922,18 @@ class ASDevice extends StateObject {
      *
      * @param string    $folderid   the folder id of the additional folder.
      * @param string    $name       the name of the additional folder (has to be unique for all folders on the device).
+     * @param int       $flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    public function EditAdditionalFolder($folderid, $name) {
+    public function EditAdditionalFolder($folderid, $name, $flags) {
+        // check if a folderid and name were sent
+        if (!$folderid || !$name) {
+            ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): No valid folderid ('%s') or name ('%s') sent. Aborting. ", $folderid, $name));
+            return false;
+        }
+        
         // check if a folder with this ID is known
         if (!isset($this->additionalfolders[$folderid])) {
             ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): folder can not be edited because there is no folder known with this folder id: '%s'. Add the folder first.", $folderid));
@@ -926,9 +941,9 @@ class ASDevice extends StateObject {
         }
 
         // check if a folder with the new name is already in the list
-        foreach ($this->additionalfolders as $k => $folder) {
-            if ($folder['name'] == $name) {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): folder can not be added because there is already an additional folder with the same name: '%s'", $name));
+        foreach ($this->additionalfolders as $existingFolderid => $folder) {
+            if ($folder['name'] == $name &&  $folderid !== $existingFolderid) {
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): folder can not be edited because there is already an additional folder with the same name: '%s'", $name));
                 return false;
             }
         }
@@ -936,8 +951,8 @@ class ASDevice extends StateObject {
         // check if a folder with the new name is already known on the device (regular folder)
         foreach($this->GetHierarchyCache()->ExportFolders() as $syncedFolderid => $folder) {
             // $folder is a SyncFolder object here
-            if ($folder->displayname == $name) {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): folder can not be added because there is already a folder with the same name synchronized: '%s'", $folderid));
+            if ($folder->displayname == $name && $folderid !== $folder->BackendId && $folderid !== $syncedFolderid) {
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->EditAdditionalFolder(): folder can not be edited because there is already a folder with the same name synchronized: '%s'", $folderid));
                 return false;
             }
         }
@@ -945,6 +960,7 @@ class ASDevice extends StateObject {
         // update the name
         $af = $this->additionalfolders;
         $af[$folderid]['name'] = $name;
+        $af[$folderid]['flags'] = $flags;
         $this->additionalfolders = $af;
 
         return true;
@@ -957,6 +973,11 @@ class ASDevice extends StateObject {
      * @return boolean
      */
     public function RemoveAdditionalFolder($folderid) {
+        // check if a folderid were sent
+        if (!$folderid) {
+            ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->RemoveAdditionalFolder(): No valid folderid ('%s') sent. Aborting. ", $folderid));
+            return false;
+        }
         // check if a folder with this ID is known
         if (!isset($this->additionalfolders[$folderid])) {
             ZLog::Write(LOGLEVEL_ERROR, sprintf("ASDevice->RemoveAdditionalFolder(): folder can not be removed because there is no folder known with this folder id: '%s'", $folderid));

src/lib/core/devicemanager.php

@@ -60,6 +60,8 @@ class DeviceManager {
     const FLD_ORIGIN_SHARED = "S";
     const FLD_ORIGIN_GAB = "G";
 
+    const FLD_FLAGS_REPLYASUSER = 1;
+
     private $device;
     private $deviceHash;
     private $saveDevice;
@@ -493,7 +495,12 @@ class DeviceManager {
     public function GetAdditionalUserSyncFolders() {
         $folders = array();
         foreach($this->device->GetAdditionalFolders() as $df) {
-            $folder = $this->getAdditionalSyncFolder($df['store'], $df['folderid'], $df['name'], $df['type'], DeviceManager::FLD_ORIGIN_SHARED);
+            if (!isset($df['flags'])) {
+                $df['flags'] = 0;
+                ZLog::Write(LOGLEVEL_WARN, sprintf("DeviceManager->GetAdditionalUserSyncFolders(): Additional folder '%s' has no flags. Please run 'z-push-admin -a fixstates' to fix this issue.", $df['name']));
+            }
+
+            $folder = $this->getAdditionalSyncFolderObject($df['store'], $df['folderid'], $df['name'], $df['type'], $df['flags'], DeviceManager::FLD_ORIGIN_SHARED);
             $folders[$folder->BackendId] = $folder;
         }
 
@@ -501,7 +508,7 @@ class DeviceManager {
         if (KOE_CAPABILITY_GAB && $this->IsKoe() && KOE_GAB_STORE != "" && KOE_GAB_NAME != "") {
             // if KOE_GAB_FOLDERID is set, use it
             if (KOE_GAB_FOLDERID != "") {
-                $folder = $this->getAdditionalSyncFolder(KOE_GAB_STORE, KOE_GAB_FOLDERID, KOE_GAB_NAME, SYNC_FOLDER_TYPE_USER_APPOINTMENT, DeviceManager::FLD_ORIGIN_GAB);
+                $folder = $this->getAdditionalSyncFolderObject(KOE_GAB_STORE, KOE_GAB_FOLDERID, KOE_GAB_NAME, SYNC_FOLDER_TYPE_USER_APPOINTMENT, 0, DeviceManager::FLD_ORIGIN_GAB);
                 $folders[$folder->BackendId] = $folder;
             }
             else {
@@ -518,7 +525,7 @@ class DeviceManager {
                     }
 
                     if ($backendGabId) {
-                        $folders[$backendGabId] = $this->getAdditionalSyncFolder(KOE_GAB_STORE, $backendGabId, KOE_GAB_NAME, SYNC_FOLDER_TYPE_USER_APPOINTMENT, DeviceManager::FLD_ORIGIN_GAB);
+                        $folders[$backendGabId] = $this->getAdditionalSyncFolderObject(KOE_GAB_STORE, $backendGabId, KOE_GAB_NAME, SYNC_FOLDER_TYPE_USER_APPOINTMENT, 0, DeviceManager::FLD_ORIGIN_GAB);
                     }
                 }
             }
@@ -1141,12 +1148,13 @@ class DeviceManager {
      * @param string    $folderid
      * @param string    $name
      * @param int       $type
+     * @param int       $flags
      * @param string    $folderOrigin
      *
      * @access private
      * @returns SyncFolder
      */
-    private function getAdditionalSyncFolder($store, $folderid, $name, $type, $folderOrigin) {
+    private function getAdditionalSyncFolderObject($store, $folderid, $name, $type, $flags, $folderOrigin) {
         $folder = new SyncFolder();
         $folder->BackendId = $folderid;
         $folder->serverid = $this->GetFolderIdForBackendId($folder->BackendId, true, $folderOrigin, $name);
@@ -1156,6 +1164,7 @@ class DeviceManager {
         // save store as custom property which is not streamed directly to the device
         $folder->NoBackendFolder = true;
         $folder->Store = $store;
+        $folder->Flags = $flags;
 
         return $folder;
     }

src/lib/core/zpushdefs.php

@@ -326,6 +326,7 @@ define("SYNC_FOLDERHIERARCHY_VERSION","FolderHierarchy:Version");
 define("SYNC_FOLDERHIERARCHY_IGNORE_STORE","FolderHierarchy:IgnoreStore");
 define("SYNC_FOLDERHIERARCHY_IGNORE_NOBCKENDFLD","FolderHierarchy:IgnoreNoBackendFolder");
 define("SYNC_FOLDERHIERARCHY_IGNORE_BACKENDID","FolderHierarchy:IgnoreBackendId");
+define("SYNC_FOLDERHIERARCHY_IGNORE_FLAGS","FolderHierarchy:IgnoreFlags");
 
 // MeetingResponse
 define("SYNC_MEETINGRESPONSE_CALENDARID","MeetingResponse:CalendarId");

src/lib/request/request.php

@@ -654,6 +654,32 @@ class Request {
         return (time() - $_SERVER["REQUEST_TIME"]) >= self::GetExpectedConnectionTimeout();
     }
 
+    /**
+     * Checks the device type if it expects the globalobjid in meeting requests encoded as hex.
+     * If it's not the case, globalobjid will be base64 encoded.
+     *
+     * WindowsOutlook and iOS device since 9.3 (?) version expect globalobjid to be hex encoded.
+     * @see https://jira.z-hub.io/projects/ZP/issues/ZP-1013
+     *
+     * @access public
+     * @return boolean
+     */
+    static public function IsGlobalObjIdHexClient() {
+        switch (self::GetDeviceType()) {
+            case "WindowsOutlook":
+                ZLog::Write(LOGLEVEL_DEBUG, "Request->IsGlobalObjIdHexClient(): WindowsOutlook");
+                return true;
+            case "iPod":
+            case "iPad":
+            case "iPhone":
+                $matches = array();
+                if (preg_match("/^Apple-.*?\/(\d{4})\./", self::GetUserAgent(), $matches) && isset($matches[1]) && $matches[1] >= 1305) {
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("Request->IsGlobalObjIdHexClient(): %s->%s", self::GetDeviceType(), self::GetUserAgent()));
+                    return true;
+                }
+        }
+        return false;
+    }
     /**----------------------------------------------------------------------------------------------------------
      * Private stuff
      */

src/lib/request/settings.php

@@ -59,13 +59,14 @@ class Settings extends RequestProcessor {
         if(self::$deviceManager->IsKoe()) {
             // define the supported capabilites
             $cap = array();
-            if(KOE_CAPABILITY_GAB)           $cap[] = "gab";
-            if(KOE_CAPABILITY_RECEIVEFLAGS)  $cap[] = "receiveflags";
-            if(KOE_CAPABILITY_SENDFLAGS)     $cap[] = "sendflags";
-            if(KOE_CAPABILITY_OOFTIMES)      $cap[] = "ooftime";
-            elseif(KOE_CAPABILITY_OOF)       $cap[] = "oof";        // 'ooftime' superseeds 'oof'. If 'ooftime' is set, 'oof' should not be defined.
-            if(KOE_CAPABILITY_NOTES)         $cap[] = "notes";
-            if(KOE_CAPABILITY_SHAREDFOLDER)  $cap[] = "sharedfolder";
+            if (defined('KOE_CAPABILITY_GAB') && KOE_CAPABILITY_GAB)                    $cap[] = "gab";
+            if (defined('KOE_CAPABILITY_RECEIVEFLAGS') && KOE_CAPABILITY_RECEIVEFLAGS)  $cap[] = "receiveflags";
+            if (defined('KOE_CAPABILITY_SENDFLAGS') && KOE_CAPABILITY_SENDFLAGS)        $cap[] = "sendflags";
+            if (defined('KOE_CAPABILITY_OOFTIMES') && KOE_CAPABILITY_OOFTIMES)          $cap[] = "ooftime";
+            elseif(defined('KOE_CAPABILITY_OOF') && KOE_CAPABILITY_OOF)                 $cap[] = "oof";        // 'ooftime' superseeds 'oof'. If 'ooftime' is set, 'oof' should not be defined.
+            if (defined('KOE_CAPABILITY_NOTES') && KOE_CAPABILITY_NOTES)                $cap[] = "notes";
+            if (defined('KOE_CAPABILITY_SHAREDFOLDER') && KOE_CAPABILITY_SHAREDFOLDER)  $cap[] = "sharedfolder";
+            if (defined('KOE_CAPABILITY_SENDAS') && KOE_CAPABILITY_SENDAS)              $cap[] = "sendas";
 
             self::$specialHeaders = array();
             self::$specialHeaders[] = "X-Push-Capabilities: ". implode(",",$cap);

src/lib/request/sync.php

@@ -1445,6 +1445,9 @@ class Sync extends RequestProcessor {
                 case SYNC_ADD:
                     self::$topCollector->AnnounceInformation(sprintf("Creating new message from mobile %d", $messageCount));
                     try {
+                        // mark the message as new message so SyncObject->Check() can differentiate
+                        $message->flags = SYNC_NEWMESSAGE;
+
                         // ignore sms messages
                         if ($foldertype == "SMS") {
                             ZLog::Write(LOGLEVEL_DEBUG, "SMS sync are not supported. Ignoring message.");

src/lib/syncobjects/syncappointment.php

@@ -92,8 +92,7 @@ class SyncAppointment extends SyncObject {
 
                     SYNC_POOMCAL_STARTTIME                              => array (  self::STREAMER_VAR      => "starttime",
                                                                                     self::STREAMER_TYPE     => self::STREAMER_TYPE_DATE,
-                                                                                    self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_REQUIRED       => self::STREAMER_CHECK_SETZERO,
-                                                                                                                        self::STREAMER_CHECK_CMPLOWER       => SYNC_POOMCAL_ENDTIME ),
+                                                                                    self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_CMPLOWER       => SYNC_POOMCAL_ENDTIME ),
                                                                                     self::STREAMER_RONOTIFY => true ),
 
 
@@ -108,8 +107,7 @@ class SyncAppointment extends SyncObject {
                                                                                     self::STREAMER_RONOTIFY => true),
                     SYNC_POOMCAL_ENDTIME                                => array (  self::STREAMER_VAR      => "endtime",
                                                                                     self::STREAMER_TYPE     => self::STREAMER_TYPE_DATE,
-                                                                                    self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_REQUIRED       => self::STREAMER_CHECK_SETONE,
-                                                                                                                        self::STREAMER_CHECK_CMPHIGHER      => SYNC_POOMCAL_STARTTIME ),
+                                                                                    self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_CMPHIGHER      => SYNC_POOMCAL_STARTTIME ),
                                                                                     self::STREAMER_RONOTIFY => true ),
 
                     SYNC_POOMCAL_RECURRENCE                             => array (  self::STREAMER_VAR      => "recurrence",
@@ -234,6 +232,41 @@ class SyncAppointment extends SyncObject {
      * @return boolean
      */
     public function Check($logAsDebug = false) {
+        // Fix starttime and endtime if they are not set on NEW appointments - see https://jira.z-hub.io/browse/ZP-983
+        if ($this->flags === SYNC_NEWMESSAGE) {
+            $time = time();
+            $calcstart = $time + 1800 - ($time % 1800); // round up to the next half hour
+
+            // Check error cases first
+            // Case 2: starttime not set, endtime in the past
+            if (!isset($this->starttime) && isset($this->endtime) && $this->endtime < $time) {
+                ZLog::Write(LOGLEVEL_WARN, "SyncAppointment->Check(): Parameter 'starttime' not set while 'endtime' is in the past (case 2). Aborting.");
+                return false;
+            }
+            // Case 3b: starttime not set, endtime in the future (3) but before the calculated starttime (3b)
+            elseif (!isset($this->starttime) && isset($this->endtime) && $this->endtime > $time && $this->endtime < $calcstart) {
+                ZLog::Write(LOGLEVEL_WARN, "SyncAppointment->Check(): Parameter 'starttime' not set while 'endtime' is in the future but before the calculated starttime (case 3b). Aborting.");
+                return false;
+            }
+            // Case 5: starttime in the future but no endtime set
+            elseif (isset($this->starttime) && $this->starttime > $time && !isset($this->endtime)) {
+                ZLog::Write(LOGLEVEL_WARN, "SyncAppointment->Check(): Parameter 'starttime' is in the future but 'endtime' is not set (case 5). Aborting.");
+                return false;
+            }
+
+            // Set starttime to the rounded up next half hour
+            // Case 1, 3a (endtime won't be changed as it's set)
+            if (!isset($this->starttime)) {
+                $this->starttime = $calcstart;
+                ZLog::Write(LOGLEVEL_WBXML, sprintf("SyncAppointment->Check(): Parameter 'starttime' was not set, setting it to %d (%s).", $this->starttime, gmstrftime("%Y%m%dT%H%M%SZ", $this->starttime)));
+            }
+            // Case 1, 4
+            if (!isset($this->endtime)) {
+                $this->endtime = $calcstart + 1800; // 30 min after calcstart
+                ZLog::Write(LOGLEVEL_WBXML, sprintf("SyncAppointment->Check(): Parameter 'endtime' was not set, setting it to %d (%s).", $this->endtime, gmstrftime("%Y%m%dT%H%M%SZ", $this->endtime)));
+            }
+        }
+
         $ret = parent::Check($logAsDebug);
 
         // semantic checks general "turn off switch"

src/lib/syncobjects/syncfolder.php

@@ -54,6 +54,7 @@ class SyncFolder extends SyncObject {
     public $Store;
     public $NoBackendFolder;
     public $BackendId;
+    public $Flags;
 
     function SyncFolder() {
         $mapping = array (
@@ -79,6 +80,10 @@ class SyncFolder extends SyncObject {
 
                     SYNC_FOLDERHIERARCHY_IGNORE_BACKENDID               => array (  self::STREAMER_VAR      => "BackendId",
                                                                                     self::STREAMER_TYPE     => self::STREAMER_TYPE_IGNORE),
+
+                    SYNC_FOLDERHIERARCHY_IGNORE_FLAGS                   => array (  self::STREAMER_VAR      => "Flags",
+                                                                                    self::STREAMER_TYPE     => self::STREAMER_TYPE_IGNORE),
+
         );
 
         parent::SyncObject($mapping);

src/lib/utils/zpushadmin.php

@@ -202,7 +202,7 @@ class ZPushAdmin {
 
             // save device data
             try {
-                if ($device->IsNewDevice()) {
+                if ($device->IsNewDevice() || $device->GetData() === false) {
                     ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::WipeDevice(): data of user '%s' not synchronized on device '%s'. Aborting.", $user, $devid));
                     return false;
                 }
@@ -422,9 +422,10 @@ class ZPushAdmin {
                 // remove hierarchcache
                 StateManager::UnLinkState($device, false);
 
-                ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
-
-                ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::ResyncDevice(): all folders synchronized to device '%s' of user '%s' marked to be re-synchronized.", $devid, $user));
+                if ($device->GetData() !== false) {
+                    ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::ResyncDevice(): all folders synchronized to device '%s' of user '%s' marked to be re-synchronized.", $devid, $user));
+                }
             }
             catch (StateNotFoundException $e) {
                 ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::ResyncDevice(): state for device '%s' of user '%s' can not be found or saved", $devid, $user));
@@ -506,6 +507,7 @@ class ZPushAdmin {
                                         'name' => $so->displayname,
                                         'type' => $so->type,
                                         'origin' => Utils::GetFolderOriginFromId($syncfolderid),
+                                        'flags' => 0,           // static folders have no flags
                                     );
                 }
             }
@@ -528,11 +530,12 @@ class ZPushAdmin {
      * @param string    $add_folderid   the folder id of the additional folder.
      * @param string    $add_name       the name of the additional folder (has to be unique for all folders on the device).
      * @param string    $add_type       AS foldertype of SYNC_FOLDER_TYPE_USER_*
+     * @param int       $add_flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    static public function AdditionalFolderAdd($user, $devid, $add_store, $add_folderid, $add_name, $add_type) {
+    static public function AdditionalFolderAdd($user, $devid, $add_store, $add_folderid, $add_name, $add_type, $add_flags) {
         // load device data
         $device = new ASDevice($devid, ASDevice::UNDEFINED, $user, ASDevice::UNDEFINED);
         try {
@@ -549,10 +552,10 @@ class ZPushAdmin {
                 return false;
             }
 
-            $status = $device->AddAdditionalFolder($add_store, $add_folderid, $add_name, $add_type);
-            if ($status)
+            $status = $device->AddAdditionalFolder($add_store, $add_folderid, $add_name, $add_type, $add_flags);
+            if ($status && $device->GetData() !== false) {
                 ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
-
+            }
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::AdditionalFolderAdd(): added folder '%s' to additional folders list of device '%s' of user '%s' with status: %s", $add_name, $devid, $user, Utils::PrintAsString($status)));
             return $status;
         }
@@ -570,11 +573,12 @@ class ZPushAdmin {
      * @param string    $devid          device id of where the folder should be updated.
      * @param string    $add_folderid   the folder id of the additional folder.
      * @param string    $add_name       the name of the additional folder (has to be unique for all folders on the device).
+     * @param int       $add_flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    static public function AdditionalFolderEdit($user, $devid, $add_folderid, $add_name) {
+    static public function AdditionalFolderEdit($user, $devid, $add_folderid, $add_name, $add_flags) {
         // load device data
         $device = new ASDevice($devid, ASDevice::UNDEFINED, $user, ASDevice::UNDEFINED);
         try {
@@ -597,10 +601,10 @@ class ZPushAdmin {
                 return false;
             }
 
-            $status = $device->EditAdditionalFolder($add_folderid, $add_name);
-            if ($status)
+            $status = $device->EditAdditionalFolder($add_folderid, $add_name, $add_flags);
+            if ($status  && $device->GetData() !== false) {
                 ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
-
+            }
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::AdditionalFolderEdit(): updated folder '%s' in additional folders list of device '%s' of user '%s' with status: %s", $add_name, $devid, $user, Utils::PrintAsString($status)));
             return $status;
         }
@@ -639,9 +643,9 @@ class ZPushAdmin {
             }
 
             $status = $device->RemoveAdditionalFolder($add_folderid);
-            if ($status)
+            if ($status && $device->GetData() !== false) {
                 ZPush::GetStateMachine()->SetState($device->GetData(), $devid, IStateMachine::DEVICEDATA);
-
+            }
             ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::AdditionalFolderRemove(): removed folder '%s' in additional folders list of device '%s' of user '%s' with status: %s", $add_folderid, $devid, $user, Utils::PrintAsString($status)));
             return $status;
         }
@@ -714,7 +718,6 @@ class ZPushAdmin {
 
                     $devices = $devData->devices;
                     $knownUsers = array_keys($devData->devices);
-                    ZLog::Write(LOGLEVEL_DEBUG, print_r($devData,1),false);
 
                     foreach ($obsoleteUsers as $ouser) {
                         $lowerOUser = strtolower($ouser);
@@ -852,7 +855,7 @@ class ZPushAdmin {
         $nouuid = 0;
         $fixed = 0;
         $asdevices = ZPush::GetStateMachine()->GetAllDevices(false);
-        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesHierarchyFolderData(): found %d devices", count($devices)));
+        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesHierarchyFolderData(): found %d devices", count($asdevices)));
 
         foreach ($asdevices as $devid) {
             try {
@@ -917,4 +920,56 @@ class ZPushAdmin {
         return array($devices, $seen, $fixed, $nouuid);
     }
 
+    /**
+     * Fixes potentially missing flags on additional folders.
+     *
+     * @access public
+     * @return array(seenDevices, devicesWithAdditionalFolders, fixedAdditionalFolders)
+     */
+    static public function FixStatesAdditionalFolderFlags() {
+        $devices = 0;
+        $devicesWithAddFolders = 0;
+        $fixed = 0;
+        $asdevices = ZPush::GetStateMachine()->GetAllDevices(false);
+        ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesAdditionalFolderFlags(): found %d devices", count($asdevices)));
+
+        foreach ($asdevices as $devid) {
+            try {
+                // get the device
+                $devicedata = ZPush::GetStateMachine()->GetState($devid, IStateMachine::DEVICEDATA);
+                $devices++;
+                $needsFixing = false;
+
+                foreach (self::ListUsers($devid) as $username) {
+                    $device = new ASDevice($devid, ASDevice::UNDEFINED, $username, ASDevice::UNDEFINED);
+                    $device->SetData($devicedata, false);
+
+                    $addFolders = $device->GetAdditionalFolders();
+                    if ($addFolders) {
+                        $devicesWithAddFolders++;
+                        foreach($addFolders as $df) {
+                            if (!isset($df['flags'])) {
+                                $device->EditAdditionalFolder($df['folderid'], $df['name'], 0);
+                                $needsFixing = true;
+                            }
+                        }
+                    }
+                    if ($device->GetData() !== false) {
+                        $devicedata = $device->GetData();
+                    }
+                }
+                if ($needsFixing) {
+                    ZPush::GetStateMachine()->SetState($devicedata, $devid, IStateMachine::DEVICEDATA);
+                    ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesAdditionalFolderFlags(): updated device '%s' because flags were fixed", $devid));
+                    $fixed++;
+                }
+            }
+            catch (StateNotFoundException $e) {
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("ZPushAdmin::FixStatesAdditionalFolderFlags(): state for device '%s' can not be found", $devid));
+            }
+        }
+        return array($devices, $devicesWithAddFolders, $fixed);
+    }
+
+
 }

src/lib/webservice/webservicedevice.php

@@ -190,17 +190,19 @@ class WebserviceDevice {
      * @param string    $add_folderid   the folder id of the additional folder.
      * @param string    $add_name       the name of the additional folder (has to be unique for all folders on the device).
      * @param string    $add_type       AS foldertype of SYNC_FOLDER_TYPE_USER_*
+     * @param int       $add_flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    public function AdditionalFolderAdd($deviceId, $add_store, $add_folderid, $add_name, $add_type) {
+    public function AdditionalFolderAdd($deviceId, $add_store, $add_folderid, $add_name, $add_type, $add_flags) {
         $user = Request::GetGETUser();
         $deviceId = preg_replace("/[^A-Za-z0-9]/", "", $deviceId);
         $add_folderid = preg_replace("/[^A-Za-z0-9]/", "", $add_folderid);
         $add_type = preg_replace("/[^0-9]/", "", $add_type);
+        $add_flags = preg_replace("/[^0-9]/", "", $add_flags);
 
-        $status = ZPushAdmin::AdditionalFolderAdd($user, $deviceId, $add_store, $add_folderid, $add_name, $add_type);
+        $status = ZPushAdmin::AdditionalFolderAdd($user, $deviceId, $add_store, $add_folderid, $add_name, $add_type, $add_flags);
         if (!$status) {
             ZPush::GetTopCollector()->AnnounceInformation(ZLog::GetLastMessage(LOGLEVEL_ERROR), true);
             throw new SoapFault("ERROR", ZLog::GetLastMessage(LOGLEVEL_ERROR));
@@ -217,21 +219,23 @@ class WebserviceDevice {
      * @param string    $deviceId       device id of where the folder should be updated.
      * @param string    $add_folderid   the folder id of the additional folder.
      * @param string    $add_name       the name of the additional folder (has to be unique for all folders on the device).
+     * @param int       $add_flags      Additional flags, like DeviceManager::FLD_FLAGS_REPLYASUSER
      *
      * @access public
      * @return boolean
      */
-    public function AdditionalFolderEdit($deviceId, $add_folderid, $add_name) {
+    public function AdditionalFolderEdit($deviceId, $add_folderid, $add_name, $add_flags) {
         $user = Request::GetGETUser();
         $deviceId = preg_replace("/[^A-Za-z0-9]/", "", $deviceId);
         $add_folderid = preg_replace("/[^A-Za-z0-9]/", "", $add_folderid);
+        $add_flags = preg_replace("/[^0-9]/", "", $add_flags);
 
-        $status = ZPushAdmin::AdditionalFolderEdit($user, $deviceId, $add_folderid, $add_name);
+        $status = ZPushAdmin::AdditionalFolderEdit($user, $deviceId, $add_folderid, $add_name, $add_flags);
         if (!$status) {
             ZPush::GetTopCollector()->AnnounceInformation(ZLog::GetLastMessage(LOGLEVEL_ERROR), true);
             throw new SoapFault("ERROR", ZLog::GetLastMessage(LOGLEVEL_ERROR));
         }
-        ZLog::Write(LOGLEVEL_INFO, sprintf("WebserviceDevice::AdditionalFolderEdit(): added folder for device '%s' of user '%s': %s", $deviceId, $user, Utils::PrintAsString($status)));
+        ZLog::Write(LOGLEVEL_INFO, sprintf("WebserviceDevice::AdditionalFolderEdit(): edited folder for device '%s' of user '%s': %s", $deviceId, $user, Utils::PrintAsString($status)));
         ZPush::GetTopCollector()->AnnounceInformation("Edited additional folder", true);
 
         return $status;

src/z-push-admin.php

@@ -75,7 +75,8 @@ include_once(ZPUSH_CONFIG);
         ZPushAdminCLI::RunCommand();
     }
     catch (ZPushException $zpe) {
-        die(get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+        fwrite(STDERR, get_class($zpe) . ": ". $zpe->getMessage() . "\n");
+        exit(1);
     }
 
 
@@ -743,6 +744,12 @@ class ZPushAdminCLI {
             printf("Devices: %d - Processed: %d - Fixed: %d - Device+User without hierarchy: %d\n",  $stat[0], $stat[1], $stat[2], $stat[3]);
         else
             echo ZLog::GetLastMessage(LOGLEVEL_ERROR) . "\n";
+
+        echo "\tChecking flags of shared folders: ";
+        if (($stat = ZPushAdmin::FixStatesAdditionalFolderFlags()) !== false)
+            printf("Devices: %d - Devices with additional folders: %d - Fixed: %d\n",  $stat[0], $stat[1], $stat[2]);
+        else
+            echo ZLog::GetLastMessage(LOGLEVEL_ERROR) . "\n";
     }
 
     /**

tools/gab-sync/gab-sync.php

@@ -70,7 +70,8 @@ include_once(SYNC_CONFIG);
         GabSyncCLI::RunCommand();
     }
     catch (Exception $ex) {
-        die(get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        fwrite(STDERR, get_class($ex) . ": ". $ex->getMessage() . PHP_EOL);
+        exit(1);
     }