Merge pull request #137 in ZP/z-push from...

Merge pull request #137 in ZP/z-push from feature/ZP-705-implement-support-for-streamer_type_stream_asplain to develop

* commit '4ccfc1f5':
  ZP-707 Remove debug.
  ZP-707 use MAPIStreamWrapper where possible.
  ZP-708 Wrap the plaintext body, html body or mime message into a StringStreamWrapper. While the wrapper correctly truncates (UTF-8 compatible), we can also truncate before loading the data in.
  ZP-707 Undo changes to the stream wrappers to be able to write at the beginning of the stream introduced with f133d268.
  ZP-706 SYNC_RESOLVERECIPIENTS_DATA is a base64 stream.
  ZP-706 Fix develop merge wrong conflict resolution.
  ZP-706 SyncMail->mimedata is a plaintext stream.
  ZP-707 Undo config changes.
  ZP-707 The stream wrapper need to truncate UTF-8 compatible.
  ZP-707 Use a MapiStreamWrapper for the AS2.5 mime message.
  ZP-707 When truncating AS12+ the general truncation size for the AS2.5 body should not be considered.
  ZP-707 Fixed instance variable access in MapiStreamWrapper.
  ZP-707 Use plain streams in mapi provider. Added writing functionality to StringStreamWrapper and MapiStreamWrapper. Reading data is encapsulated into a StringStreamWrapper by the Streamer.
  ZP-707 Initial implementation. Needs fixing.
  ZP-706 AirSyncBaseBody data is a plain data stream.

src/backend/imap/imap.php

@@ -1024,37 +1024,39 @@ class BackendIMAP extends BackendDiff {
             if (Request::GetProtocolVersion() >= 12.0) {
                 $output->asbody = new SyncBaseBody();
 
+                $data = "";
                 switch($bpReturnType) {
                     case SYNC_BODYPREFERENCE_PLAIN:
-                        $output->asbody->data = $plainBody;
+                        $data = $plainBody;
                         break;
                     case SYNC_BODYPREFERENCE_HTML:
                         if ($htmlBody == "") {
-                            $output->asbody->data = $plainBody;
+                            $data = $plainBody;
                             $bpReturnType = SYNC_BODYPREFERENCE_PLAIN;
                         }
                         else {
-                            $output->asbody->data = $htmlBody;
+                            $data = $htmlBody;
                         }
                         break;
                     case SYNC_BODYPREFERENCE_MIME:
                         //We don't need to create a new MIME mail, we already have one!!
-                        $output->asbody->data = $mail;
+                        $data = $mail;
                         break;
                     case SYNC_BODYPREFERENCE_RTF:
-                        ZLog::Write(LOGLEVEL_DEBUG, "BackendIMAP->GetMessage RTF Format NOT CHECKED");
-                        $output->asbody->data = base64_encode($plainBody);
+                        ZLog::Write(LOGLEVEL_DEBUG, "BackendIMAP->GetMessage RTF Format NOT SUPPORTED");
+                        // TODO: this is broken. This is no RTF.
+                        $data = base64_encode($plainBody);
                         break;
                 }
                 // truncate body, if requested
-                if(strlen($output->asbody->data) > $truncsize) {
-                    $output->asbody->data = Utils::Utf8_truncate($output->asbody->data, $truncsize);
+                if(strlen($data) > $truncsize) {
+                    $data = Utils::Utf8_truncate($data, $truncsize);
                     $output->asbody->truncated = 1;
                 }
-
+                $output->asbody->data = StringStreamWrapper::Open($data);
                 $output->asbody->type = $bpReturnType;
                 $output->nativebodytype = $bpReturnType;
-                $output->asbody->estimatedDataSize = strlen($output->asbody->data);
+                $output->asbody->estimatedDataSize = strlen($data);
 
                 $bpo = $contentparameters->BodyPreference($output->asbody->type);
                 if (Request::GetProtocolVersion() >= 14.0 && $bpo->GetPreview()) {
@@ -1068,28 +1070,22 @@ class BackendIMAP extends BackendDiff {
             else { // ASV_2.5
                 $output->bodytruncated = 0;
                 /* BEGIN fmbiete's contribution r1528, ZP-320 */
+                $data = "";
                 if ($bpReturnType == SYNC_BODYPREFERENCE_MIME) {
-                    if (strlen($mail) > $truncsize) {
-                        $output->mimedata = Utils::Utf8_truncate($mail, $truncsize);
-                        $output->mimetruncated = 1;
-                    }
-                    else {
-                        $output->mimetruncated = 0;
-                        $output->mimedata = $mail;
-                    }
-                    $output->mimesize = strlen($output->mimedata);
+                   $data = $mail;
                 }
                 else {
-                    // truncate body, if requested
-                    if (strlen($plainBody) > $truncsize) {
-                        $output->body = Utils::Utf8_truncate($plainBody, $truncsize);
-                        $output->bodytruncated = 1;
-                    }
-                    else {
-                        $output->body = $plainBody;
-                        $output->bodytruncated = 0;
-                    }
-                    $output->bodysize = strlen($output->body);
+                   $data = $plainBody;
+                }
+
+                $output->mimesize = strlen($data);
+                if (strlen($data) > $truncsize) {
+                    $output->mimedata = StringStreamWrapper::Open(Utils::Utf8_truncate($data, $truncsize));
+                    $output->mimetruncated = 1;
+                }
+                else {
+                    $output->mimetruncated = 0;
+                    $output->mimedata = StringStreamWrapper::Open($data);
                 }
                 /* END fmbiete's contribution r1528, ZP-320 */
             }

src/backend/zarafa/mapiprovider.php

@@ -2375,24 +2375,39 @@ class MAPIProvider {
                 if (isset($message->asbody))
                     $message->asbody->type = $bpReturnType;
                 return $stat;
-       }
+        }
+
+        $stream = mapi_openproperty($mapimessage, $property, IID_IStream, 0, 0);
+        $stat = mapi_stream_stat($stream);
+        $streamsize = $stat['cb'];
 
-        $body = mapi_message_openproperty($mapimessage, $property);
         //set the properties according to supported AS version
         if (Request::GetProtocolVersion() >= 12.0) {
             $message->asbody = new SyncBaseBody();
             $message->asbody->type = $bpReturnType;
-            if ($bpReturnType == SYNC_BODYPREFERENCE_RTF)
-                $message->asbody->data = base64_encode($body);
-            elseif (isset($message->internetcpid) && $bpReturnType == SYNC_BODYPREFERENCE_HTML)
-                $message->asbody->data = Utils::ConvertCodepageStringToUtf8($message->internetcpid, $body);
-            else
-                $message->asbody->data = w2u($body);
-            $message->asbody->estimatedDataSize = strlen($message->asbody->data);
+            if ($bpReturnType == SYNC_BODYPREFERENCE_RTF) {
+                $body = mapi_stream_read($stream, $streamsize);
+                $message->asbody->data = StringStreamWrapper::Open(base64_encode($body));
+            }
+            elseif (isset($message->internetcpid) && $bpReturnType == SYNC_BODYPREFERENCE_HTML) {
+                // if PR_HTML is UTF-8 we can stream it directly, else we have to convert to UTF-8 & wrap it
+                if (Utils::GetCodepageCharset($message->internetcpid) == "utf-8") {
+                    $message->asbody->data = MAPIStreamWrapper::Open($stream);
+                }
+                else {
+                    $body = mapi_stream_read($stream, $streamsize);
+                    $message->asbody->data = StringStreamWrapper::Open(Utils::ConvertCodepageStringToUtf8($message->internetcpid, $body));
+                }
+            }
+            else {
+                $message->asbody->data = MAPIStreamWrapper::Open($stream);
+            }
+            $message->asbody->estimatedDataSize = $streamsize;
         }
         else {
+            $body = mapi_stream_read($stream, $streamsize);
             $message->body = str_replace("\n","\r\n", w2u(str_replace("\r", "", $body)));
-            $message->bodysize = strlen($message->body);
+            $message->bodysize = $streamsize;
             $message->bodytruncated = 0;
         }
 
@@ -2419,22 +2434,20 @@ class MAPIProvider {
                 if (Request::GetProtocolVersion() >= 12.0) {
                     if (!isset($message->asbody))
                         $message->asbody = new SyncBaseBody();
-                    //TODO data should be wrapped in a MapiStreamWrapper
-                    $message->asbody->data = mapi_stream_read($stream, $streamsize);
+                    $message->asbody->data = MapiStreamWrapper::Open($stream);
                     $message->asbody->estimatedDataSize = $streamsize;
                     $message->asbody->truncated = 0;
                 }
                 else {
-                    $message->mimetruncated = 0;
-                    //TODO mimedata should be a wrapped in a MapiStreamWrapper
-                    $message->mimedata = mapi_stream_read($stream, $streamsize);
+                    $message->mimedata = MapiStreamWrapper::Open($stream);
                     $message->mimesize = $streamsize;
+                    $message->mimetruncated = 0;
                 }
                 unset($message->body, $message->bodytruncated);
                 return true;
             }
             else {
-                ZLog::Write(LOGLEVEL_ERROR, sprintf("Error opening attachment for imtoinet"));
+                ZLog::Write(LOGLEVEL_ERROR, sprintf("MAPIProvider->imtoinet(): got no stream or content from mapi_inetmapi_imtoinet()"));
             }
         }
 
@@ -2468,12 +2481,11 @@ class MAPIProvider {
             //only set the truncation size data if device set it in request
             if (    $bpo->GetTruncationSize() != false &&
                     $bpReturnType != SYNC_BODYPREFERENCE_MIME &&
-                    $message->asbody->estimatedDataSize > $bpo->GetTruncationSize() &&
-                    $contentparameters->GetTruncation() != SYNC_TRUNCATION_ALL // do not truncate message if the whole is requested, e.g. on fetch
+                    $message->asbody->estimatedDataSize > $bpo->GetTruncationSize()
                 ) {
-                $message->asbody->data = Utils::Utf8_truncate($message->asbody->data, $bpo->GetTruncationSize());
+                // truncate data stream
+                ftruncate($message->asbody->data, $bpo->GetTruncationSize());
                 $message->asbody->truncated = 1;
-
             }
             // set the preview or windows phones won't show the preview of an email
             if (Request::GetProtocolVersion() >= 14.0 && $bpo->GetPreview()) {
@@ -2630,15 +2642,16 @@ class MAPIProvider {
      * @return void
      */
     private function setASbody($asbody, &$props, $appointmentprops) {
-        if (isset($asbody->type) && isset($asbody->data) && strlen($asbody->data) > 0) {
+        // TODO: fix checking for the length
+        if (isset($asbody->type) && isset($asbody->data) /*&& strlen($asbody->data) > 0*/) {
             switch ($asbody->type) {
                 case SYNC_BODYPREFERENCE_PLAIN:
                 default:
                 //set plain body if the type is not in valid range
-                    $props[$appointmentprops["body"]] = u2w($asbody->data);
+                    $props[$appointmentprops["body"]] = stream_get_contents($asbody->data);
                     break;
                 case SYNC_BODYPREFERENCE_HTML:
-                    $props[$appointmentprops["html"]] = u2w($asbody->data);
+                    $props[$appointmentprops["html"]] = stream_get_contents($asbody->data);
                     break;
                 case SYNC_BODYPREFERENCE_RTF:
                     break;

src/backend/zarafa/mapistreamwrapper.php

@@ -48,6 +48,7 @@ class MAPIStreamWrapper {
     private $mapistream;
     private $position;
     private $streamlength;
+    private $toTruncate;
 
     /**
      * Opens the stream
@@ -68,6 +69,7 @@ class MAPIStreamWrapper {
             return false;
 
         $this->position = 0;
+        $this->toTruncate = false;
 
         // this is our stream!
         $this->mapistream = $contextOptions[self::PROTOCOL]['stream'];
@@ -91,8 +93,18 @@ class MAPIStreamWrapper {
      */
     public function stream_read($len) {
         $len = ($this->position + $len > $this->streamlength) ? ($this->streamlength - $this->position) : $len;
+
+        // read 4 additional bytes from the stream so we can always truncate correctly
+        if ($this->toTruncate)
+            $len += 4;
         $data = mapi_stream_read($this->mapistream, $len);
         $this->position += strlen($data);
+
+        // we need to truncate UTF8 compatible if ftruncate() was called
+        if ($this->toTruncate && $this->position >= $this->streamlength) {
+            $data = Utils::Utf8_truncate($data, $this->streamlength);
+        }
+
         return $data;
     }
 
@@ -137,6 +149,23 @@ class MAPIStreamWrapper {
         return ($this->position >= $this->streamlength);
     }
 
+    /**
+     * Truncates the stream to the new size.
+     *
+     * @param int $new_size
+     * @return boolean
+     */
+    public function stream_truncate ($new_size) {
+        $this->streamlength = $new_size;
+        $this->toTruncate = true;
+
+        if ($this->position > $this->streamlength) {
+            ZLog::Write(LOGLEVEL_WARN, sprintf("MAPIStreamWrapper->stream_truncate(): stream position (%d) ahead of new size of %d. Repositioning pointer to end of stream.", $this->position, $this->streamlength));
+            $this->position = $this->streamlength;
+        }
+        return true;
+    }
+
     /**
      * Retrieves information about a stream
      *

src/lib/core/streamer.php

@@ -196,6 +196,11 @@ class Streamer implements Serializable {
                                 if(!$decoder->getElementEndTag())
                                     return false;
                             }
+                            else if($map[self::STREAMER_TYPE] == self::STREAMER_TYPE_STREAM_ASPLAIN) {
+                                $decoded = StringStreamWrapper::Open($decoder->getElementContent());
+                                if(!$decoder->getElementEndTag())
+                                    return false;
+                            }
                             else {
                                 $subdecoder = new $map[self::STREAMER_TYPE]();
                                 if($subdecoder->Decode($decoder) === false)

src/lib/syncobjects/syncbasebody.php

@@ -56,7 +56,8 @@ class SyncBaseBody extends SyncObject {
                     SYNC_AIRSYNCBASE_TYPE                               => array (self::STREAMER_VAR        => "type"),
                     SYNC_AIRSYNCBASE_ESTIMATEDDATASIZE                  => array (self::STREAMER_VAR        => "estimatedDataSize"),
                     SYNC_AIRSYNCBASE_TRUNCATED                          => array (self::STREAMER_VAR        => "truncated"),
-                    SYNC_AIRSYNCBASE_DATA                               => array (self::STREAMER_VAR        => "data"),
+                    SYNC_AIRSYNCBASE_DATA                               => array (self::STREAMER_VAR        => "data",
+                                                                                  self::STREAMER_TYPE       => self::STREAMER_TYPE_STREAM_ASPLAIN),
         );
         if(Request::GetProtocolVersion() >= 14.0) {
             $mapping[SYNC_AIRSYNCBASE_PREVIEW]                          =  array (self::STREAMER_VAR        => "preview");

src/lib/syncobjects/syncmail.php

@@ -130,7 +130,8 @@ class SyncMail extends SyncObject {
                     SYNC_POOMMAIL_MIMETRUNCATED                         => array (  self::STREAMER_VAR      => "mimetruncated",
                                                                                     self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_ZEROORONE      => self::STREAMER_CHECK_SETZERO)),
 
-                    SYNC_POOMMAIL_MIMEDATA                              => array (  self::STREAMER_VAR      => "mimedata"), //TODO mimedata should be of a type stream
+                    SYNC_POOMMAIL_MIMEDATA                              => array (  self::STREAMER_VAR      => "mimedata",
+                                                                                    self::STREAMER_TYPE     => self::STREAMER_TYPE_STREAM_ASPLAIN),
 
                     SYNC_POOMMAIL_MIMESIZE                              => array (  self::STREAMER_VAR      => "mimesize",
                                                                                     self::STREAMER_CHECKS   => array(   self::STREAMER_CHECK_CMPHIGHER      => -1)),

src/lib/syncobjects/syncresolverecipientspicture.php

@@ -58,7 +58,9 @@ class SyncResolveRecipientsPicture extends SyncObject {
             $mapping[SYNC_RESOLVERECIPIENTS_MAXSIZE]        = array (  self::STREAMER_VAR      => "maxsize");
             $mapping[SYNC_RESOLVERECIPIENTS_MAXPICTURES]    = array (  self::STREAMER_VAR      => "maxpictures");
             $mapping[SYNC_RESOLVERECIPIENTS_STATUS]         = array (  self::STREAMER_VAR      => "status");
-            $mapping[SYNC_RESOLVERECIPIENTS_DATA]           = array (  self::STREAMER_VAR      => "data");
+            $mapping[SYNC_RESOLVERECIPIENTS_DATA]           = array (  self::STREAMER_VAR      => "data",
+                                                                       self::STREAMER_TYPE     => self::STREAMER_TYPE_STREAM_ASBASE64,
+                                                                     );
         }
 
         parent::SyncObject($mapping);

src/lib/utils/stringstreamwrapper.php

@@ -145,6 +145,24 @@ class StringStreamWrapper {
         return ($this->position >= $this->stringlength);
     }
 
+    /**
+     * Truncates the stream to the new size.
+     *
+     * @param int $new_size
+     * @return boolean
+     */
+    public function stream_truncate ($new_size) {
+        // cut the string!
+        $this->stringstream = Utils::Utf8_truncate($this->stringstream, $new_size);
+        $this->streamlength = strlen($this->stringstream);
+    
+        if ($this->position > $this->streamlength) {
+            ZLog::Write(LOGLEVEL_WARN, sprintf("MAPIStreamWrapper->stream_truncate(): stream position (%d) ahead of new size of %d. Repositioning pointer to end of stream.", $this->position, $this->streamlength));
+            $this->position = $this->streamlength;
+        }
+        return true;
+    }
+
     /**
     * Retrieves information about a stream
     *