ZP-702 Implement plain and base64 encoded stream support into the wbxml

encoder and streamer. Removed padding filter and added a
null-char-replace filter used when streaming plain streams (tested but
not part of this ticket). 

Released under the Affero GNU General Public License (AGPL) version 3.

src/index.php

@@ -71,7 +71,6 @@ include_once('lib/core/devicemanager.php');
 include_once('lib/core/zpush.php');
 include_once('include/z_syslog.php');
 include_once('lib/core/zlog.php');
-include_once('lib/core/paddingfilter.php');
 include_once('lib/interface/ibackend.php');
 include_once('lib/interface/ichanges.php');
 include_once('lib/interface/iexportchanges.php');

src/lib/core/streamer.php

@@ -55,13 +55,15 @@ class Streamer implements Serializable {
     const STREAMER_TYPE_DATE = 1;
     const STREAMER_TYPE_HEX = 2;
     const STREAMER_TYPE_DATE_DASHES = 3;
-    const STREAMER_TYPE_STREAM = 4;
+    const STREAMER_TYPE_STREAM = 4; // deprecated
     const STREAMER_TYPE_IGNORE = 5;
     const STREAMER_TYPE_SEND_EMPTY = 6;
     const STREAMER_TYPE_NO_CONTAINER = 7;
     const STREAMER_TYPE_COMMA_SEPARATED = 8;
     const STREAMER_TYPE_SEMICOLON_SEPARATED = 9;
     const STREAMER_TYPE_MULTIPART = 10;
+    const STREAMER_TYPE_STREAM_ASBASE64 = 11;
+    const STREAMER_TYPE_STREAM_ASPLAIN = 12;
 
     protected $mapping;
     public $flags;
@@ -318,8 +320,11 @@ class Streamer implements Serializable {
                     else if(isset($map[self::STREAMER_TYPE]) && $map[self::STREAMER_TYPE] == self::STREAMER_TYPE_HEX) {
                         $encoder->content(strtoupper(bin2hex($this->$map[self::STREAMER_VAR])));
                     }
-                    else if(isset($map[self::STREAMER_TYPE]) && $map[self::STREAMER_TYPE] == self::STREAMER_TYPE_STREAM) {
-                        $encoder->content($this->$map[self::STREAMER_VAR]);
+                    else if(isset($map[self::STREAMER_TYPE]) && $map[self::STREAMER_TYPE] == self::STREAMER_TYPE_STREAM_ASPLAIN) {
+                        $encoder->contentStream($this->$map[self::STREAMER_VAR], false);
+                    }
+                    else if(isset($map[self::STREAMER_TYPE]) && ($map[self::STREAMER_TYPE] == self::STREAMER_TYPE_STREAM_ASBASE64 || $map[self::STREAMER_TYPE] == self::STREAMER_TYPE_STREAM)) {
+                        $encoder->contentStream($this->$map[self::STREAMER_VAR], true);
                     }
                     // implode comma or semicolon arrays into a string
                     else if(isset($map[self::STREAMER_TYPE]) && is_array($this->$map[self::STREAMER_VAR]) &&

src/lib/syncobjects/syncitemoperationsattachment.php

@@ -52,7 +52,7 @@ class SyncItemOperationsAttachment extends SyncObject {
         $mapping = array(
             SYNC_AIRSYNCBASE_CONTENTTYPE                        => array (  self::STREAMER_VAR      => "contenttype"),
             SYNC_ITEMOPERATIONS_DATA                            => array (  self::STREAMER_VAR      => "data",
-                                                                            self::STREAMER_TYPE     => self::STREAMER_TYPE_STREAM,
+                                                                            self::STREAMER_TYPE     => self::STREAMER_TYPE_STREAM_ASBASE64,
                                                                             self::STREAMER_PROP     => self::STREAMER_TYPE_MULTIPART),
         );
 

src/lib/core/paddingfilter.php → src/lib/wbxml/replacenullcharfilter.php

 <?php
 /***********************************************
-* File      :   paddingfilter.php
+* File      :   replacenullcharfilter.php
 * Project   :   Z-Push
-* Descr     :   Our own filter for stream padding with zero strings.
+* Descr     :   Filters null characters out of a stream.
 *
-* Created   :   18.07.2012
+* Created   :   11.09.2015
 *
-* Copyright 2007 - 2013 Zarafa Deutschland GmbH
+* Copyright 2015 Zarafa Deutschland GmbH
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License, version 3,
@@ -41,20 +41,10 @@
 * Consult LICENSE file for details
 ************************************************/
 
-/* Define our filter class
- *
- * Usage: stream_filter_append($stream, 'padding.X');
- * where X is a number a stream will be padded to be
- * multiple of (e.g. padding.3 will pad the stream
- * to be multiple of 3 which is useful in base64
- * encoding).
- *
- * */
-class padding_filter extends php_user_filter {
-    private $padding = 4; // default padding
+class replace_nullchar_filter extends php_user_filter {
 
     /**
-     * This method is called whenever data is read from or written to the attached stream
+     * This method is called whenever data is read from or written to the attached stream.
      *
      * @see php_user_filter::filter()
      *
@@ -69,32 +59,12 @@ class padding_filter extends php_user_filter {
      */
     function filter($in, $out, &$consumed, $closing) {
         while ($bucket = stream_bucket_make_writeable($in)) {
-            if ($this->padding != 0 && $bucket->datalen < 8192) {
-                $bucket->data .= str_pad($bucket->data, $this->padding, 0x0);
-            }
-            $consumed += ($this->padding != 0 && $bucket->datalen < 8192) ? ($bucket->datalen + $this->padding) : $bucket->datalen;
+            $bucket->data = str_replace("\0", "", $bucket->data);
+            $consumed += $bucket->datalen;
             stream_bucket_append($out, $bucket);
         }
         return PSFS_PASS_ON;
     }
-
-    /**
-     * Called when creating the filter
-     *
-     * @see php_user_filter::onCreate()
-     *
-     * @access public
-     * @return boolean
-     */
-    function onCreate() {
-        $delim = strrpos($this->filtername, '.');
-        if ($delim !== false) {
-            $padding = substr($this->filtername, $delim + 1);
-            if (is_numeric($padding))
-                $this->padding = $padding;
-        }
-        return true;
-    }
 }
 
-stream_filter_register("padding.*", "padding_filter");
+stream_filter_register('replacenullchar', 'replace_nullchar_filter');
\ No newline at end of file

src/lib/wbxml/wbxmlencoder.php

@@ -165,33 +165,47 @@ class WBXMLEncoder extends WBXMLDefs {
     }
 
     /**
-     * Puts content on the output stack
+     * Puts content on the output stack.
      *
-     * @param string|resource $content
+     * @param string $content
      *
      * @access public
-     * @return string
+     * @return
      */
     public function content($content) {
-        if (is_resource($content)) {
-            if (($stat = fstat($content)) && isset($stat['size']) && !$stat['size']) {
-                ZLog::Write(LOGLEVEL_DEBUG, "Ommit writing 0 size stream to WBXML");
-                return;
-            }
-            fseek($content, 0, SEEK_SET);	// fstat seeks to end of file to determine size!
-        }
-        else {
-            // We need to filter out any \0 chars because it's the string terminator in WBXML. We currently
-            // cannot send \0 characters within the XML content anywhere.
-            $content = str_replace("\0","",$content);
+        // We need to filter out any \0 chars because it's the string terminator in WBXML. We currently
+        // cannot send \0 characters within the XML content anywhere.
+        $content = str_replace("\0","",$content);
 
-            if("x" . $content == "x")
-                return;
-        }
+        if("x" . $content == "x")
+            return;
         $this->_outputStack();
         $this->_content($content);
     }
 
+    /**
+     * Puts content of a stream on the output stack.
+     *
+     * @param resource $stream
+     * @param boolean $asBase64     if true, the data will be encoded as base64, default: false
+     *
+     * @access public
+     * @return
+     */
+    public function contentStream($stream, $asBase64 = false) {
+        if (!$asBase64) {
+            include_once('lib/wbxml/replacenullcharfilter.php');
+            $rnc_filter = stream_filter_append($stream, 'replacenullchar');
+        }
+
+        $this->_outputStack();
+        $this->_contentStream($stream, $asBase64);
+
+        if (!$asBase64) {
+            stream_filter_remove($rnc_filter);
+        }
+    }
+
     /**
      * Gets the value of multipart
      *
@@ -278,19 +292,46 @@ class WBXMLEncoder extends WBXMLDefs {
     }
 
     /**
-     * Outputs actual data
+     * Outputs actual data.
      *
      * @access private
-     * @param string|resource $content
+     * @param string $content
      * @return
      */
     private function _content($content) {
         $this->logContent($content);
         $this->outByte(WBXML_STR_I);
-        if (is_resource($content))
-            $this->outTermStream ($content);
-        else
-            $this->outTermStr($content);
+        $this->outTermStr($content);
+    }
+
+    /**
+     * Outputs actual data coming from a stream, optionally encoded as base64.
+     *
+     * @access private
+     * @param resource $stream
+     * @param boolean  $asBase64
+     * @return
+     */
+    private function _contentStream($stream, $asBase64) {
+        // write full stream, including the finalizing terminator to the output stream (stuff outTermStr() would do)
+        $this->outByte(WBXML_STR_I);
+        if ($asBase64) {
+            $out_filter = stream_filter_append($this->_out, 'convert.base64-encode');
+        }
+        stream_copy_to_stream($stream, $this->_out);
+        if ($asBase64) {
+            stream_filter_remove($out_filter);
+        }
+        fwrite($this->_out, chr(0));
+
+        // data is out, do some logging
+        $stat = fstat($stream);
+        $logContent = sprintf("<<< %d bytes of %s data >>>", $stat['size'], $asBase64 ? "base64 encoded":"plain");
+        $this->logContent($logContent);
+
+        // write the meta data also to the _outLog stream, WBXML_STR_I was already written by outByte() above
+        fwrite($this->_outLog, $logContent);
+        fwrite($this->_outLog, chr(0));
     }
 
     /**
@@ -359,28 +400,6 @@ class WBXMLEncoder extends WBXMLDefs {
         fwrite($this->_outLog, chr(0));
     }
 
-    /**
-     * Outputs stream content base64 encoded with string terminator
-     *
-     * @param resource $stream
-     *
-     * @access private
-     * @return
-     */
-    private function outTermStream($stream) {
-        $out_filter = stream_filter_append($this->_out, 'convert.base64-encode');
-		stream_copy_to_stream($stream, $this->_out);
-        stream_filter_remove($out_filter);
-        fwrite($this->_out, chr(0));
-        fseek($stream, 0, SEEK_SET);
-
-        $outLog_filter = stream_filter_append($this->_outLog, 'convert.base64-encode');
-        stream_copy_to_stream($stream, $this->_outLog);
-        stream_filter_remove($outLog_filter);
-        fwrite($this->_outLog, chr(0));
-        fseek($stream, 0, SEEK_SET);
-    }
-
     /**
      * Output attributes
      * We don't actually support this, because to do so, we would have
@@ -505,7 +524,7 @@ class WBXMLEncoder extends WBXMLDefs {
     /**
      * Logs content to ZLog
      *
-     * @param string|resource $content
+     * @param string $content
      *
      * @access private
      * @return
@@ -515,14 +534,7 @@ class WBXMLEncoder extends WBXMLDefs {
             return;
 
         $spaces = str_repeat(" ", count($this->logStack));
-        if (is_resource($content)) {
-            $stat = fstat($content);
-            fseek($content, 0, SEEK_SET);	// fstat seeks to end of file to determine size!
-            ZLog::Write(LOGLEVEL_WBXML,"O " . $spaces . " <<< ".$stat['size']." bytes of base64 encoded data >>>");
-        }
-        else {
-            ZLog::Write(LOGLEVEL_WBXML,"O " . $spaces . $content);
-        }
+        ZLog::Write(LOGLEVEL_WBXML,"O " . $spaces . $content);
     }
 
     /**