Commit 7cc6292a authored by Sebastian Kummer's avatar Sebastian Kummer

Merge branch 'bugfix/ZP-848-folderstat-should-not-be-considered' of...

Merge branch 'bugfix/ZP-848-folderstat-should-not-be-considered' of https://stash.z-hub.io/scm/zp/z-push.git into bugfix/ZP-848-folderstat-should-not-be-considered
parents 0735bd49 585a812a
......@@ -10,7 +10,7 @@
*
* Created : 01.10.2011
*
* Copyright 2007 - 2015 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -114,6 +114,7 @@ class BackendZarafa implements IBackend, ISearchProvider {
$this->changesSink = false;
$this->changesSinkFolders = array();
$this->changesSinkStores = array();
$this->changesSinkHierarchyHash = false;
$this->wastebasket = false;
$this->session = false;
$this->folderStatCache = array();
......@@ -858,7 +859,8 @@ class BackendZarafa implements IBackend, ISearchProvider {
return false;
}
ZLog::Write(LOGLEVEL_DEBUG, "ZarafaBackend->HasChangesSink(): created");
$this->changesSinkHierarchyHash = $this->getHierarchyHash();
ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZarafaBackend->HasChangesSink(): created - HierarchyHash: %s", $this->changesSinkHierarchyHash));
// advise the main store and also to check if the connection supports it
return $this->adviseStoreToSink($this->defaultstore);
......@@ -900,9 +902,26 @@ class BackendZarafa implements IBackend, ISearchProvider {
* @return array
*/
public function ChangesSink($timeout = 30) {
// clear the folder stats cache
unset($this->folderStatCache);
$notifications = array();
$hierarchyNotifications = array();
$sinkresult = @mapi_sink_timedwait($this->changesSink, $timeout * 1000);
// reverse array so that the changes on folders are before changes on messages and
// it's possible to filter such notifications
$sinkresult = array_reverse($sinkresult, true);
foreach ($sinkresult as $sinknotif) {
// add a notification on a folder
if ($sinknotif['objtype'] == MAPI_FOLDER) {
$hierarchyNotifications[$sinknotif['entryid']] = IBackend::HIERARCHYNOTIFICATION;
}
// change on a message, remove hierarchy notification
if (isset($sinknotif['parentid']) && $sinknotif['objtype'] == MAPI_MESSAGE && isset($notifications[$sinknotif['parentid']])) {
unset($hierarchyNotifications[$sinknotif['parentid']]);
}
// TODO check if adding $sinknotif['objtype'] = MAPI_MESSAGE wouldn't break anything
// check if something in the monitored folders changed
if (isset($sinknotif['parentid']) && array_key_exists($sinknotif['parentid'], $this->changesSinkFolders)) {
$notifications[] = $this->changesSinkFolders[$sinknotif['parentid']];
......@@ -912,6 +931,15 @@ class BackendZarafa implements IBackend, ISearchProvider {
$notifications[] = $this->changesSinkFolders[$sinknotif['oldparentid']];
}
}
// validate hierarchy notifications by comparing the hierarchy hashes (too many false positives otherwise)
if (!empty($hierarchyNotifications)) {
$hash = $this->getHierarchyHash();
if ($hash !== $this->changesSinkHierarchyHash) {
ZLog::Write(LOGLEVEL_DEBUG, sprintf("BackendZarafa->ChangesSink() Hierarchy notification, pending validation. HierarchyHash: %s", $hash));
$notifications[] = IBackend::HIERARCHYNOTIFICATION;
}
}
return $notifications;
}
......@@ -1371,6 +1399,20 @@ class BackendZarafa implements IBackend, ISearchProvider {
* Private methods
*/
/**
* Returns a hash representing changes in the hierarchy of the main user.
* It changes if a folder is added, renamed or deleted.
*
* @access private
* @return string
*/
private function getHierarchyHash() {
$rootfolder = mapi_msgstore_openentry($this->defaultstore);
$hierarchy = mapi_folder_gethierarchytable($rootfolder, CONVENIENT_DEPTH);
return md5(serialize(mapi_table_queryallrows($hierarchy, array(PR_DISPLAY_NAME, PR_PARENT_ENTRYID))));
}
/**
* Advises a store to the changes sink
*
......
......@@ -10,7 +10,7 @@
*
* Created : 11.04.2011
*
* Copyright 2007 - 2015 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -67,6 +67,7 @@ class DeviceManager {
private $loopdetection;
private $hierarchySyncRequired;
private $additionalFoldersHash;
/**
* Constructor
......@@ -97,6 +98,8 @@ class DeviceManager {
$this->stateManager = new StateManager();
$this->stateManager->SetDevice($this->device);
$this->additionalFoldersHash = $this->getAdditionalFoldersHash();
}
/**
......@@ -608,13 +611,20 @@ class DeviceManager {
}
/**
* Indicates if the hierarchy should be resynchronized
* e.g. during PING
* Indicates if the hierarchy should be resynchronized based on the general folder state and
* if additional folders changed.
*
* @access public
* @return boolean
*/
public function IsHierarchySyncRequired() {
$this->loadDeviceData();
// if the hash of the additional folders changed, we have to sync the hierarchy
if ($this->additionalFoldersHash != $this->getAdditionalFoldersHash()) {
$this->hierarchySyncRequired = true;
}
// check if a hierarchy sync might be necessary
if ($this->device->GetFolderUUID(false) === false)
$this->hierarchySyncRequired = true;
......@@ -622,6 +632,10 @@ class DeviceManager {
return $this->hierarchySyncRequired;
}
private function getAdditionalFoldersHash() {
return md5(serialize($this->device->GetAdditionalFolders()));
}
/**
* Indicates if a full hierarchy resync should be triggered due to loops
*
......
......@@ -21,7 +21,7 @@
*
* Created : 26.12.2011
*
* Copyright 2007 - 2013 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -201,12 +201,13 @@ class StateManager {
* Gets the state for a specified synckey (uuid + counter)
*
* @param string $synckey
* @param boolean $forceHierarchyLoading, default: false
*
* @access public
* @return string
* @throws StateInvalidException, StateNotFoundException
*/
public function GetSyncState($synckey) {
public function GetSyncState($synckey, $forceHierarchyLoading = false) {
// No sync state for sync key '0'
if($synckey == "0") {
$this->oldStateCounter = 0;
......@@ -217,8 +218,8 @@ class StateManager {
list($this->uuid, $this->oldStateCounter) = self::ParseStateKey($synckey);
// make sure the hierarchy cache is in place
if ($this->hierarchyOperation)
$this->loadHierarchyCache();
if ($this->hierarchyOperation || $forceHierarchyLoading)
$this->loadHierarchyCache($forceHierarchyLoading);
// the state machine will discard any sync states before this one, as they are no longer required
return $this->statemachine->GetState($this->device->GetDeviceId(), IStateMachine::DEFTYPE, $this->uuid, $this->oldStateCounter, $this->deleteOldStates);
......@@ -473,12 +474,14 @@ class StateManager {
* Loads the HierarchyCacheState and initializes the HierarchyChache
* if this is an hierarchy operation
*
* @param boolean $forceLoading, default: false
*
* @access private
* @return boolean
* @throws StateNotFoundException
*/
private function loadHierarchyCache() {
if (!$this->hierarchyOperation)
private function loadHierarchyCache($forceLoading = false) {
if (!$this->hierarchyOperation && $forceLoading == false)
return false;
ZLog::Write(LOGLEVEL_DEBUG, sprintf("StateManager->loadHierarchyCache(): '%s-%s-%s-%d'", $this->device->GetDeviceId(), $this->uuid, IStateMachine::HIERARCHY, $this->oldStateCounter));
......
......@@ -14,7 +14,7 @@
*
* Created : 06.01.2012
*
* Copyright 2007 - 2013 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -70,6 +70,7 @@ class SyncCollections implements Iterator {
private $lastSyncTime;
private $waitingTime = 0;
private $hierarchyExporterChecked = false;
private $loggedGlobalWindowSizeOverwrite = false;
......@@ -118,17 +119,18 @@ class SyncCollections implements Iterator {
* Loads all collections known for the current device
*
* @param boolean $overwriteLoaded (opt) overwrites Collection with saved state if set to true
* @param boolean $loadState (opt) indicates if the collection sync state should be loaded, default true
* @param boolean $loadState (opt) indicates if the collection sync state should be loaded, default false
* @param boolean $checkPermissions (opt) if set to true each folder will pass
* through a backend->Setup() to check permissions.
* If this fails a StatusException will be thrown.
* @param boolean $loadHierarchy (opt) if the hierarchy sync states should be loaded, default false
*
* @access public
* @throws StatusException with SyncCollections::ERROR_WRONG_HIERARCHY if permission check fails
* @throws StateInvalidException if the sync state can not be found or relation between states is invalid ($loadState = true)
* @return boolean
*/
public function LoadAllCollections($overwriteLoaded = false, $loadState = false, $checkPermissions = false) {
public function LoadAllCollections($overwriteLoaded = false, $loadState = false, $checkPermissions = false, $loadHierarchy = false) {
$this->loadStateManager();
// this operation should not remove old state counters
......@@ -144,6 +146,10 @@ class SyncCollections implements Iterator {
$invalidStates = true;
}
// load the hierarchy data - there are no permissions to verify so we just set it to false
if ($loadHierarchy && !$this->LoadCollection(false, $loadState, false))
throw new StatusException("Invalid states found while loading hierarchy data. Forcing hierarchy sync");
if ($invalidStates)
throw new StateInvalidException("Invalid states found while loading collections. Forcing sync");
......@@ -180,6 +186,10 @@ class SyncCollections implements Iterator {
// in case there is something wrong with the state, just stop here
// later when trying to retrieve the SyncParameters nothing will be found
if ($folderid === false) {
throw new StatusException(sprintf("SyncCollections->LoadCollection(): could not get FOLDERDATA state of the hierarchy uuid: %s", $spa->GetUuid()), self::ERROR_WRONG_HIERARCHY);
}
// we also generate a fake change, so a sync on this folder is triggered
$this->changes[$folderid] = 1;
......@@ -196,7 +206,8 @@ class SyncCollections implements Iterator {
// load the latest known syncstate if requested
if ($addStatus && $loadState === true) {
try {
$this->addparms[$folderid]["state"] = $this->stateManager->GetSyncState($spa->GetLatestSyncKey());
// make sure the hierarchy cache is loaded when we are loading hierarchy states
$this->addparms[$folderid]["state"] = $this->stateManager->GetSyncState($spa->GetLatestSyncKey(), ($folderid === false));
}
catch (StateNotFoundException $snfe) {
// if we can't find the state, first we should try a sync of that folder, so
......@@ -493,7 +504,7 @@ class SyncCollections implements Iterator {
if (!empty($classes)) {
// initialize all possible folders
foreach ($this->collections as $folderid => $spa) {
if ($onlyPingable && $spa->GetPingableFlag() !== true)
if (($onlyPingable && $spa->GetPingableFlag() !== true) || ! $folderid)
continue;
// get the user store if this is a additional folder
......@@ -544,7 +555,7 @@ class SyncCollections implements Iterator {
throw new StatusException("SyncCollections->CheckForChanges(): PolicyKey changed. Provisioning required.", self::ERROR_WRONG_HIERARCHY);
// Check if a hierarchy sync is necessary
if (ZPush::GetDeviceManager()->IsHierarchySyncRequired())
if ($this->countHierarchyChange())
throw new StatusException("SyncCollections->CheckForChanges(): HierarchySync required.", self::HIERARCHY_CHANGED);
// Check if there are newer requests
......@@ -567,15 +578,21 @@ class SyncCollections implements Iterator {
$validNotifications = false;
foreach ($notifications as $folderid) {
// check if the notification on the folder is within our filter
if ($this->CountChange($folderid)) {
ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CheckForChanges(): Notification received on folder '%s'", $folderid));
$validNotifications = true;
$this->waitingTime = time()-$started;
}
else {
ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CheckForChanges(): Notification received on folder '%s', but it is not relevant", $folderid));
}
// Check hierarchy notifications
if ($folderid === IBackend::HIERARCHYNOTIFICATION) {
// check received hierarchy notifications by exporting
if ($this->countHierarchyChange(true))
throw new StatusException("SyncCollections->CheckForChanges(): HierarchySync required.", self::HIERARCHY_CHANGED);
}
// check if the notification on the folder is within our filter
else if ($this->CountChange($folderid)) {
ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CheckForChanges(): Notification received on folder '%s'", $folderid));
$validNotifications = true;
$this->waitingTime = time()-$started;
}
else {
ZLog::Write(LOGLEVEL_DEBUG, sprintf("SyncCollections->CheckForChanges(): Notification received on folder '%s', but it is not relevant", $folderid));
}
}
if ($validNotifications)
return true;
......@@ -678,6 +695,55 @@ class SyncCollections implements Iterator {
return ($changecount > 0);
}
/**
* Checks the hierarchy for changes.
*
* @param boolean export changes, default: false
*
* @access private
* @return boolean indicating if changes were found or not
*/
private function countHierarchyChange($exportChanges = false) {
$folderid = false;
$spa = $this->GetCollection($folderid);
// Check with device manager if the hierarchy should be reloaded.
// New additional folders are loaded here.
if (ZPush::GetDeviceManager()->IsHierarchySyncRequired()) {
ZLog::Write(LOGLEVEL_DEBUG, "SyncCollections->countHierarchyChange(): DeviceManager says HierarchySync is required.");
return true;
}
$changecount = false;
if ($exportChanges || $this->hierarchyExporterChecked === false) {
try {
$changesMem = ZPush::GetDeviceManager()->GetHierarchyChangesWrapper();
// the hierarchyCache should now fully be initialized - check for changes in the additional folders
$changesMem->Config(ZPush::GetAdditionalSyncFolders());
$exporter = ZPush::GetBackend()->GetExporter();
if ($exporter !== false && isset($this->addparms[$folderid]["state"])) {
$exporter->Config($this->addparms[$folderid]["state"]);
$ret = $exporter->InitializeExporter($changesMem);
while(is_array($exporter->Synchronize()));
if ($ret !== false)
$changecount = $changesMem->GetChangeCount();
$this->hierarchyExporterChecked = true;
}
}
catch (StatusException $ste) {
throw new StatusException("SyncCollections->countHierarchyChange(): exporter can not be re-configured.", self::ERROR_WRONG_HIERARCHY, null, LOGLEVEL_WARN);
}
// start over if exporter can not be configured atm
if ($changecount === false )
ZLog::Write(LOGLEVEL_WARN, "SyncCollections->countHierarchyChange(): no changes received from Exporter.");
}
return ($changecount > 0);
}
/**
* Returns an array with all folderid and the amount of changes found
*
......
......@@ -10,7 +10,7 @@
*
* Created : 02.01.2012
*
* Copyright 2007 - 2013 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -46,6 +46,7 @@
************************************************/
interface IBackend {
const HIERARCHYNOTIFICATION = 'hierarchynotification';
/**
* Returns a IStateMachine implementation used to save states
*
......
......@@ -6,7 +6,7 @@
*
* Created : 16.02.2012
*
* Copyright 2007 - 2015 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -246,6 +246,7 @@ class FolderChange extends RequestProcessor {
// update SPA & save it
$spa->SetSyncKey($newsynckey);
$spa->SetFolderId(false);
self::$deviceManager->GetStateManager()->SetSynchedFolderState($spa);
// invalidate all pingable flags
......
......@@ -6,7 +6,7 @@
*
* Created : 16.02.2012
*
* Copyright 2007 - 2015 Zarafa Deutschland GmbH
* Copyright 2007 - 2016 Zarafa Deutschland GmbH
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
......@@ -258,6 +258,7 @@ class FolderSync extends RequestProcessor {
// update SPA & save it
$spa->SetSyncKey($newsynckey);
$spa->SetFolderId(false);
self::$deviceManager->GetStateManager()->SetSynchedFolderState($spa);
// invalidate all pingable flags
......
......@@ -65,7 +65,7 @@ class Ping extends RequestProcessor {
// Load all collections - do load states and check permissions
try {
$sc->LoadAllCollections(true, true, true);
$sc->LoadAllCollections(true, true, true, true);
}
catch (StateInvalidException $siex) {
// if no params are present, indicate to send params, else do hierarchy sync
......
......@@ -581,7 +581,7 @@ class Sync extends RequestProcessor {
// Load all collections - do not overwrite existing (received!), load states and check permissions
try {
$sc->LoadAllCollections(false, true, true);
$sc->LoadAllCollections(false, true, true, true);
}
catch (StateInvalidException $siex) {
$status = SYNC_STATUS_INVALIDSYNCKEY;
......@@ -607,6 +607,14 @@ class Sync extends RequestProcessor {
if (!$sc->HasCollections())
$status = SYNC_STATUS_SYNCREQUESTINCOMPLETE;
}
else if (isset($hbinterval)) {
// load the hierarchy data - there are no permissions to verify so we just set it to false
if (!$sc->LoadCollection(false, true, false)) {
$status = SYNC_STATUS_FOLDERHIERARCHYCHANGED;
self::$topCollector->AnnounceInformation(sprintf("StatusException code: %d", $status), $this->singleFolder);
$this->saveMultiFolderInfo("exeption", "StatusException");
}
}
// HEARTBEAT & Empty sync
if ($status == SYNC_STATUS_SUCCESS && (isset($hbinterval) || $emptysync == true)) {
......
......@@ -106,8 +106,8 @@ class ZPushAdmin {
$sc = new SyncCollections();
$sc->SetStateManager($stateManager);
// load all collections of device without loading states or checking permissions
$sc->LoadAllCollections(true, false, false);
// load all collections of device without loading states, checking permissions or loading the hierarchy
$sc->LoadAllCollections(true, false, false, false);
if ($sc->GetLastSyncTime())
$device->SetLastSyncTime($sc->GetLastSyncTime());
......@@ -831,4 +831,67 @@ class ZPushAdmin {
return array($processed, $deleted);
}
/**
* Fixes hierarchy states writing folderdata states.
*
* @access public
* @return array(seenDevices, seenHierarchyStates, fixedHierarchyStates, usersWithoutHierarchy)
*/
static public function FixStatesHierarchyFolderData() {
$devices = 0;
$seen = 0;
$nouuid = 0;
$fixed = 0;
$asdevices = ZPush::GetStateMachine()->GetAllDevices(false);
ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesHierarchyFolderData(): found %d devices", count($devices)));
foreach ($asdevices as $devid) {
try {
// get the device
$devicedata = ZPush::GetStateMachine()->GetState($devid, IStateMachine::DEVICEDATA);
$devices++;
// get hierarchy UUID, check if FD is there else create it
foreach (self::ListUsers($devid) as $username) {
$device = new ASDevice($devid, ASDevice::UNDEFINED, $username, ASDevice::UNDEFINED);
$device->SetData($devicedata, false);
// get hierarchy UUID
$hierarchyUuid = $device->GetFolderUUID(false);
if ($hierarchyUuid == false) {
ZLog::Write(LOGLEVEL_WARN, sprintf("ZPushAdmin::FixStatesHierarchyFolderData(): device %s user '%s' has no hierarchy synchronized! Ignoring.", $devid, $username));
$nouuid++;
continue;
}
$seen++;
// try getting the FOLDERDATA for that state
try {
$data = ZPush::GetStateMachine()->GetState($device->GetDeviceId(), IStateMachine::FOLDERDATA, $hierarchyUuid);
}
catch(StateNotFoundException $snfe) {
// No FD found, search all states, and find the highest counter for the hierarchy UUID
$allStates = ZPush::GetStateMachine()->GetAllStatesForDevice($devid);
$maxCounter = 1;
foreach ($allStates as $state) {
if ($state["uuid"] == $hierarchyUuid && $state['counter'] > $maxCounter && ($state['type'] == "" || $state['type'] == false)) {
$maxCounter = $state['counter'];
}
}
// generate FOLDERDATA
$spa = new SyncParameters();
$spa->SetSyncKey(StateManager::BuildStateKey($hierarchyUuid, $maxCounter));
$spa->SetFolderId(false);
ZLog::Write(LOGLEVEL_DEBUG, sprintf("ZPushAdmin::FixStatesHierarchyFolderData(): write data for %s", $spa->GetSyncKey()));
ZPush::GetStateMachine()->SetState($spa, $device->GetDeviceId(), IStateMachine::FOLDERDATA, $hierarchyUuid);
$fixed++;
}
}
}
catch (StateNotFoundException $e) {}
}
return array($devices, $seen, $fixed, $nouuid);
}
}
......@@ -306,7 +306,6 @@ class WBXMLEncoder extends WBXMLDefs {
private function _contentStream($stream, $asBase64) {
// write full stream, including the finalizing terminator to the output stream (stuff outTermStr() would do)
$this->outByte(self::WBXML_STR_I);
fseek($stream, 0, SEEK_SET);
if ($asBase64) {
$out_filter = stream_filter_append($this->_out, 'convert.base64-encode');
}
......
......@@ -714,6 +714,12 @@ class ZPushAdminCLI {
printf("Processed: %d - Deleted: %d\n", $stat[0], $stat[1]);
else
echo ZLog::GetLastMessage(LOGLEVEL_ERROR) . "\n";
echo "\tChecking for hierarchy folder data state: ";
if (($stat = ZPushAdmin::FixStatesHierarchyFolderData()) !== false)
printf("Devices: %d - Processed: %d - Fixed: %d - Device+User without hierarchy: %d\n", $stat[0], $stat[1], $stat[2], $stat[3]);
else
echo ZLog::GetLastMessage(LOGLEVEL_ERROR) . "\n";
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment