diff options
| -rw-r--r-- | combox/events.py | 67 | ||||
| -rw-r--r-- | tests/events_test.py | 104 |
2 files changed, 157 insertions, 14 deletions
diff --git a/combox/events.py b/combox/events.py index ff27651..0c19691 100644 --- a/combox/events.py +++ b/combox/events.py @@ -21,6 +21,7 @@ import logging from os import path from threading import Lock +from threading import Timer from watchdog.events import LoggingEventHandler @@ -205,6 +206,26 @@ class NodeDirMonitor(LoggingEventHandler): return False + def delete_later(self, file_cb_path): + """`file_cb_path' deleted if it is still under 'file_deleted'. + + This is used by the on_deleted method. + + This is a workaround to make combox predict official Google + Drive client's behavior. + """ + with self.lock: + num = self.silo.node_get('file_deleted', file_cb_path) + + if num == self.num_nodes: + # remove the corresponding file under the combox + # directory. + rm_path(file_cb_path) + # remove file info from silo. + self.silo.remove(file_cb_path) + self.silo.node_rem('file_deleted', file_cb_path) + + def housekeep(self): """Recursively traverses node directory, discovers changes and updates silo and combox directory. @@ -420,6 +441,32 @@ class NodeDirMonitor(LoggingEventHandler): if num == self.num_nodes: os.mkdir(file_cb_path) self.silo.node_rem('file_created', file_cb_path) + elif (not event.is_directory) and path.exists(file_cb_path): + # This can either mean the file was create on this + # computer or if this is a Google Drive node directory and + # the official Google Drive client is in use this means + # the file was modified. + # + # Google Drive client's behavior when a file (shard) is + # modified in the Google Drive node directory: + # + # - First it deletes the file. + # - Creates the latest version the file. + with self.lock: + num = self.silo.node_get('file_deleted', file_cb_path) + if num: + # This means we're in the Google Drive node + # directory and the official Google Drive client + # is in use and the file was actually modified on + # another computer. + self.silo.node_rem('file_deleted', file_cb_path) + self.silo.node_set('file_modified', file_cb_path) + num = self.silo.node_get('file_modified', file_cb_path) + if num == self.num_nodes: + decrypt_and_glue(file_cb_path, self.config) + # update db. + self.silo.update(file_cb_path) + self.silo.node_rem('file_modified', file_cb_path) elif (not event.is_directory) and (not path.exists(file_cb_path)): # shard created. @@ -460,14 +507,18 @@ class NodeDirMonitor(LoggingEventHandler): with self.lock: self.silo.node_set('file_deleted', file_cb_path) num = self.silo.node_get('file_deleted', file_cb_path) - - if num == self.num_nodes: - # remove the corresponding file under the combox - # directory. - rm_path(file_cb_path) - # remove file info from silo. - self.silo.remove(file_cb_path) - self.silo.node_rem('file_deleted', file_cb_path) + # If we are in a Google Drive node directory and + # the official Google Drive client is in use, at + # this point we cannot tell if the file was + # deleted; it can be a file modification or rename + # or deletion. + # + # Therefore, wait for 2secs and then delete the + # file_cb_path iff the file_cb_path was really + # removed on the another computer. + delayed_thread = Timer(3, self.delete_later, + [file_cb_path]) + delayed_thread.start() def on_modified(self, event): diff --git a/tests/events_test.py b/tests/events_test.py index a8b25ab..b995d6c 100644 --- a/tests/events_test.py +++ b/tests/events_test.py @@ -30,12 +30,14 @@ from nose.tools import * from watchdog.observers import Observer from combox.config import get_nodedirs -from combox.crypto import decrypt_and_glue, split_and_encrypt +from combox.crypto import (decrypt_and_glue, split_and_encrypt, + encrypt_shards) from combox.events import ComboxDirMonitor, NodeDirMonitor from combox.file import (relative_path, purge_dir, hash_file, read_file, write_file, move_shards, rm_shards, mk_nodedir, rm_nodedir, - move_nodedir, node_paths) + move_nodedir, node_paths, rm_path, + split_data, write_shards) from combox.silo import ComboxSilo from tests.utils import (get_config, shardedp, dirp, renamedp, @@ -400,7 +402,7 @@ class TestEvents(object): # Test - Shard deletion. rm_shards(the_guide, self.config) - time.sleep(1) + time.sleep(4) assert not path.exists(the_guide) ## check if the new file's info is removed from silo @@ -416,6 +418,99 @@ class TestEvents(object): observers[i].join() + def test_GoogleDrive_file_modify(self): + """Simulates Google Drive client's file modification behavior and + checks if combox is interpreting it properly. + """ + + nodes = get_nodedirs(self.config) + num_nodes = len(get_nodedirs(self.config)) + + nmonitors = [] + observers = [] + + # create an observer for each node directory and make it + # monitor them. + for node in nodes: + nmonitor = NodeDirMonitor(self.config, self.silo_lock, + self.nodem_lock) + observer = Observer() + observer.schedule(nmonitor, node, recursive=True) + observer.start() + + nmonitors.append(nmonitor) + observers.append(observer) + + # Test - shard modification + lorem_content = read_file(self.lorem) + self.lorem_copy = "%s.copy" % self.lorem + + copyfile(self.lorem, self.lorem_copy) + split_and_encrypt(self.lorem_copy, self.config, + lorem_content) + self.silo.update(self.lorem_copy) + shardedp(self.lorem_copy) + + lorem_copy_hash = self.silo.db.get(self.lorem_copy) + + ipsum_content = read_file(self.ipsum) + lorem_copy_content = "%s\n%s" % (lorem_content, ipsum_content) + + time.sleep(2) + + # Modify shards in the first n-1 node directories in the usual + # way. For the nth node directory simulate Google Drive + # official client's way of modifiying the shard. + + rel_path = relative_path(self.lorem_copy, self.config) + + # no. of shards = no. of nodes. + SHARDS = len(self.config['nodes_info'].keys()) + + f_shards = split_data(lorem_copy_content, SHARDS) + + # encrypt shards + ciphered_shards = encrypt_shards(f_shards, self.config['topsecret']) + + # write ciphered shards to disk + f_basename = rel_path + # gets the list of node' directories. + nodes = get_nodedirs(self.config) + last_node_index = len(nodes) - 1 + + nodes_subset = nodes[:last_node_index] + last_node = nodes[last_node_index] + + # write n-1 shards to the first n-1 node directories + write_shards(ciphered_shards, nodes_subset, f_basename) + + + # now for the nth shard, simulate Google Drive's client + # behavior. + last_shard_path = "%s.shard%d" % (path.join(last_node, f_basename), + last_node_index) + # remove the shard first + rm_path(last_shard_path) + # write the latest version of the shard + write_file(last_shard_path, ciphered_shards[last_node_index]) + time.sleep(3) + + self.silo.reload() + assert lorem_copy_content == read_file(self.lorem_copy) + + ## check if the lorem_copy's info is updated in silo + assert lorem_copy_hash != self.silo.db.get(self.lorem_copy) + assert_equal(None, self.silo.node_get('file_modified', + self.lorem_copy)) + + self.purge_list.append(self.lorem_copy) + + # stop the zarking observers. + for i in range(num_nodes): + observers[i].stop() + observers[i].join() + + def untest_NDM(self): """ Tests the NodeDirMonitor class. @@ -726,9 +821,6 @@ class TestEvents(object): rm_shards(self.TEST_FILE, self.config) - os.remove(self.lorem_ipsum) - rm_shards(self.lorem_ipsum, self.config) - rm_shards(self.lorem, self.config) rm_shards(self.ipsum, self.config) |