combox

split and encrypted files between online file storage providers
git clone git://git.ricketyspace.net/combox.git
Log | Files | Refs

commit 2b96ea454523354de88da661219395e02694ee84
parent 6bc6c1eecc8a34ab80dc4c6e4db8ab40e6ad4712
Author: Siddharth Ravikumar <sravik@bgsu.edu>
Date:   Fri,  1 May 2015 09:36:46 -0400

Updated combox.events.NodeDirMonitor.housekeep's 're-construct file' functionality.

   - Now the housekeep method re-contructs the files, whose shards
     were created when combox was off, in the combox directory only if
     all of its shards have made it to the respective node
     directories.

   - If it finds only some of all the shards of a file missing, it
     stores this information in the `file_created' dict inside the
     silo.

   - tests.events_tests.TestEvents.test_NDM_housekeep_create tests the
     're-construct file' functionality of
     combox.events.NodeDirMonitor.housekeep method.

	modified:   combox/events.py
	modified:   tests/events_test.py

Diffstat:
combox/events.py | 57+++++++++++++++++++++++++++++++++------------------------
tests/events_test.py | 30++++++++++++++++++++++++++++++
2 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/combox/events.py b/combox/events.py @@ -252,30 +252,39 @@ class NodeDirMonitor(LoggingEventHandler): # deleted in the 'file_deleted' dict inside the silo. self.silo.node_set('file_deleted', fpath, del_num) - for root, dirs, files in os.walk(get_nodedirs(self.config)[0]): - for f in files: - shard = path.join(root, f) - - if not self.shardp(shard): - continue - - file_cb_path = cb_path(shard, self.config) - if not self.silo.exists(file_cb_path): - print file_cb_path, "was created remotely. Creating it locally now..." - decrypt_and_glue(file_cb_path, self.config) - self.silo.update(file_cb_path) - elif self.silo.exists(file_cb_path): - file_content = decrypt_and_glue(file_cb_path, - self.config, - write=False) - file_content_hash = hash_file(file_cb_path, file_content) - - if self.silo.stale(file_cb_path, file_content_hash): - # file modified - print file_cb_path, "modified remotely. Updating local copy." - decrypt_and_glue(file_cb_path, self.config) - self.silo.update(file_cb_path) - + # Re-construct files created on another computer when combox + # was switched off. Only files who's all the shards have made + # it to the combox directory are re-constructed. + + # Dict that keeps track of the files that were created. + files_created = {} + for node_dir in get_nodedirs(self.config): + for root, dirs, files in os.walk(node_dir): + for f in files: + shard = path.join(root, f) + + if not self.shardp(shard): + continue + + file_cb_path = cb_path(shard, self.config) + if not self.silo.exists(file_cb_path): + if not files_created.get(file_cb_path): + files_created[file_cb_path] = 1 + else: + files_created[file_cb_path] += 1 + + for f_cb_path, crt_num in files_created.items(): + if crt_num == self.num_nodes: + print f_cb_path, "was created remotely. Creating it locally now..." + decrypt_and_glue(f_cb_path, self.config) + # update silo. + self.silo.update(f_cb_path) + self.silo.node_rem('file_created', f_cb_path) + elif crt_num > 0: + # means, all the shards of the file have not arrived + # yet, so, we store the no. of shards that did arrive + # in the 'file_created' dict inside the silo. + self.silo.node_set('file_created', f_cb_path, crt_num) print "combox node monitor is done with the drudgery." print "Do what you want to the combox directory." diff --git a/tests/events_test.py b/tests/events_test.py @@ -599,6 +599,36 @@ class TestEvents(object): self.purge_list.append(testf2) + def test_NDM_housekeep_create(self): + """Testing NodeDirMonitor's housekeep method's delete functionality.""" + + # test shard creation + hmutant = "%s.mutant" % self.TEST_FILE + hmutant_content = read_file(self.TEST_FILE) + split_and_encrypt(hmutant, self.config, + hmutant_content) + + lorem_c = "%s.c" % self.lorem + split_and_encrypt(lorem_c, self.config, + read_file(self.lorem)) + # delete lorem_c's shard0 + remove(node_paths(lorem_c, self.config, True)[0]) + + ndm = NodeDirMonitor(self.config, self.silo_lock, + self.nodem_lock) + ndm.housekeep() + + assert path.exists(hmutant) + assert hmutant_content == read_file(hmutant) + + assert not path.exists(lorem_c) + assert_equal(len(get_nodedirs(self.config))-1, + self.silo.node_get('file_created', lorem_c)) + + self.purge_list.append(hmutant) + self.purge_list.append(lorem_c) + + def untest_NDM_housekeep(self): """Testing NodeDirMonitor's housekeep method."""