From 2b96ea454523354de88da661219395e02694ee84 Mon Sep 17 00:00:00 2001
From: Siddharth Ravikumar <sravik@bgsu.edu>
Date: Fri, 1 May 2015 09:36:46 -0400
Subject: Updated combox.events.NodeDirMonitor.housekeep's 're-construct file'
 functionality.

   - Now the housekeep method re-contructs the files, whose shards
     were created when combox was off, in the combox directory only if
     all of its shards have made it to the respective node
     directories.

   - If it finds only some of all the shards of a file missing, it
     stores this information in the `file_created' dict inside the
     silo.

   - tests.events_tests.TestEvents.test_NDM_housekeep_create tests the
     're-construct file' functionality of
     combox.events.NodeDirMonitor.housekeep method.

	modified:   combox/events.py
	modified:   tests/events_test.py
---
 combox/events.py     | 57 ++++++++++++++++++++++++++++++----------------------
 tests/events_test.py | 30 +++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 24 deletions(-)

diff --git a/combox/events.py b/combox/events.py
index 1969a85..cc854ba 100644
--- a/combox/events.py
+++ b/combox/events.py
@@ -252,30 +252,39 @@ class NodeDirMonitor(LoggingEventHandler):
                 # deleted in the 'file_deleted' dict inside the silo.
                 self.silo.node_set('file_deleted', fpath, del_num)
 
-        for root, dirs, files in os.walk(get_nodedirs(self.config)[0]):
-            for f in files:
-                shard = path.join(root, f)
-
-                if not self.shardp(shard):
-                    continue
-
-                file_cb_path = cb_path(shard, self.config)
-                if not self.silo.exists(file_cb_path):
-                    print file_cb_path, "was created remotely. Creating it locally now..."
-                    decrypt_and_glue(file_cb_path, self.config)
-                    self.silo.update(file_cb_path)
-                elif self.silo.exists(file_cb_path):
-                    file_content = decrypt_and_glue(file_cb_path,
-                                                    self.config,
-                                                    write=False)
-                    file_content_hash = hash_file(file_cb_path, file_content)
-
-                    if self.silo.stale(file_cb_path, file_content_hash):
-                        # file modified
-                        print file_cb_path, "modified remotely. Updating local copy."
-                        decrypt_and_glue(file_cb_path, self.config)
-                        self.silo.update(file_cb_path)
-
+        # Re-construct files created on another computer when combox
+        # was switched off. Only files who's all the shards have made
+        # it to the combox directory are re-constructed.
+
+        # Dict that keeps track of the files that were created.
+        files_created = {}
+        for node_dir in get_nodedirs(self.config):
+            for root, dirs, files in os.walk(node_dir):
+                for f in files:
+                    shard = path.join(root, f)
+
+                    if not self.shardp(shard):
+                        continue
+
+                    file_cb_path = cb_path(shard, self.config)
+                    if not self.silo.exists(file_cb_path):
+                        if not files_created.get(file_cb_path):
+                            files_created[file_cb_path] = 1
+                        else:
+                            files_created[file_cb_path] += 1
+
+        for f_cb_path, crt_num in files_created.items():
+            if crt_num == self.num_nodes:
+                print f_cb_path, "was created remotely. Creating it locally now..."
+                decrypt_and_glue(f_cb_path, self.config)
+                # update silo.
+                self.silo.update(f_cb_path)
+                self.silo.node_rem('file_created', f_cb_path)
+            elif crt_num > 0:
+                # means, all the shards of the file have not arrived
+                # yet, so, we store the no. of shards that did arrive
+                # in the 'file_created' dict inside the silo.
+                self.silo.node_set('file_created', f_cb_path, crt_num)
 
         print "combox node monitor is done with the drudgery."
         print "Do what you want to the combox directory."
diff --git a/tests/events_test.py b/tests/events_test.py
index abb490a..a8b25ab 100644
--- a/tests/events_test.py
+++ b/tests/events_test.py
@@ -599,6 +599,36 @@ class TestEvents(object):
         self.purge_list.append(testf2)
 
 
+    def test_NDM_housekeep_create(self):
+        """Testing NodeDirMonitor's housekeep method's delete functionality."""
+
+        # test shard creation
+        hmutant = "%s.mutant" % self.TEST_FILE
+        hmutant_content = read_file(self.TEST_FILE)
+        split_and_encrypt(hmutant, self.config,
+                          hmutant_content)
+
+        lorem_c = "%s.c" % self.lorem
+        split_and_encrypt(lorem_c, self.config,
+                          read_file(self.lorem))
+        # delete lorem_c's shard0
+        remove(node_paths(lorem_c, self.config, True)[0])
+
+        ndm = NodeDirMonitor(self.config, self.silo_lock,
+                             self.nodem_lock)
+        ndm.housekeep()
+
+        assert path.exists(hmutant)
+        assert hmutant_content == read_file(hmutant)
+
+        assert not path.exists(lorem_c)
+        assert_equal(len(get_nodedirs(self.config))-1,
+                         self.silo.node_get('file_created', lorem_c))
+
+        self.purge_list.append(hmutant)
+        self.purge_list.append(lorem_c)
+
+
     def untest_NDM_housekeep(self):
         """Testing NodeDirMonitor's housekeep method."""
 
-- 
cgit v1.2.3