From 277408c058283bf73fbe6355fc7db0f6dbcbc24b Mon Sep 17 00:00:00 2001 From: Siddharth Ravikumar Date: Wed, 1 Apr 2015 09:45:45 -0400 Subject: refactored the way locks work in combox.siloComboxSilo. modified: combox/cbox.py modified: combox/events.py modified: combox/silo.py modified: tests/events_test.py modified: tests/silo_test.py --- combox/cbox.py | 4 +++- combox/events.py | 13 ++++++----- combox/silo.py | 14 ++++++++---- tests/events_test.py | 64 +++++++++++++++++++++++++++------------------------- tests/silo_test.py | 20 ++++++++-------- 5 files changed, 64 insertions(+), 51 deletions(-) diff --git a/combox/cbox.py b/combox/cbox.py index 4424274..add5883 100644 --- a/combox/cbox.py +++ b/combox/cbox.py @@ -25,6 +25,7 @@ import yaml from argparse import ArgumentParser from os import path from sys import exit +from threading import Lock from watchdog.observers import Observer from combox.config import config_cb @@ -39,7 +40,8 @@ def run_cb(config): Runs combox. """ c_path = path.abspath(config['combox_dir']) - event_handler = ComboxDirMonitor(config) + db_lock = Lock() + event_handler = ComboxDirMonitor(config, db_lock) observer = Observer() observer.schedule(event_handler, c_path, recursive=True) diff --git a/combox/events.py b/combox/events.py index fa47ceb..3ef4386 100644 --- a/combox/events.py +++ b/combox/events.py @@ -20,6 +20,7 @@ import os import logging from os import path +from threading import Lock from watchdog.events import LoggingEventHandler @@ -37,7 +38,7 @@ class ComboxDirMonitor(LoggingEventHandler): """ - def __init__(self, config): + def __init__(self, config, dblock): """ config: a dictinary which contains combox configuration. """ @@ -48,7 +49,7 @@ class ComboxDirMonitor(LoggingEventHandler): datefmt='%Y-%m-%d %H:%M:%S') self.config = config - self.silo = ComboxSilo(self.config) + self.silo = ComboxSilo(self.config, dblock) self.housekeep() @@ -57,7 +58,7 @@ class ComboxDirMonitor(LoggingEventHandler): """ Re-reads the silo from disk. """ - self.silo = ComboxSilo(self.config) + self.silo.reload() def housekeep(self): @@ -186,7 +187,7 @@ class NodeDirMonitor(LoggingEventHandler): """ - def __init__(self, config): + def __init__(self, config, dblock): """ config: a dictinary which contains combox configuration. """ @@ -197,14 +198,14 @@ class NodeDirMonitor(LoggingEventHandler): datefmt='%Y-%m-%d %H:%M:%S') self.config = config - self.silo = ComboxSilo(self.config) + self.silo = ComboxSilo(self.config, dblock) def silo_update(self): """ Re-reads the silo from disk. """ - self.silo = ComboxSilo(self.config) + self.silo.reload() def shardp(self, path): diff --git a/combox/silo.py b/combox/silo.py index 55345c9..f391d2e 100644 --- a/combox/silo.py +++ b/combox/silo.py @@ -30,14 +30,14 @@ class ComboxSilo(object): """ - def __init__(self, config): + def __init__(self, config, lock): """config: a dictinary which contains combox configuration. """ self.config = config - silo = path.join(config['silo_dir'], 'silo.db') - self.db = pickledb.load(silo, True) + self.silo_path = path.join(config['silo_dir'], 'silo.db') + self.db = pickledb.load(self.silo_path, True) ## things we need for housekeep the node directory. self.node_dicts = ['shard_created', 'shard_modified', 'shard_moved', @@ -48,7 +48,13 @@ class ComboxSilo(object): if not self.db.get(ndict): self.db.dcreate(ndict) - self.lock = Lock() + self.lock = lock + + + def reload(self): + """Re-loads the DB from disk.""" + with self.lock: + self.db = pickledb.load(self.silo_path, True) def update(self, filep): diff --git a/tests/events_test.py b/tests/events_test.py index 496c78e..d754d44 100644 --- a/tests/events_test.py +++ b/tests/events_test.py @@ -24,6 +24,7 @@ from filecmp import cmp from glob import glob from os import path, remove from shutil import copyfile +from threading import Lock from nose.tools import * from watchdog.observers import Observer @@ -51,6 +52,7 @@ class TestEvents(object): def setup_class(self): """Set things up.""" + self.silo_lock = Lock() self.config = get_config() self.FILES_DIR = self.config['combox_dir'] self.NODE_DIR = get_nodedirs(self.config)[0] @@ -70,7 +72,7 @@ class TestEvents(object): Tests the ComboxDirMonitor class. """ - event_handler = ComboxDirMonitor(self.config) + event_handler = ComboxDirMonitor(self.config, self.silo_lock) observer = Observer() observer.schedule(event_handler, self.FILES_DIR, recursive=True) observer.start() @@ -84,7 +86,7 @@ class TestEvents(object): ## check if the shards were created. shardedp(self.TEST_FILE_COPY_0) ## check if the new file's info is in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert silo.exists(self.TEST_FILE_COPY_0) # Test - File deletion. @@ -92,7 +94,7 @@ class TestEvents(object): time.sleep(1) path_deletedp(self.TEST_FILE_COPY_0) ## check if the new file's info is removed from silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.exists(self.TEST_FILE_COPY_0) # Test - directory creation @@ -126,7 +128,7 @@ class TestEvents(object): renamedp(self.TEST_DIR_1, self.TEST_DIR_1_NEW) renamedp(self.TEST_FILE_COPY_1, self.TEST_FILE_COPY_1_NEW) ## check if the new file's info is updated in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.exists(self.TEST_FILE_COPY_1) assert silo.exists(self.TEST_FILE_COPY_1_NEW) @@ -148,7 +150,7 @@ class TestEvents(object): time.sleep(1) shardedp(self.lorem_file_copy) ## check if the lorem_file_copy's info is stored in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) lorem_file_copy_hash = silo.db.get(self.lorem_file_copy) self.ipsum_file = path.join(self.FILES_DIR, 'ipsum.txt') @@ -160,7 +162,7 @@ class TestEvents(object): write_file(self.lorem_file_copy, lorem_copy_content) time.sleep(1) ## check if the lorem_file_copy's info is updated in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert lorem_file_copy_hash != silo.db.get(self.lorem_file_copy) @@ -178,7 +180,7 @@ class TestEvents(object): time.sleep(1) path_deletedp(self.lorem_file_copy) ## check if the lorem_file_copy's info is deleted from silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.exists(self.lorem_file_copy) observer.stop() @@ -191,10 +193,10 @@ class TestEvents(object): # test file deletion and addition os.rename(self.lorem, self.lorem_moved) - cdm = ComboxDirMonitor(self.config) + cdm = ComboxDirMonitor(self.config, self.silo_lock) cdm.housekeep() - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.exists(self.lorem) assert silo.exists(self.lorem_moved) shardedp(self.lorem_moved) @@ -208,10 +210,10 @@ class TestEvents(object): copyfile(self.lorem, self.lorem_ipsum) assert path.exists(self.lorem_ipsum) - cdm = ComboxDirMonitor(self.config) + cdm = ComboxDirMonitor(self.config, self.silo_lock) cdm.housekeep() - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert silo.exists(self.lorem_ipsum) ipsum_content = read_file(self.ipsum) @@ -222,7 +224,7 @@ class TestEvents(object): cdm.housekeep() - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.stale(self.lorem_ipsum) @@ -231,7 +233,7 @@ class TestEvents(object): Tests the NodeDirMonitor class. """ - event_handler = NodeDirMonitor(self.config) + event_handler = NodeDirMonitor(self.config, self.silo_lock) observer = Observer() observer.schedule(event_handler, self.NODE_DIR, recursive=True) observer.start() @@ -248,7 +250,7 @@ class TestEvents(object): time.sleep(1) assert fmutant_content == read_file(self.TEST_FILE_MUTANT) ## check if the new file's info is in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert silo.exists(self.TEST_FILE_MUTANT) # Test - directory creation @@ -272,7 +274,7 @@ class TestEvents(object): assert not path.exists(self.TEST_FILE_MUTANT) ## check if the new file's info is removed from silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert not silo.exists(self.TEST_FILE_MUTANT) # Test - directory deletion inside node directory @@ -289,11 +291,11 @@ class TestEvents(object): split_and_encrypt(self.lorem_file_copy, self.config, lorem_content) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.update(self.lorem_file_copy) shardedp(self.lorem_file_copy) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) lorem_file_copy_hash = silo.db.get(self.lorem_file_copy) self.ipsum_file = path.join(self.FILES_DIR, 'ipsum.txt') @@ -304,7 +306,7 @@ class TestEvents(object): lorem_copy_content) time.sleep(1) ## check if the lorem_file_copy's info is updated in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert lorem_copy_content == read_file(self.lorem_file_copy) assert lorem_file_copy_hash != silo.db.get(self.lorem_file_copy) @@ -318,7 +320,7 @@ class TestEvents(object): def test_NDM_onmoved(self): """Testing on_moved method in NodeDirMonitor""" - event_handler = NodeDirMonitor(self.config) + event_handler = NodeDirMonitor(self.config, self.silo_lock) observer = Observer() observer.schedule(event_handler, self.NODE_DIR, recursive=True) observer.start() @@ -326,7 +328,7 @@ class TestEvents(object): self.testf = "%s.onm" % self.TEST_FILE copyfile(self.TEST_FILE, self.testf) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.update(self.testf) split_and_encrypt(self.testf, self.config) @@ -339,7 +341,7 @@ class TestEvents(object): time.sleep(1) assert path.isfile(self.testf_moved) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert silo.exists(self.testf_moved) # test directory move/rename @@ -352,7 +354,7 @@ class TestEvents(object): split_and_encrypt(dirt_lorem, self.config) time.sleep(1) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.update(dirt_lorem) dirt_m = path.join(self.FILES_DIR, "mof") @@ -363,7 +365,7 @@ class TestEvents(object): assert path.isdir(dirt_m) assert path.isfile(dirt_m_lorem) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert silo.exists(dirt_m_lorem) self.purge_list.append(self.testf_moved) @@ -382,11 +384,11 @@ class TestEvents(object): copyfile(self.TEST_FILE, testf1) copyfile(self.lorem, testf2) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.update(testf1) silo.update(testf2) - ndm = NodeDirMonitor(self.config) + ndm = NodeDirMonitor(self.config, self.silo_lock) ndm.housekeep() assert not path.exists(testf1) @@ -402,7 +404,7 @@ class TestEvents(object): split_and_encrypt(hmutant, self.config, hmutant_content) - ndm = NodeDirMonitor(self.config) + ndm = NodeDirMonitor(self.config, self.silo_lock) ndm.housekeep() assert path.exists(hmutant) @@ -417,11 +419,11 @@ class TestEvents(object): split_and_encrypt(lcopy, self.config, lcopy_content) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.update(lcopy) shardedp(lcopy) - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) lcopy_hash = silo.db.get(lcopy) ipsum_content = read_file(self.ipsum) @@ -430,11 +432,11 @@ class TestEvents(object): split_and_encrypt(lcopy, self.config, lcopy_content) - ndm = NodeDirMonitor(self.config) + ndm = NodeDirMonitor(self.config, self.silo_lock) ndm.housekeep() ## check if the lorem_file_copy's info is updated in silo - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert lcopy_content == read_file(lcopy) assert hash_file(lcopy, lcopy_content) == silo.db.get(lcopy) @@ -446,7 +448,7 @@ class TestEvents(object): """Testing shardp method in NodeDirMonitor class""" shard = 'some.shard0' not_shard = 'some.extension' - ndm = NodeDirMonitor(self.config) + ndm = NodeDirMonitor(self.config, self.silo_lock) assert_equal(True, ndm.shardp(shard)) assert_equal(False, ndm.shardp(not_shard)) diff --git a/tests/silo_test.py b/tests/silo_test.py index 9c4fc8f..74f3002 100644 --- a/tests/silo_test.py +++ b/tests/silo_test.py @@ -20,6 +20,7 @@ import yaml from shutil import copyfile from os import path, remove +from threading import Lock from combox.silo import ComboxSilo from combox.file import read_file, write_file, hash_file @@ -37,6 +38,7 @@ class TestSilo(object): def setup_class(self): """Set things up.""" + self.silo_lock = Lock() self.config = get_config() self.FILES_DIR = self.config['combox_dir'] @@ -49,7 +51,7 @@ class TestSilo(object): """ Tests the ComboxSilo class. """ - csilo = ComboxSilo(self.config) + csilo = ComboxSilo(self.config, self.silo_lock) # Test - update csilo.update(self.LOREM) @@ -100,7 +102,7 @@ class TestSilo(object): node directories are created. """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) keys = silo.db.db.keys() node_dicts = ['shard_created', 'shard_modified', 'shard_moved', @@ -113,7 +115,7 @@ class TestSilo(object): """Tests node_set method, in ComboxSilo class, when type is 'shard_created'. """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.node_set('shard_created', self.LOREM) silo.node_set('shard_created', self.LOREM) silo.node_set('shard_created', self.LOREM) @@ -132,7 +134,7 @@ class TestSilo(object): """Tests node_set method, in ComboxSilo class, when type is 'shard_modified'. """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.node_set('shard_modified', self.LOREM) silo.node_set('shard_modified', self.LOREM) silo.node_set('shard_modified', self.LOREM) @@ -151,7 +153,7 @@ class TestSilo(object): """Tests node_set method, in ComboxSilo class, when type is 'shard_moved'. """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.node_set('shard_moved', self.LOREM) silo.node_set('shard_moved', self.LOREM) silo.node_set('shard_moved', self.LOREM) @@ -170,7 +172,7 @@ class TestSilo(object): """Tests node_set method in ComboxSilo class, when type is 'shard_deleted'. """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.node_set('shard_deleted', self.LOREM) silo.node_set('shard_deleted', self.LOREM) silo.node_set('shard_deleted', self.LOREM) @@ -188,7 +190,7 @@ class TestSilo(object): def test_csilo_nodeget(self): """Tests node_get method in ComboxSilo class """ - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) assert_equal(None, silo.node_get('shard_created', self.LOREM)) silo.node_set('shard_created', self.LOREM) @@ -199,14 +201,14 @@ class TestSilo(object): def teardown(self): """Cleans up things after each test in this class""" - silo = ComboxSilo(self.config) + silo = ComboxSilo(self.config, self.silo_lock) silo.db.deldb() @classmethod def teardown_class(self): """Purge the mess created by this test""" - csilo = ComboxSilo(self.config) + csilo = ComboxSilo(self.config, self.silo_lock) csilo.remove(self.LOREM) csilo.remove(self.IPSUM) rm_nodedirs(self.config) -- cgit v1.2.3