# -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Dr. Robert C. Green II.
#
# This file is part of Combox.
#
# Combox is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Combox is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Combox (see COPYING). If not, see
# <http://www.gnu.org/licenses/>.
import os
from hashlib import sha512
from os import path
from sys import exit
from glob import glob
from combox.config import get_nodedirs
from combox.log import log_e
def relative_path(p, config, comboxd=True):
"""Returns the relative path to the `p` w. r. t combox or node directory.
:param str p:
Path to a directory or file.
:param dict config:
A dictionary that contains configuration information about
combox.
:param bool comboxd:
True if `p` is a path relative to combox directory.
:returns:
Path relative to combox directory if `comboxd` is `True`,
otherwise returns path relative to the node directory.
:rtype: str
:raises ValueError:
If path `p` is not under the combox directory or the node
directory.
"""
directory = None
if comboxd:
directory = '%s/' % config['combox_dir']
else:
for node in get_nodedirs(config):
if p.startswith(node):
directory = '%s/' % node
if directory is None:
err_msg = "invalid path %s" % p
raise ValueError, err_msg
return p.partition(directory)[2]
def cb_path(node_path, config):
"""Returns absolute path of file/directory under the combox directory.
:param str node_path:
Path to a file/directory under a node directory.
:param dict config:
A dictionary that contains configuration information about
combox.
:returns:
Returns the corresponding path to the file/directory under the
combox directory.
:rtype: str
"""
if node_path[:-1].endswith('shard'):
# partition function is used to remove the `.shard.N' from the
# file name.
rel_file_path = relative_path(node_path,
config,
False).partition('.shard')[0]
file_cb_path = path.join(config['combox_dir'],
rel_file_path)
else:
file_cb_path = path.join(config['combox_dir'],
relative_path(node_path, config, False))
return file_cb_path
def node_path(cb_path, config, isfile):
"""Returns abs. path of file/directory in node directory.
:param str cb_path:
Path to a file/directory under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
:param bool isfile:
True if `cb_path` is not a directory.
:returns:
If `cb_path` is a file, it returns the path to its shard under
the first node directory.
If `cb_path` is a directory, it returns the corresponding path
to this directory under the first node directory.
:rtype: str
"""
if isfile:
# partition function is used to remove the `.shard.N' from the
# file name.
rel_file_path = relative_path(cb_path, config)
file_node_path = path.join(get_nodedirs(config)[0],
rel_file_path)
file_node_path = "%s.shard0" % file_node_path
else:
file_node_path = path.join(get_nodedirs(config)[0],
relative_path(cb_path, config))
return file_node_path
def node_paths(cb_path, config, isfile):
"""Returns a list of abs. paths of a file/directory in node directories.
:param str cb_path:
Path to a file/directory under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
:param bool isfile:
True if `cb_path` is not a directory.
:returns:
If `cb_path` is a file, it returns a list of paths to its
shards under the node directories.
If `cb_path` is a directory, it returns a list its
corresponding paths under the node directories.
:rtype: list
"""
n_paths = []
nodes = get_nodedirs(config)
rel_path = relative_path(cb_path, config)
if isfile:
shard_no = 0
for node in nodes:
file_shard = '%s.shard%d' % (rel_path, shard_no)
n_path = path.join(node, file_shard)
n_paths.append(n_path)
shard_no += 1
else:
for node in nodes:
n_path = path.join(node, rel_path)
n_paths.append(n_path)
return n_paths
def mk_nodedir(directory, config):
"""Creates directory `directory` under all the node directories.
:param str directory:
Path to a directory under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
"""
nodes = get_nodedirs(config)
rel_path = relative_path(directory, config)
for node in nodes:
dir_path = path.join(node, rel_path)
mk_dir(dir_path)
def mk_dir(directory):
"""Creates a directory.
:param str directory:
Path to a directory to create.
"""
try:
os.mkdir(directory)
except OSError, e:
log_e("Error when trying to make directory %s" % directory)
def rm_nodedir(directory, config):
"""Removes directory `directory` under all the node directories.
:param str directory:
Path to a directory under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
"""
nodes = get_nodedirs(config)
rel_path = relative_path(directory, config)
for node in nodes:
dir_path = path.join(node, rel_path)
rm_path(dir_path)
def rm_path(fpath):
"""Removes path `fpath`.
:param str fpath:
It can be a file or an empty directory.
"""
try:
if path.isfile(fpath):
os.remove(fpath)
elif path.isdir(fpath):
purge_dir(fpath)
os.rmdir(fpath)
except OSError, e:
log_e("Error when trying to remove path %s" % fpath)
def move_nodedir(src, dest, config):
"""Moves directory `src` to `dest`.
:param str src:
Old path to the directory.
:param str dest:
New path to the directory
:param dict config:
A dictionary that contains configuration information about
combox.
"""
nodes = get_nodedirs(config)
src_rel_path = relative_path(src, config)
dest_rel_path = relative_path(dest, config)
for node in nodes:
src_dir_path = path.join(node, src_rel_path)
dest_dir_path = path.join(node, dest_rel_path)
try:
os.renames(src_dir_path, dest_dir_path)
except OSError, e:
log_e("Error when trying to rename %s -> %s" % (src_dir_path, dest_dir_path))
def rm_shards(fpath, config):
"""Removes the file shards of `fpath` under the node directories.
:param str fpath:
Path to file under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
"""
nodes = get_nodedirs(config)
rel_path = relative_path(fpath, config)
for node in nodes:
shard_glob = "%s.shard*" % path.join(node, rel_path)
shard_glob = glob(shard_glob)
if not len(shard_glob):
# shard was already deleted.
continue
# there's always only one shard in each node directory. So,
# the glob() will alawys return a list of size 1.
shard = shard_glob[0]
try:
os.remove(shard)
except OSError, e:
log_e("Error when trying to remove shard %s" % shard)
def move_shards(src, dest, config):
"""Move the shards in node directories.
This function is used when a file is moved to different location
inside the combox directory. It moves the shards of the respective
file to the new location under the node directories.
:param str src:
Old path to the file that was moved under the combox directory.
:param str dest:
New path to the file that was moved under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
"""
nodes = get_nodedirs(config)
src_rel_path = relative_path(src, config)
dest_rel_path = relative_path(dest, config)
for node in nodes:
src_shard_glob = "%s.shard*" % path.join(node, src_rel_path)
# there's always only one shard in each node directory.
glob_list = glob(src_shard_glob)
if glob_list:
src_shard = glob_list[0]
else:
# shards are not there!, so we return.
return
# get shard number
shard_no = src_shard.partition('.shard')[2]
dest_shard = "%s.shard%s" % (path.join(node, dest_rel_path),
shard_no)
try:
os.renames(src_shard, dest_shard)
except OSError, e:
log_e("Error when trying to rename shard %s -> %s" % (src_shard, dest_shard))
def purge_dir(p):
"""
Purge everything under the given directory `p`.
Directory `p` itself is not deleted.
:param str p:
Path to directory that needs to be purged.
"""
p = path.abspath(p)
if path.isfile(p):
return os.remove(p)
for f in os.listdir(p):
f_path = path.join(p, f)
if path.isfile(f_path):
os.remove(f_path)
else:
purge_dir(f_path)
os.rmdir(f_path)
def split_data(data, n):
"""Split `data` into `n` parts and return them as a list.
:param str data:
Stream of bytes or string.
:param int n:
Number of parts the `data` has to be split.
:returns:
List of strings -- the `data` divided into `n` parts.
:rtype: list
"""
d_parts = []
# File size in bytes.
data_size = len(data)
# No. of bytes for each data part.
part_size = data_size / n
# Take note of remaining bytes; this is non-zero when data_size is
# not divisible by `n'.
rem_bytes = data_size % n
start = 0
end = part_size
while end <= data_size:
d_parts.append(data[start:end])
start = end
end = end + part_size
# read the remaining bytes into the last data part.
end += start + rem_bytes
d_parts[n-1] += data[start:end]
return d_parts
def glue_data(d_parts):
"""Glue different parts of the `data` into a single whole.
:param list d_parts:
List containing different parts of the `data`. Each part is a
sequence of bytes.
:returns:
The `data` glued into a single whole.
:rtype: str
"""
data = ''
for part in d_parts:
data += part
return data
def read_file(filename):
"""Read file `filename` and return it as a string.
:param str filename:
Absolute pathname of the file.
:returns:
Content of file `filename` as a string.
:rtype: str
"""
file_ = None
content = ''
with open(filename, 'rb') as f:
for line in f:
content = content + line
return content
def hash_file(filename, file_content=None):
"""Does a SHA512 hash on the contents of file.
Returns the hexdigest of the file content's hash.
:param str filename:
Absolute pathname of the file.
:param str file_content:
If not ``None``, SHA512 hash of `file_content` is returned.
:returns:
If `file_content` is ``None``, returns the SHA512 hash of
contents of file `filename`.
If `file_content` is not ``None``, returns the SHA512 hash of
`file_content`.
:rtype: str
"""
if not file_content:
file_content = read_file(filename)
return sha512(file_content).hexdigest()
def write_file(filename, filecontent):
"""Write `filecontent` to `filename`.
:param str filename:
Absolute pathname of the file.
:param str filecontent:
Data to write to `filename`.
"""
file_ = None
try:
file_ = open(filename, 'wb')
file_.write(filecontent)
file_.close()
except IOError:
log_e("Error creating and writing content to %s" % filename)
exit(1)
def write_shards(shards, directories, shard_basename):
"""Write shards to node directories.
:param list shards:
List of strings (data).
:param list directories:
List of paths of node directories.
:param str shard_basename:
Base name for the shards.
"""
shard_no = 0
for directory in directories:
# partial filename of the shard
p_filename = path.join(directory, shard_basename)
shard_name = "%s.shard%s" % (p_filename, shard_no)
write_file(shard_name, shards[shard_no])
shard_no += 1
def read_shards(directories, shard_basename):
"""Read the shards of a file from node directories and return it as a list.
:param list directories:
List of paths of node directories from which to read the
shards.
:param str shard_basename:
Base name for the shards; the canonical file name.
:returns:
List of contents of the shards of the file with basename
`shard_basename`.
"""
# get the names of the file shards
file_shards = []
for directory in directories:
filename_glob = "%s.shard*" % path.join(directory, shard_basename)
file_shard = glob(filename_glob)[0]
file_shards.append(file_shard)
shards = []
for file_shard in sorted(file_shards):
shard_content = read_file(file_shard)
shards.append(shard_content)
return shards
def no_of_shards(cb_path, config):
"""Returns the no. of shards that exists for `cb_path` under the node directories.
:param str cb_path:
Path to a file under the combox directory.
:param dict config:
A dictionary that contains configuration information about
combox.
:returns:
The number of shards that exists for the file `cb_path`.
:rtype: int
"""
no_shards_there = 0
shard_paths = node_paths(cb_path, config, isfile=True)
for shard in shard_paths:
if path.isfile(shard):
no_shards_there += 1
return no_shards_there
