From d0e29c74f84a2ed3014a516c0106172619314bdc Mon Sep 17 00:00:00 2001 From: Anirudha Bose Date: Tue, 14 Jun 2016 21:42:02 +0530 Subject: [PATCH] ceph-disk: Compatibility fixes for Python 3 ceph-disk: Misc cleanups Signed-off-by: Anirudha Bose --- src/ceph-disk/ceph_disk/main.py | 90 +++++++++++++++++--------------- src/ceph-disk/tests/test_main.py | 13 +++-- src/ceph-disk/tox.ini | 4 +- 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py index 3593076a9d8..902545691d0 100755 --- a/src/ceph-disk/ceph_disk/main.py +++ b/src/ceph-disk/ceph_disk/main.py @@ -18,6 +18,8 @@ # GNU Library Public License for more details. # +from __future__ import print_function + import argparse import base64 import errno @@ -250,7 +252,7 @@ class filelock(object): def acquire(self): assert not self.fd - self.fd = file(self.fn, 'w') + self.fd = open(self.fn, 'w') fcntl.lockf(self.fd, fcntl.LOCK_EX) def release(self): @@ -265,8 +267,8 @@ class Error(Exception): """ def __str__(self): - doc = self.__doc__.strip() - return ': '.join([doc] + [str(a) for a in self.args]) + doc = _bytes2str(self.__doc__.strip()) + return ': '.join([doc] + [_bytes2str(a) for a in self.args]) class MountError(Error): @@ -324,11 +326,8 @@ def is_systemd(): """ Detect whether systemd is running """ - with file('/proc/1/comm', 'rb') as i: - for line in i: - if 'systemd' in line: - return True - return False + with open('/proc/1/comm', 'r') as f: + return 'systemd' in f.read() def is_upstart(): @@ -336,9 +335,7 @@ def is_upstart(): Detect whether upstart is running """ (out, err, _) = command(['init', '--version']) - if 'upstart' in out: - return True - return False + return 'upstart' in out def maybe_mkdir(*a, **kw): @@ -352,7 +349,7 @@ def maybe_mkdir(*a, **kw): os.unlink(*a) try: os.mkdir(*a, **kw) - except OSError, e: + except OSError as e: if e.errno == errno.EEXIST: pass else: @@ -361,10 +358,7 @@ def maybe_mkdir(*a, **kw): def which(executable): """find the location of an executable""" - if 'PATH' in os.environ: - envpath = os.environ['PATH'] - else: - envpath = os.defpath + envpath = os.environ.get('PATH') or os.defpath PATH = envpath.split(os.pathsep) locations = PATH + [ @@ -414,7 +408,9 @@ def command(arguments, **kwargs): This returns the output of the command and the return code of the process in a tuple: (output, returncode). """ - arguments = _get_command_executable(arguments) + + arguments = list(map(_bytes2str, _get_command_executable(arguments))) + LOG.info('Running command: %s' % ' '.join(arguments)) process = subprocess.Popen( arguments, @@ -422,7 +418,12 @@ def command(arguments, **kwargs): stderr=subprocess.PIPE, **kwargs) out, err = process.communicate() - return out, err, process.returncode + + return _bytes2str(out), _bytes2str(err), process.returncode + + +def _bytes2str(string): + return string.decode('utf-8') if isinstance(string, bytes) else string def command_check_call(arguments): @@ -744,7 +745,7 @@ def is_mounted(dev): Check if the given device is mounted. """ dev = os.path.realpath(dev) - with file('/proc/mounts', 'rb') as proc_mounts: + with open('/proc/mounts', 'rb') as proc_mounts: for line in proc_mounts: fields = line.split() if len(fields) < 3: @@ -754,7 +755,7 @@ def is_mounted(dev): if mounts_dev.startswith('/') and os.path.exists(mounts_dev): mounts_dev = os.path.realpath(mounts_dev) if mounts_dev == dev: - return path + return _bytes2str(path) return None @@ -819,6 +820,8 @@ def must_be_one_line(line): :raises: TruncatedLineError or TooManyLinesError :return: Content of the line, or None if line isn't valid. """ + line = _bytes2str(line) + if line[-1:] != '\n': raise TruncatedLineError(line) line = line[:-1] @@ -837,7 +840,7 @@ def read_one_line(parent, name): """ path = os.path.join(parent, name) try: - line = file(path, 'rb').read() + line = open(path, 'rb').read() except IOError as e: if e.errno == errno.ENOENT: return None @@ -864,8 +867,8 @@ def write_one_line(parent, name, text): """ path = os.path.join(parent, name) tmp = '{path}.{pid}.tmp'.format(path=path, pid=os.getpid()) - with file(tmp, 'wb') as tmp_file: - tmp_file.write(text + '\n') + with open(tmp, 'wb') as tmp_file: + tmp_file.write(text.encode('utf-8') + b'\n') os.fsync(tmp_file.fileno()) path_set_context(tmp) os.rename(tmp, path) @@ -957,7 +960,7 @@ def get_ceph_user(): pwd.getpwnam(CEPH_PREF_USER) return CEPH_PREF_USER except KeyError: - print "No such user: " + CEPH_PREF_USER + print("No such user:", CEPH_PREF_USER) sys.exit(2) else: try: @@ -975,7 +978,7 @@ def get_ceph_group(): grp.getgrnam(CEPH_PREF_GROUP) return CEPH_PREF_GROUP except KeyError: - print "No such group: " + CEPH_PREF_GROUP + print("No such group:", CEPH_PREF_GROUP) sys.exit(2) else: try: @@ -1002,7 +1005,7 @@ def _check_output(args=None, **kwargs): error = subprocess.CalledProcessError(ret, cmd) error.output = out + err raise error - return out + return _bytes2str(out) def get_conf(cluster, variable): @@ -1145,7 +1148,7 @@ def _dmcrypt_map( if dev: return dev - if isinstance(key, types.TupleType): + if isinstance(key, tuple): # legacy, before lockbox assert os.path.exists(key[0]) keypath = key[0] @@ -1431,9 +1434,9 @@ def zap(dev): # isn't too thorough. lba_size = 4096 size = 33 * lba_size - with file(dev, 'wb') as dev_file: + with open(dev, 'wb') as dev_file: dev_file.seek(-size, os.SEEK_END) - dev_file.write(size * '\0') + dev_file.write(size * b'\0') command_check_call( [ @@ -1977,7 +1980,7 @@ class PrepareSpace(object): ' (ceph-osd will resize and allocate)', self.name, getattr(self.args, self.name)) - with file(getattr(self.args, self.name), 'wb') as space_file: + with open(getattr(self.args, self.name), 'wb') as space_file: pass LOG.debug('%s is file %s', @@ -3193,7 +3196,7 @@ def find_cluster_by_uuid(_uuid): try: fsid = get_fsid(cluster) except Error as e: - if e.message != 'getting cluster uuid from configuration failed': + if 'getting cluster uuid from configuration failed' not in str(e): raise e no_fsid.append(cluster) else: @@ -3268,7 +3271,7 @@ def activate( LOG.debug('Marking with init system %s', init) init_path = os.path.join(path, init) - with file(init_path, 'w'): + with open(init_path, 'w'): path_set_context(init_path) # remove markers for others, just in case. @@ -3797,10 +3800,11 @@ def main_activate_all(args): ) except Exception as e: - print >> sys.stderr, '{prog}: {msg}'.format( - prog=args.prog, - msg=e, + print( + '{prog}: {msg}'.format(prog=args.prog, msg=e), + file=sys.stderr ) + err = True finally: @@ -3813,7 +3817,7 @@ def main_activate_all(args): def is_swap(dev): dev = os.path.realpath(dev) - with file('/proc/swaps', 'rb') as proc_swaps: + with open('/proc/swaps', 'rb') as proc_swaps: for line in proc_swaps.readlines()[1:]: fields = line.split() if len(fields) < 3: @@ -3829,8 +3833,8 @@ def is_swap(dev): def get_oneliner(base, name): path = os.path.join(base, name) if os.path.isfile(path): - with open(path, 'r') as _file: - return _file.readline().rstrip() + with open(path, 'rb') as _file: + return _bytes2str(_file.readline().rstrip()) return None @@ -4120,7 +4124,7 @@ def list_devices(): uuid_map = {} space_map = {} - for base, parts in sorted(partmap.iteritems()): + for base, parts in sorted(partmap.items()): for p in parts: dev = get_dev_path(p) part_uuid = get_partition_uuid(dev) @@ -4159,7 +4163,7 @@ def list_devices(): str(uuid_map) + ", space_map = " + str(space_map)) devices = [] - for base, parts in sorted(partmap.iteritems()): + for base, parts in sorted(partmap.items()): if parts: disk = {'path': get_dev_path(base)} partitions = [] @@ -4202,11 +4206,11 @@ def main_list_protected(args): else: selected_devices = devices if args.format == 'json': - print json.dumps(selected_devices) + print(json.dumps(selected_devices)) else: output = list_format_plain(selected_devices) if output: - print output + print(output) ########################### @@ -4244,7 +4248,7 @@ def set_suppress(path): raise Error('not a block device', path) base = get_dev_name(disk) - with file(SUPPRESS_PREFIX + base, 'w') as f: # noqa + with open(SUPPRESS_PREFIX + base, 'w') as f: # noqa pass LOG.info('set suppress flag on %s', base) diff --git a/src/ceph-disk/tests/test_main.py b/src/ceph-disk/tests/test_main.py index f59824c0187..d76a5d8b604 100644 --- a/src/ceph-disk/tests/test_main.py +++ b/src/ceph-disk/tests/test_main.py @@ -21,6 +21,11 @@ import tempfile import unittest from ceph_disk import main +try: + import builtins +except: + import __builtin__ as builtins + def fail_to_mount(dev, fstype, options): raise main.MountError(dev + " mount fail") @@ -147,7 +152,7 @@ class TestCephDisk(object): main.PTYPE['plain']['osd']['ready']: 'plain', main.PTYPE['luks']['osd']['ready']: 'luks', } - for (ptype, type) in ptype2type.iteritems(): + for (ptype, type) in ptype2type.items(): for holders in ((), ("dm_0",), ("dm_0", "dm_1")): dev = { 'dmcrypt': { @@ -175,7 +180,7 @@ class TestCephDisk(object): main.PTYPE['plain']['journal']['ready']: 'plain', main.PTYPE['luks']['journal']['ready']: 'luks', } - for (ptype, type) in ptype2type.iteritems(): + for (ptype, type) in ptype2type.items(): for holders in ((), ("dm_0",)): dev = { 'path': '/dev/Xda2', @@ -314,7 +319,7 @@ class TestCephDisk(object): main.PTYPE['plain']['osd']['ready']: 'plain', main.PTYPE['luks']['osd']['ready']: 'LUKS', } - for (partition_type, type) in partition_type2type.iteritems(): + for (partition_type, type) in partition_type2type.items(): # # dmcrypt data partition with one holder # @@ -682,7 +687,7 @@ class TestCephDiskDeactivateAndDestroy(unittest.TestCase): def setup_class(self): main.setup_logging(verbose=True, log_stdout=False) - @patch('__builtin__.open') + @patch('{0}.open'.format(builtins.__name__)) def test_main_deactivate(self, mock_open): data = tempfile.mkdtemp() main.setup_statedir(data) diff --git a/src/ceph-disk/tox.ini b/src/ceph-disk/tox.ini index 786136e058f..109c34ecaae 100644 --- a/src/ceph-disk/tox.ini +++ b/src/ceph-disk/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,py27 +envlist = flake8,py27,py34 [testenv] setenv = @@ -23,4 +23,4 @@ commands = coverage run --append --source=ceph_disk {envbindir}/py.test -vv {tox coverage report --omit=*test*,*tox* --show-missing [testenv:flake8] -commands = flake8 --ignore=H105,H405 ceph_disk tests +commands = flake8 --ignore=H105,H405,E127 ceph_disk tests