ceph/qa/workunits/rgw/test_rgw_datacache.py

#!/usr/bin/python3

import logging as log
from configobj import ConfigObj
import subprocess
import json
import os

"""
Runs a test against a rgw with the data cache enabled. A client must be
set in the config for this task. This client must be the same client
that is in the config for the `rgw` task.

In the `overrides` section `datacache` and `datacache` must be configured for
the `rgw` task and the ceph conf overrides must contain the below config
variables in the client section.

`s3cmd` must be added as an extra_package to the install task.

In the `workunit` task, `- rgw/run-datacache.sh` must be set for the client that
is in the config for the `rgw` task. The `RGW_DATACACHE_PATH` variable must be
set in the workunit's `env` and it must match the `datacache_path` given to the
`rgw` task in `overrides`.
Ex:
- install:
    extra_packages:
      deb: ['s3cmd']
      rpm: ['s3cmd']
- overrides:
    rgw:
      datacache: true
      datacache_path: /tmp/rgw_datacache
    install:
      extra_packages:
        deb: ['s3cmd']
        rpm: ['s3cmd']
    ceph:
      conf:
        client:
          rgw d3n l1 datacache persistent path: /tmp/rgw_datacache/
          rgw d3n l1 datacache size: 10737417240
          rgw d3n l1 local datacache enabled: true
          rgw enable ops log: true
- rgw:
    client.0:
- workunit:
    clients:
      client.0:
      - rgw/run-datacache.sh
    env:
      RGW_DATACACHE_PATH: /tmp/rgw_datacache
    cleanup: true
"""

log.basicConfig(level=log.DEBUG)

""" Constants """
USER = 'rgw_datacache_user'
DISPLAY_NAME = 'DatacacheUser'
ACCESS_KEY = 'NX5QOQKC6BH2IDN8HC7A'
SECRET_KEY = 'LnEsqNNqZIpkzauboDcLXLcYaWwLQ3Kop0zAnKIn'
BUCKET_NAME = 'datacachebucket'
FILE_NAME = '7M.dat'
GET_FILE_NAME = '7M-get.dat'

def exec_cmd(cmd):
    log.debug("exec_cmd(%s)", cmd)
    try:
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
        out, err = proc.communicate()
        if proc.returncode == 0:
            log.info('command succeeded')
            if out is not None: log.info(out)
            return out
        else:
            raise Exception("error: %s \nreturncode: %s" % (err, proc.returncode))
    except Exception as e:
        log.error('command failed')
        log.error(e)
        return False

def get_radosgw_endpoint():
    out = exec_cmd('sudo netstat -nltp | egrep "rados|valgr"')  # short for radosgw/valgrind
    x = out.decode('utf8').split(" ")
    port = [i for i in x if ':' in i][0].split(':')[1]
    log.info('radosgw port: %s' % port)
    proto = "http"
    hostname = '127.0.0.1'

    if port == '443':
        proto = "https"

    endpoint = hostname

    log.info("radosgw endpoint is: %s", endpoint)
    return endpoint, proto

def create_s3cmd_config(path, proto):
    """
    Creates a minimal config file for s3cmd
    """
    log.info("Creating s3cmd config...")

    use_https_config = "False"
    log.info("proto for s3cmd config is %s", proto)
    if proto == "https":
        use_https_config = "True"

    s3cmd_config = ConfigObj(
        indent_type='',
        infile={
            'default':
                {
                'host_bucket': 'no.way.in.hell',
                'use_https': use_https_config,
                },
            }
    )

    f = open(path, 'wb')
    s3cmd_config.write(f)
    f.close()
    log.info("s3cmd config written")

def get_cmd_output(cmd_out):
    out = cmd_out.decode('utf8')
    out = out.strip('\n')
    return out

def main():
    """
    execute the datacache test
    """
    # setup for test
    cache_dir = os.environ['RGW_DATACACHE_PATH']
    log.debug("datacache dir from config is: %s", cache_dir)

    out = exec_cmd('pwd')
    pwd = get_cmd_output(out)
    log.debug("pwd is: %s", pwd)

    endpoint, proto = get_radosgw_endpoint()

    # create 7M file to put
    outfile = pwd + '/' + FILE_NAME
    exec_cmd('dd if=/dev/urandom of=%s bs=1M count=7' % (outfile))

    # create user
    exec_cmd('radosgw-admin user create --uid %s --display-name %s --access-key %s --secret %s'
            % (USER, DISPLAY_NAME, ACCESS_KEY, SECRET_KEY))

    # create s3cmd config
    s3cmd_config_path = pwd + '/s3cfg'
    create_s3cmd_config(s3cmd_config_path, proto)

    # create a bucket
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s mb s3://%s'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME))

    # put an object in the bucket
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s put %s s3://%s'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, outfile, BUCKET_NAME))

    # get object from bucket
    get_file_path = pwd + '/' + GET_FILE_NAME
    exec_cmd('s3cmd --access_key=%s --secret_key=%s --config=%s --no-check-hostname --host=%s get s3://%s/%s %s --force'
            % (ACCESS_KEY, SECRET_KEY, s3cmd_config_path, endpoint, BUCKET_NAME, FILE_NAME, get_file_path))

    # get info of object
    out = exec_cmd('radosgw-admin object stat --bucket=%s --object=%s' % (BUCKET_NAME, FILE_NAME))

    json_op = json.loads(out)
    cached_object_name = json_op['manifest']['prefix']
    log.debug("Cached object name is: %s", cached_object_name)

    # check that the cache is enabled (does the cache directory empty)
    out = exec_cmd('find %s -type f | wc -l' % (cache_dir))
    chk_cache_dir = int(get_cmd_output(out))
    log.debug("Check cache dir content: %s", chk_cache_dir)
    if chk_cache_dir == 0:
        log.info("NOTICE: datacache test object not found, inspect if datacache was bypassed or disabled during this check.")
        return

    # list the files in the cache dir for troubleshooting
    out = exec_cmd('ls -l %s' % (cache_dir))
    # get name of cached object and check if it exists in the cache
    out = exec_cmd('find %s -name "*%s1"' % (cache_dir, cached_object_name))
    cached_object_path = get_cmd_output(out)
    log.debug("Path of file in datacache is: %s", cached_object_path)
    out = exec_cmd('basename %s' % (cached_object_path))
    basename_cmd_out = get_cmd_output(out)
    log.debug("Name of file in datacache is: %s", basename_cmd_out)

    # check to see if the cached object is in Ceph
    out = exec_cmd('rados ls -p default.rgw.buckets.data')
    rados_ls_out = get_cmd_output(out)
    log.debug("rados ls output is: %s", rados_ls_out)

    assert(basename_cmd_out in rados_ls_out)
    log.debug("RGW Datacache test SUCCESS")

    # remove datacache dir
    #cmd = exec_cmd('rm -rf %s' % (cache_dir))
    #log.debug("RGW Datacache dir deleted")
    #^ commenting for future reference - the work unit will continue running tests and if the cache_dir is removed
    #  all the writes to cache will fail with errno 2 ENOENT No such file or directory.

main()
log.info("Completed Datacache tests")