rgw/kms/vault - s3tests for both old and new test logic.

Test both "old" and "new" transit logic with s3tests.  Does not test
migration - that will need to be done separately.  I've added
a "flavor" parameter so the test logic can tell the difference
between the "old" engine and the "new" engine.  The vault
keys creation logic now has options to determine whether
the keys created are exportable (needed for the old transit
engine), or not (should be the case going forward with the
new transit engine.)

Fixes: http://tracker.ceph.com/issues/48746
Signed-off-by: Marcus Watts <mwatts@redhat.com>

qa/suites/rgw/crypt/2-kms/vault_old.yaml

@@ -0,0 +1,24 @@
+overrides:
+  ceph:
+    conf:
+      client:
+        rgw crypt s3 kms backend: vault
+        rgw crypt vault auth: token
+        rgw crypt vault secret engine: transit
+        rgw crypt vault prefix: /v1/transit/export/encryption-key/
+  rgw:
+    client.0:
+      use-vault-role: client.0
+
+tasks:
+- vault:
+    client.0:
+      install_url: https://releases.hashicorp.com/vault/1.2.2/vault_1.2.2_linux_amd64.zip
+      install_sha256: 7725b35d9ca8be3668abe63481f0731ca4730509419b4eb29fa0b0baa4798458
+      root_token: test_root_token
+      engine: transit
+      flavor: old
+      prefix: /v1/transit/keys/
+      secrets:
+        - path: my-key-1
+        - path: my-key-2

qa/suites/rgw/crypt/2-kms/vault_transit.yaml

@@ -5,7 +5,7 @@ overrides:
         rgw crypt s3 kms backend: vault
         rgw crypt vault auth: token
         rgw crypt vault secret engine: transit
-        rgw crypt vault prefix: /v1/transit/export/encryption-key/
+        rgw crypt vault prefix: /v1/transit/
   rgw:
     client.0:
       use-vault-role: client.0

qa/suites/rgw/crypt/4-tests/s3tests.yaml

@@ -8,6 +8,9 @@ tasks:
       vault_kv:
         key_path: my-key-1
         key_path2: my-key-2
-      vault_transit:
+      vault_old:
         key_path: my-key-1/1
         key_path2: my-key-2/1
+      vault_transit:
+        key_path: my-key-1
+        key_path2: my-key-2

qa/tasks/rgw.py

@@ -133,6 +133,7 @@ def start_rgw(ctx, config, clients):
             if not ctx.vault.root_token:
                 raise ConfigError('vault: no "root_token" specified')
             # create token on file
+            ctx.rgw.vault_role = vault_role
             ctx.cluster.only(client).run(args=['echo', '-n', ctx.vault.root_token, run.Raw('>'), token_path])
             log.info("Token file content")
             ctx.cluster.only(client).run(args=['cat', token_path])

qa/tasks/s3tests.py

@@ -324,9 +324,18 @@ def configure(ctx, config):
                 s3tests_conf['DEFAULT']['kms_keyid2'] = key['id']
 
         elif hasattr(ctx, 'vault'):
-            properties = properties['vault_%s' % ctx.vault.engine]
-            s3tests_conf['DEFAULT']['kms_keyid'] = properties['key_path']
-            s3tests_conf['DEFAULT']['kms_keyid2'] = properties['key_path2']
+            engine_or_flavor = vars(ctx.vault).get('flavor',ctx.vault.engine)
+            keys=[]
+            for name in (x['Path'] for x in vars(ctx.vault).get('keys', {}).get(ctx.rgw.vault_role)):
+                keys.append(name)
+
+            keys.extend(['testkey-1','testkey-2'])
+            if engine_or_flavor == "old":
+                keys=[keys[i] + "/1" for i in range(len(keys))]
+
+            properties = properties.get('vault_%s' % engine_or_flavor, {})
+            s3tests_conf['DEFAULT']['kms_keyid'] = properties.get('key_path', keys[0])
+            s3tests_conf['DEFAULT']['kms_keyid2'] = properties.get('key_path2', keys[1])
         elif hasattr(ctx.rgw, 'pykmip_role'):
             keys=[]
             for name in (x['Name'] for x in ctx.pykmip.keys[ctx.rgw.pykmip_role]):

qa/tasks/vault.py

@@ -185,14 +185,17 @@ def create_secrets(ctx, config):
     engine = cconfig.get('engine')
     prefix = cconfig.get('prefix')
     secrets = cconfig.get('secrets')
+    flavor = cconfig.get('flavor')
     if secrets is None:
         raise ConfigError("No secrets specified, please specify some.")
 
+    ctx.vault.keys[cclient] = []
     for secret in secrets:
         try:
             path = secret['path']
         except KeyError:
             raise ConfigError('Missing "path" field in secret')
+        exportable = secret.get("exportable", flavor == "old")
 
         if engine == 'kv':
             try:
@@ -204,12 +207,14 @@ def create_secrets(ctx, config):
             except KeyError:
                 raise ConfigError('Missing "secret" field in secret')
         elif engine == 'transit':
-            data = {"exportable": "true"}
+            data = {"exportable": "true" if exportable else "false"}
         else:
             raise Exception("Unknown or missing secrets engine")
 
         send_req(ctx, cconfig, cclient, urljoin(prefix, path), json.dumps(data))
 
+        ctx.vault.keys[cclient].append({ 'Path': path });
+
     log.info("secrets created")
     yield
 
@@ -224,15 +229,28 @@ def task(ctx, config):
     tasks:
     - vault:
         client.0:
-          version: 1.2.2
+          install_url: http://my.special.place/vault.zip
+          install_sha256: zipfiles-sha256-sum-much-larger-than-this
           root_token: test_root_token
-          engine: kv
-          prefix: /v1/kv/data/
+          engine: transit
+          flavor: old
+          prefix: /v1/transit/keys
           secrets:
             - path: kv/teuthology/key_a
-              secret: YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo=
+              secret: base64_only_if_using_kv_aWxkCmNlcGguY29uZgo=
+              exportable: true
             - path: kv/teuthology/key_b
-              secret: aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo=
+              secret: base64_only_if_using_kv_dApzcmMKVGVzdGluZwo=
+
+    engine can be 'kv' or 'transit'
+    prefix should be /v1/kv/data/ for kv, /v1/transit/keys/ for transit
+    flavor should be 'old' only if testing the original transit logic
+        otherwise omit.
+    for kv only: 256-bit key value should be specified via secret,
+        otherwise should omit.
+    for transit: exportable may be used to make individual keys exportable.
+    flavor may be set to 'old' to make all keys exportable by default,
+        which is required by the original transit logic.
     """
     all_clients = ['client.{id}'.format(id=id_)
                    for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')]
@@ -255,6 +273,10 @@ def task(ctx, config):
     ctx.vault.root_token = None
     ctx.vault.prefix = config[client].get('prefix')
     ctx.vault.engine = config[client].get('engine')
+    ctx.vault.keys = {}
+    q=config[client].get('flavor')
+    if q:
+        ctx.vault.flavor = q
 
     with contextutil.nested(
         lambda: download(ctx=ctx, config=config),