From 523c623b282ce5cf4e4bb7bcf2b35c4009d89b3a Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Wed, 8 Apr 2020 10:50:21 +0800 Subject: [PATCH] test/rgw: upload using a NamedTemporaryFile in boto, it tries to figure out the MIME type of a file by its name, if the file-like objects has an attribute of "name". in Python2, the "name" is always "", fortunately. while in Python3, `TemporaryFile` also have a "name" which is its fd, and it is an integer now. so we have following error when sending a `TemporaryFile` using `upload_part_from_file()`: ``` 2020-04-08T02:25:34.660 INFO:tasks.rgw_multisite_tests:Traceback (most recent call last): 2020-04-08T02:25:34.661 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/git.ceph.com_git_teuthology_wip-py3/virtualenv/lib/python3.5/site-packages/nose/case.py", line 198, in runTest 2020-04-08T02:25:34.661 INFO:tasks.rgw_multisite_tests: self.test(*self.arg) 2020-04-08T02:25:34.662 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/github.com_tchaikov_ceph_wip-qa-py3/qa/tasks/rgw_multi/tests_ps.py", line 2567, in test_ps_creation_triggers 2020-04-08T02:25:34.662 INFO:tasks.rgw_multisite_tests: uploader.upload_part_from_file(fp, 1) 2020-04-08T02:25:34.663 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/git.ceph.com_git_teuthology_wip-py3/virtualenv/lib/python3.5/site-packages/boto/s3/multipart.py", line 260, in upload_part_from_file 2020-04-08T02:25:34.663 INFO:tasks.rgw_multisite_tests: query_args=query_args, size=size) 2020-04-08T02:25:34.664 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/git.ceph.com_git_teuthology_wip-py3/virtualenv/lib/python3.5/site-packages/boto/s3/key.py", line 1293, in set_contents_from_file 2020-04-08T02:25:34.664 INFO:tasks.rgw_multisite_tests: chunked_transfer=chunked_transfer, size=size) 2020-04-08T02:25:34.664 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/git.ceph.com_git_teuthology_wip-py3/virtualenv/lib/python3.5/site-packages/boto/s3/key.py", line 750, in send_file 2020-04-08T02:25:34.665 INFO:tasks.rgw_multisite_tests: chunked_transfer=chunked_transfer, size=size) 2020-04-08T02:25:34.665 INFO:tasks.rgw_multisite_tests: File "/home/teuthworker/src/git.ceph.com_git_teuthology_wip-py3/virtualenv/lib/python3.5/site-packages/boto/s3/key.py", line 920, in _send_file_internal 2020-04-08T02:25:34.666 INFO:tasks.rgw_multisite_tests: self.content_type = mimetypes.guess_type(self.path)[0] 2020-04-08T02:25:34.666 INFO:tasks.rgw_multisite_tests: File "/usr/lib/python3.5/mimetypes.py", line 289, in guess_type 2020-04-08T02:25:34.667 INFO:tasks.rgw_multisite_tests: return _db.guess_type(url, strict) 2020-04-08T02:25:34.667 INFO:tasks.rgw_multisite_tests: File "/usr/lib/python3.5/mimetypes.py", line 114, in guess_type 2020-04-08T02:25:34.667 INFO:tasks.rgw_multisite_tests: scheme, url = urllib.parse.splittype(url) 2020-04-08T02:25:34.668 INFO:tasks.rgw_multisite_tests: File "/usr/lib/python3.5/urllib/parse.py", line 881, in splittype 2020-04-08T02:25:34.668 INFO:tasks.rgw_multisite_tests: match = _typeprog.match(url) 2020-04-08T02:25:34.669 INFO:tasks.rgw_multisite_tests:TypeError: expected string or bytes-like object ``` to address this issue, in this change, a `NamedTemporaryFile` is used instead of `TemporaryFile`. the former does have a "name" which is a `str`. Signed-off-by: Kefu Chai --- src/test/rgw/rgw_multi/tests_ps.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/rgw/rgw_multi/tests_ps.py b/src/test/rgw/rgw_multi/tests_ps.py index e4b218e1a1e..b3aead9c38c 100644 --- a/src/test/rgw/rgw_multi/tests_ps.py +++ b/src/test/rgw/rgw_multi/tests_ps.py @@ -2563,7 +2563,7 @@ def test_ps_creation_triggers(): fp.write('bar') fp.close() uploader = bucket.initiate_multipart_upload('multipart') - fp = tempfile.TemporaryFile(mode='r') + fp = tempfile.NamedTemporaryFile(mode='r') uploader.upload_part_from_file(fp, 1) uploader.complete_upload() fp.close() @@ -2634,7 +2634,7 @@ def test_ps_s3_creation_triggers_on_master(): fp.write('bar') fp.close() uploader = bucket.initiate_multipart_upload('multipart') - fp = tempfile.TemporaryFile(mode='r') + fp = tempfile.NamedTemporaryFile(mode='r') uploader.upload_part_from_file(fp, 1) uploader.complete_upload() fp.close() @@ -2709,7 +2709,7 @@ def test_ps_s3_multipart_on_master(): assert_equal(status/100, 2) # create objects in the bucket using multi-part upload - fp = tempfile.TemporaryFile(mode='w+b') + fp = tempfile.NamedTemporaryFile(mode='w+b') content = bytearray(os.urandom(1024*1024)) fp.write(content) fp.flush() @@ -2903,7 +2903,7 @@ def test_ps_s3_metadata_on_master(): # create objects in the bucket using COPY bucket.copy_key('copy_of_foo', bucket.name, key.name) # create objects in the bucket using multi-part upload - fp = tempfile.TemporaryFile(mode='w') + fp = tempfile.NamedTemporaryFile(mode='w') fp.write('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') fp.close() uploader = bucket.initiate_multipart_upload('multipart_foo',