mirror of
https://github.com/ceph/ceph
synced 2024-12-27 14:03:25 +00:00
rgw/examples: adding examples for boto3 extensions to AWS S3
Signed-off-by: Yuval Lifshitz <yuvalif@yahoo.com>
This commit is contained in:
Yuval Lifshitz
parent
3ec16fee44
commit
fc5626d5cf
14
examples/boto3/README.md
Normal file
14
examples/boto3/README.md
Normal file
@ -0,0 +1,14 @@
|
||||
# Introduction
|
||||
This directory contains examples on how to use boto3 to execrsize thew RadosGW extensions to the S3 API supported by AWS.
|
||||
|
||||
# Users
|
||||
For the standard boto3 client to support these extensions, the: ``service-2.sdk-extras.json`` file should be placed under: ``~/.aws/models/s3/2006-03-01/`` directory.
|
||||
For more information see [here](https://github.com/boto/botocore/blob/develop/botocore/loaders.py#L33).
|
||||
|
||||
# Developers
|
||||
Anyone developing an extension to the S3 API supported by AWS, please modify ``service-2.sdk-extras.json`` (all extensions should go into the same file), so that boto3 could be used to test the new API.
|
||||
In addition, python files with code samples should be added to this directory demonstrating use of the new API.
|
||||
When testing you changes please:
|
||||
- make sure that the modified file is in the boto3 path as explained above
|
||||
- make sure that the standard S3 tests suit is not broken, even with the extensions files in the path
|
||||
|
||||
36
examples/boto3/delete_notification.py
Executable file
36
examples/boto3/delete_notification.py
Executable file
@ -0,0 +1,36 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import boto3
|
||||
import sys
|
||||
|
||||
if len(sys.argv) == 3:
|
||||
# bucket name as first argument
|
||||
bucketname = sys.argv[1]
|
||||
# notification name as second argument
|
||||
notification_name = sys.argv[2]
|
||||
elif len(sys.argv) == 2:
|
||||
# bucket name as first argument
|
||||
bucketname = sys.argv[1]
|
||||
notification_name = ""
|
||||
else:
|
||||
print 'Usage: ' + sys.argv[0] + ' <bucket> [notification]'
|
||||
sys.exit(1)
|
||||
|
||||
# endpoint and keys from vstart
|
||||
endpoint = 'http://127.0.0.1:8000'
|
||||
access_key='0555b35654ad1656d804'
|
||||
secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
|
||||
|
||||
client = boto3.client('s3',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key)
|
||||
|
||||
# deleting a specific notification congifuration from a bucket (when NotificationId is provided) or
|
||||
# deleting all notification configurations on a bucket (without deleting the bucket itself) are extension to AWS S3 API
|
||||
|
||||
if notification_name == "":
|
||||
print client.delete_bucket_notification_configuration(Bucket=bucketname)
|
||||
else:
|
||||
print client.delete_bucket_notification_configuration(Bucket=bucketname,
|
||||
Notification=notification_name)
|
||||
28
examples/boto3/get_notification.py
Executable file
28
examples/boto3/get_notification.py
Executable file
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import boto3
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print 'Usage: ' + sys.argv[0] + ' <bucket> <notification>'
|
||||
sys.exit(1)
|
||||
|
||||
# bucket name as first argument
|
||||
bucketname = sys.argv[1]
|
||||
# notification name as second argument
|
||||
notification_name = sys.argv[2]
|
||||
|
||||
# endpoint and keys from vstart
|
||||
endpoint = 'http://127.0.0.1:8000'
|
||||
access_key='0555b35654ad1656d804'
|
||||
secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
|
||||
|
||||
client = boto3.client('s3',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key)
|
||||
|
||||
# getting a specific notification configuration is an extension to AWS S3 API
|
||||
|
||||
print client.get_bucket_notification_configuration(Bucket=bucketname,
|
||||
Notification=notification_name)
|
||||
25
examples/boto3/list_unordered.py
Executable file
25
examples/boto3/list_unordered.py
Executable file
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import boto3
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print 'Usage: ' + sys.argv[0] + ' <bucket>'
|
||||
sys.exit(1)
|
||||
|
||||
# bucket name as first argument
|
||||
bucketname = sys.argv[1]
|
||||
|
||||
# endpoint and keys from vstart
|
||||
endpoint = 'http://127.0.0.1:8000'
|
||||
access_key='0555b35654ad1656d804'
|
||||
secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
|
||||
|
||||
client = boto3.client('s3',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key)
|
||||
|
||||
# geting an unordered list of objets is an extension to AWS S3 API
|
||||
|
||||
print client.list_objects(Bucket=bucketname, AllowUnordered=True)
|
||||
44
examples/boto3/notification_filters.py
Executable file
44
examples/boto3/notification_filters.py
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import boto3
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 4:
|
||||
print 'Usage: ' + sys.argv[0] + ' <bucket> <topic ARN> <notification Id>'
|
||||
sys.exit(1)
|
||||
|
||||
# bucket name as first argument
|
||||
bucketname = sys.argv[1]
|
||||
# topic ARN as second argument
|
||||
topic_arn = sys.argv[2]
|
||||
# notification id as third argument
|
||||
notification_id = sys.argv[3]
|
||||
|
||||
# endpoint and keys from vstart
|
||||
endpoint = 'http://127.0.0.1:8000'
|
||||
access_key='0555b35654ad1656d804'
|
||||
secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
|
||||
|
||||
client = boto3.client('s3',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key)
|
||||
|
||||
# regex filter on the object name and metadata based filtering are extension to AWS S3 API
|
||||
# bucket and topic should be created beforehand
|
||||
|
||||
topic_conf_list = [{'Id': notification_id,
|
||||
'TopicArn': topic_arn,
|
||||
'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
|
||||
'Filter': {
|
||||
'Metadata': {
|
||||
'FilterRules': [{'Name': 'x-amz-meta-foo', 'Value': 'bar'},
|
||||
{'Name': 'x-amz-meta-hello', 'Value': 'world'}]
|
||||
},
|
||||
'Key': {
|
||||
'FilterRules': [{'Name': 'regex', 'Value': '([a-z]+)'}]
|
||||
}
|
||||
}}]
|
||||
|
||||
print client.put_bucket_notification_configuration(Bucket=bucketname,
|
||||
NotificationConfiguration={'TopicConfigurations': topic_conf_list})
|
||||
128
examples/boto3/service-2.sdk-extras.json
Normal file
128
examples/boto3/service-2.sdk-extras.json
Normal file
@ -0,0 +1,128 @@
|
||||
{
|
||||
"version": 1.0,
|
||||
"merge": {
|
||||
"operations":{
|
||||
"DeleteBucketNotificationConfiguration":{
|
||||
"name":"DeleteBucketNotificationConfiguration",
|
||||
"http":{
|
||||
"method":"DELETE",
|
||||
"requestUri":"/{Bucket}?notification",
|
||||
"responseCode":204
|
||||
},
|
||||
"input":{"shape":"DeleteBucketNotificationConfigurationRequest"},
|
||||
"documentationUrl":"https://docs.ceph.com/docs/master/radosgw/s3/bucketops/#delete-notification",
|
||||
"documentation":"<p>Deletes the notification configuration from the bucket.</p>"
|
||||
}
|
||||
},
|
||||
"shapes": {
|
||||
"ListObjectsRequest": {
|
||||
"members": {
|
||||
"AllowUnordered": {
|
||||
"shape":"AllowUnordered",
|
||||
"documentation":"<p>Allow the listing results to be returned in unsorted order. This may be faster when listing very large buckets.</p>",
|
||||
"location":"querystring",
|
||||
"locationName":"allow-unordered"
|
||||
}
|
||||
}
|
||||
},
|
||||
"AllowUnordered":{"type":"boolean"},
|
||||
"GetBucketNotificationConfigurationRequest":{
|
||||
"type":"structure",
|
||||
"required":["Bucket"],
|
||||
"members":{
|
||||
"Bucket":{
|
||||
"shape":"BucketName",
|
||||
"documentation":"<p>Name of the bucket to get the notifications configuration for.</p>",
|
||||
"location":"uri",
|
||||
"locationName":"Bucket"
|
||||
},
|
||||
"Notification":{
|
||||
"shape":"NotificationId",
|
||||
"documentation":"<p>Id of the specific notification on the bucket for which the configuration should be retrieved.</p>",
|
||||
"location":"querystring",
|
||||
"locationName":"notification-id"
|
||||
}
|
||||
}
|
||||
},
|
||||
"DeleteBucketNotificationConfigurationRequest":{
|
||||
"type":"structure",
|
||||
"required":["Bucket"],
|
||||
"members":{
|
||||
"Bucket":{
|
||||
"shape":"BucketName",
|
||||
"documentation":"<p>Name of the bucket to delete the notifications configuration from.</p>",
|
||||
"location":"uri",
|
||||
"locationName":"Bucket"
|
||||
},
|
||||
"Notification":{
|
||||
"shape":"NotificationId",
|
||||
"documentation":"<p>Id of the specific notification on the bucket to be deleted.</p>",
|
||||
"location":"querystring",
|
||||
"locationName":"notification-id"
|
||||
}
|
||||
}
|
||||
},
|
||||
"FilterRule":{
|
||||
"type":"structure",
|
||||
"members":{
|
||||
"Name":{
|
||||
"shape":"FilterRuleName",
|
||||
"documentation":"<p>The object key name prefix, suffix or regex identifying one or more objects to which the filtering rule applies. The maximum length is 1,024 characters. Overlapping prefixes and suffixes are supported.</p>"
|
||||
},
|
||||
"Value":{
|
||||
"shape":"FilterRuleValue",
|
||||
"documentation":"<p>The value that the filter searches for in object key names.</p>"
|
||||
}
|
||||
},
|
||||
"documentation":"<p>Specifies the Amazon S3 object key name to filter on and whether to filter on the suffix, prefix or regex of the key name.</p>"
|
||||
},
|
||||
"FilterRuleName":{
|
||||
"type":"string",
|
||||
"enum":[
|
||||
"prefix",
|
||||
"suffix",
|
||||
"regex"
|
||||
]
|
||||
},
|
||||
"NotificationConfigurationFilter":{
|
||||
"type":"structure",
|
||||
"members":{
|
||||
"Key":{
|
||||
"shape":"S3KeyFilter",
|
||||
"documentation":"<p/>",
|
||||
"locationName":"S3Key"
|
||||
},
|
||||
"Metadata":{
|
||||
"shape":"S3MetadataFilter",
|
||||
"documentation":"<p/>",
|
||||
"locationName":"S3Metadata"
|
||||
}
|
||||
},
|
||||
"documentation":"<p>Specifies object key name filtering rules. For information about key name filtering, see <a href=\"https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html\">Configuring Event Notifications</a> in the <i>Amazon Simple Storage Service Developer Guide</i>.</p>"
|
||||
},
|
||||
"S3KeyFilter":{
|
||||
"type":"structure",
|
||||
"members":{
|
||||
"FilterRules":{
|
||||
"shape":"FilterRuleList",
|
||||
"documentation":"<p/>",
|
||||
"locationName":"FilterRule"
|
||||
}
|
||||
},
|
||||
"documentation":"<p>A container for object key name prefix, suffix and regex filtering rules.</p>"
|
||||
},
|
||||
"S3MetadataFilter":{
|
||||
"type":"structure",
|
||||
"members":{
|
||||
"FilterRules":{
|
||||
"shape":"FilterRuleList",
|
||||
"documentation":"<p/>",
|
||||
"locationName":"FilterRule"
|
||||
}
|
||||
},
|
||||
"documentation":"<p>A container for metadata filtering rules.</p>"
|
||||
}
|
||||
},
|
||||
"documentation":"<p/>"
|
||||
}
|
||||
}
|
||||
41
examples/boto3/topic_with_endpoint.py
Executable file
41
examples/boto3/topic_with_endpoint.py
Executable file
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import boto3
|
||||
import sys
|
||||
import urlparse
|
||||
from botocore.client import Config
|
||||
|
||||
if len(sys.argv) == 3:
|
||||
# topic name as first argument
|
||||
topic_name = sys.argv[1]
|
||||
# region name as second argument
|
||||
region_name = sys.argv[2]
|
||||
elif len(sys.argv) == 2:
|
||||
# topic name as first argument
|
||||
topic_name = sys.argv[1]
|
||||
region_name = ""
|
||||
else:
|
||||
print 'Usage: ' + sys.argv[0] + ' <topic name> [region name]'
|
||||
sys.exit(1)
|
||||
|
||||
# endpoint and keys from vstart
|
||||
endpoint = 'http://127.0.0.1:8000'
|
||||
access_key='0555b35654ad1656d804'
|
||||
secret_key='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=='
|
||||
|
||||
client = boto3.client('sns',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
region_name=region_name,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(signature_version='s3'))
|
||||
|
||||
# to see the list of available "regions" use:
|
||||
# radosgw-admin realm zonegroup list
|
||||
|
||||
# this is standard AWS services call, using custom attributes to add AMQP endpoint information to the topic
|
||||
|
||||
endpoint_args = 'push-endpoint=amqp://127.0.0.1:5672&amqp-exchange=ex1&amqp-ack-level=broker'
|
||||
attributes = {nvp[0] : nvp[1] for nvp in urlparse.parse_qsl(endpoint_args, keep_blank_values=True)}
|
||||
|
||||
print client.create_topic(Name=topic_name, Attributes=attributes)
|
||||
Loading…
Reference in New Issue
Block a user