diff --git a/examples/lua/config/prometheus.yml b/examples/lua/config/prometheus.yml new file mode 100644 index 00000000000..37deee67cfc --- /dev/null +++ b/examples/lua/config/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 2s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'rgw' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 1s + + static_configs: + - targets: ['127.0.0.1:9091'] \ No newline at end of file diff --git a/examples/lua/elasticsearch_adapter.lua b/examples/lua/elasticsearch_adapter.lua new file mode 100644 index 00000000000..a0c542fed88 --- /dev/null +++ b/examples/lua/elasticsearch_adapter.lua @@ -0,0 +1,114 @@ +local elasticsearch = require ("elasticsearch") +local json = require ("lunajson") + +local client = elasticsearch.client{ + hosts = { + { + host = "localhost", + port = "9200" + } + } +} + +local copyfrom = {} +if (Request.CopyFrom ~= nil) then + copyfrom = { + Tenant = Request.CopyFrom.Tenant, + Bucket = Request.CopyFrom.Bucket, + Object = { + Name = Request.CopyFrom.Object.Name, + Instance = Request.CopyFrom.Object.Instance, + Id = Request.CopyFrom.Object.Id, + Size = Request.CopyFrom.Object.Size, + MTime = Request.CopyFrom.Object.MTime + } + } +end + +local res, status = client:index{ + index = "rgw", + type = "Request", + id = Request.Id, + body = + { + RGWOp = Request.RGWOp, + DecodedURI = Request.DecodedURI, + ContentLength = Request.ContentLength, + GenericAttributes = json.encode(Request.GenericAttributes), + Response = { + HTTPStatusCode = Request.Response.HTTPStatusCode, + HTTPStatus = Request.Response.HTTPStatus, + RGWCode = Request.Response.RGWCode, + Message = Request.Response.Message + }, + SwiftAccountName = Request.SwiftAccountName, + Bucket = { + Tenant = Request.Bucket.Tenant, + Name = Request.Bucket.Name, + Marker = Request.Bucket.Marker, + Id = Request.Bucket.Id, + Count = Request.Bucket.Count, + Size = Request.Bucket.Size, + ZoneGroupId = Request.Bucket.ZoneGroupId, + CreationTime = Request.Bucket.CreationTime, + MTime = Request.Bucket.MTime, + Quota = { + MaxSize = Request.Bucket.Quota.MaxSize, + MaxObjects = Request.Bucket.Quota.MaxObjects, + Enabled = Request.Bucket.Quota.Enabled, + Rounded = Request.Bucket.Quota.Rounded + }, + PlacementRule = { + Name = Request.Bucket.PlacementRule.Name, + StorageClass = Request.Bucket.PlacementRule.StorageClass + }, + User = { + Tenant = Request.Bucket.User.Tenant, + Id = Request.Bucket.User.Id + } + }, + Object = { + Name = Request.Object.Name, + Instance = Request.Object.Instance, + Id = Request.Object.Id, + Size = Request.Object.Size, + MTime = Request.Object.MTime + }, + CopyFrom = copyfrom, + ObjectOwner = { + DisplayName = Request.ObjectOwner.DisplayName, + User = { + Tenant = Request.ObjectOwner.User.Tenant, + Id = Request.ObjectOwner.User.Id + } + }, + ZoneGroup = { + Name = Request.ZoneGroup.Name, + Endpoint = Request.ZoneGroup.Endpoint + }, + Environment = json.encode(Request.Environment), + Policy = json.encode(Request.Policy), + UserPolicies = json.encode(Request.UserPolicies), + RGWId = Request.RGWId, + HTTP = { + Parameters = json.encode(Request.HTTP.Parameters), + Resources = json.encode(Request.HTTP.Resources), + Metadata = json.encode(Request.HTTP.Metadata), + Host = Request.HTTP.Host, + Method = Request.HTTP.Method, + URI = Request.HTTP.URI, + QueryString = Request.HTTP.QueryString, + Domain = Request.HTTP.Domain + }, + Time = Request.Time, + Dialect = Request.Dialect, + Id = Request.Id, + TransactionId = Request.TransactionId, + Tags = json.encode(Request.Tags), + User = { + Tenant = Request.User.Tenant, + Id = Request.User.Id + } + } +} + diff --git a/examples/lua/elasticsearch_adapter.md b/examples/lua/elasticsearch_adapter.md new file mode 100644 index 00000000000..a32b5d36f64 --- /dev/null +++ b/examples/lua/elasticsearch_adapter.md @@ -0,0 +1,59 @@ +# Introduction + +This directory contains an example `elasticsearch_adapter.lua` on how to +use [Lua Scripting](https://docs.ceph.com/en/latest/radosgw/lua-scripting/) +to push fields of the RGW requests +to [Elasticsearch](https://www.elastic.co/elasticsearch/). + +## Elasticsearch + +Install and run Elasticsearch using docker: +```bash +docker network create elastic +docker pull elasticsearch:2.4.6 +docker run --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:2.4.6 +``` + +[Full documentation for Elasticsearch installation](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html) + +## Usage + +* Upload the script: + +```bash +radosgw-admin script put --infile=elasticsearch_adapter.lua --context=postRequest +``` + +* Add the packages used in the script: + +```bash +radosgw-admin script-package add --package='elasticsearch 1.0.0-1' --allow-compilation +radosgw-admin script-package add --package='lunajson' --allow-compilation +radosgw-admin script-package add --package='lua-cjson 2.1.0-1' --allow-compilation +``` + +* Restart radosgw. + +* Send a request: +```bash +s3cmd --host=localhost:8000 --host-bucket="localhost:8000/%(bucket)" --access_key=0555b35654ad1656d804 --secret_key=h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== mb s3://mybucket +s3cmd --host=localhost:8000 --host-bucket="localhost:8000/%(bucket)" --access_key=0555b35654ad1656d804 --secret_key=h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== put -P /etc/hosts s3://mybucket +curl http://localhost:8000/mybucket/hosts +``` + +* Search by bucket id from Elasticsearch: +```bash +curl -X GET "localhost:9200/rgw/_search?pretty" -H 'Content-Type: application/json' -d' +{ + "query": { + "match": { + "Bucket.Id": "05382336-b2db-409f-82dc-f28ab5fef978.4471.4471" + } + } +} +' +``` + +## Requirements +* Lua 5.3 + diff --git a/examples/lua/img/prometheus.png b/examples/lua/img/prometheus.png new file mode 100644 index 00000000000..7a3b63f647a Binary files /dev/null and b/examples/lua/img/prometheus.png differ diff --git a/examples/lua/prometheus_adapter.lua b/examples/lua/prometheus_adapter.lua new file mode 100644 index 00000000000..4f0af9a3b91 --- /dev/null +++ b/examples/lua/prometheus_adapter.lua @@ -0,0 +1,23 @@ +local http = require("socket.http") +local ltn12 = require("ltn12") + +local respbody = {} +local op = "rgw_other_request_content_length" +if (Request.RGWOp == "put_obj") then + op = "rgw_put_request_content_length" +elseif (Request.RGWOp == "get_obj") then + op = "rgw_get_request_content_length" +end +local field = op .. " " .. tostring(Request.ContentLength) .. "\n" + +local body, code, headers, status = http.request{ + url = "http://127.0.0.1:9091/metrics/job/rgw", + method = "POST", + headers = { + ["Content-Type"] = "application/x-www-form-urlencoded", + ["Content-Length"] = string.len(field) + }, + source = ltn12.source.string(field), + sink = ltn12.sink.table(respbody), +} + diff --git a/examples/lua/prometheus_adapter.md b/examples/lua/prometheus_adapter.md new file mode 100644 index 00000000000..eae1d81515e --- /dev/null +++ b/examples/lua/prometheus_adapter.md @@ -0,0 +1,59 @@ +# Introduction + +This directory contains an example `prometheus_adapter.lua` on how to +use [Lua Scripting](https://docs.ceph.com/en/latest/radosgw/lua-scripting/) +to push metrics from the RGW requests to [Prometheus](https://prometheus.io/), +specifically to collect information on object sizes. + +## Prometheus + +As every single run of a lua script is short-lived, +so [Pushgateway](https://github.com/prometheus/pushgateway) +should be used as an intermediate service to enable Prometheus to scrape data +from RGW. + +* Install and run Pushgateway using docker: + +```bash +docker pull prom/pushgateway +docker run -p 9091:9091 -it prom/pushgateway +``` + +* Install and run Prometheus using docker: + +```bash +docker pull prom/prometheus +docker run --network host -v ${CEPH_DIR}/examples/lua/config/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus +``` + +[Full documentation for Prometheus installation](https://prometheus.io/docs/prometheus/latest/installation/) + +## Usage + +* Upload the script: + +```bash +radosgw-admin script put --infile=prometheus_adapter.lua --context=postRequest +``` + +* Add the packages used in the script: + +```bash +radosgw-admin script-package add --package='luasocket' --allow-compilation +``` + +* Restart radosgw. + +* Send a request: +```bash +s3cmd --host=localhost:8000 --host-bucket="localhost:8000/%(bucket)" --access_key=0555b35654ad1656d804 --secret_key=h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== mb s3://mybucket +s3cmd --host=localhost:8000 --host-bucket="localhost:8000/%(bucket)" --access_key=0555b35654ad1656d804 --secret_key=h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q== put -P /etc/hosts s3://mybucket +curl http://localhost:8000/mybucket/hosts +``` + +* Open `http://localhost:9090` by browser and search for `rgw_request_content_length` +![](img/prometheus.png) + +## Requirements +* Lua 5.3 or higher +