diff --git a/doc/dev/developer_guide/jaegertracing.rst b/doc/dev/developer_guide/jaegertracing.rst index d70b72c385b..73a48ad83c2 100644 --- a/doc/dev/developer_guide/jaegertracing.rst +++ b/doc/dev/developer_guide/jaegertracing.rst @@ -16,19 +16,8 @@ it across the distributed system. BASIC ARCHITECTURE AND TERMINOLOGY ---------------------------------- -* TRACE: A trace shows the data/execution path through a system. -* SPAN: A single unit of a trace, it is a data structure that stores - information like operation name, timestamps, ordering in a trace. -* JAEGER CLIENT: language-specific implementations of the OpenTracing API. -* JAEGER AGENT: a daemon that listens for spans sent over User Datagram Protocol. - The agent is meant to be placed on the same host as the instrumented - application. (acts like a sidecar listener) -* JAEGER COLLECTOR: Jaeger agent sends the spans to this daemon which then - stitches the spans together to form a trace(if enabled, also persists a database - for these traces) -* JAEGER QUERY AND CONSOLE FRONTEND: UI based frontend to checkout the jaeger - traces, navigate to http://localhost:16686 (if using default `all-in-one - docker `_ to access the Jaeger UI. +refer to the `Ceph Tracing documentation <../../../jaegertracing/#basic-architecture-and-terminology>`_ + HOW TO GET STARTED USING TRACING? --------------------------------- @@ -71,4 +60,4 @@ steps needed: $ bin/rados -p test bench 5 write --no-cleanup .. seealso:: - `using-jaeger-cpp-client-for-distributed-tracing-in-ceph `_ + `using-jaeger-cpp-client-for-distributed-tracing-in-ceph ` \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 62619726df7..cca7031cb8d 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -109,3 +109,4 @@ about Ceph, see our `Architecture`_ section. releases/index security/index Glossary + Tracing diff --git a/doc/jaegertracing/index.rst b/doc/jaegertracing/index.rst new file mode 100644 index 00000000000..7dcc70cc329 --- /dev/null +++ b/doc/jaegertracing/index.rst @@ -0,0 +1,80 @@ +JAEGER- DISTRIBUTED TRACING +=========================== + +Jaeger provides ready to use tracing services for distributed +systems and is becoming the widely used standard because of their simplicity and +standardization. + + +BASIC ARCHITECTURE AND TERMINOLOGY +---------------------------------- + +* TRACE: A trace shows the data/execution path through a system. +* SPAN: A single unit of a trace, it is a data structure that stores + information like operation name, timestamps, ordering in a trace. +* JAEGER CLIENT: language-specific implementations of the OpenTracing API. +* JAEGER AGENT: a daemon that listens for spans sent over User Datagram Protocol. + The agent is meant to be placed on the same host as the instrumented + application. (acts like a sidecar listener) +* JAEGER COLLECTOR: Jaeger agent sends the spans to this daemon which then + stitches the spans together to form a trace(if enabled, also persists a database + for these traces) +* JAEGER QUERY AND CONSOLE FRONTEND: UI based frontend to checkout the jaeger + traces, navigate to http://:16686 + + +read more about jaeger tracing:. + + https://www.jaegertracing.io/docs/ + + +JAEGER DEPLOYMENT +----------------- + +there are couple of ways to deploy jaeger. +please refer to: + +`jaeger deployment `_ + +`jaeger performance tuning `_ + + +In addition, spans are being sent to local jaeger agent, so the jaeger agent must be running on each host (not in all-in-one mode). +otherwise, spans of hosts without active jaeger agent will be lost. + +HOW TO ENABLE TRACING IN CEPH +----------------------------- + +tracing in Ceph is disabled by default. +it could be enabled globally, or for each entity seperately (e.g. rgw). + + Enable tracing globally:: + + $ ceph config set global jaeger_tracing_enable true + + + Enable tracing for each entity:: + + $ ceph config set jaeger_tracing_enable true + + +TRACES IN RGW +------------- + +traces of RGW can be found under Service `rgw` in Jaeger Frontend. + +every user request is being traced. each trace contains tags for +`Operation name`, `User id`, `Object name` and `Bucket name`. + +there is also `Upload id` tag for Multipart upload operations. + +rgw service in Jaeger Frontend: + +.. image:: ./rgw_jaeger.png + :width: 400 + + +osd service in Jaeger Frontend: + +.. image:: ./osd_jaeger.png + :width: 400 diff --git a/doc/jaegertracing/osd_jaeger.png b/doc/jaegertracing/osd_jaeger.png new file mode 100644 index 00000000000..b881e1623cf Binary files /dev/null and b/doc/jaegertracing/osd_jaeger.png differ diff --git a/doc/jaegertracing/rgw_jaeger.png b/doc/jaegertracing/rgw_jaeger.png new file mode 100644 index 00000000000..9f063022c25 Binary files /dev/null and b/doc/jaegertracing/rgw_jaeger.png differ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1cda11504a7..1e32e4029b8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -418,10 +418,8 @@ set(libcommon_files osdc/error_code.cc librbd/Features.cc librbd/io/IoOperations.cc - ${mds_files}) -if(WITH_JAEGER) - list(APPEND libcommon_files common/tracer.cc) -endif() + ${mds_files} + common/tracer.cc) set_source_files_properties(ceph_ver.c APPEND PROPERTY OBJECT_DEPENDS ${CMAKE_BINARY_DIR}/src/include/ceph_ver.h) add_library(common-objs OBJECT ${libcommon_files}) @@ -678,9 +676,7 @@ set(ceph_osd_srcs osd/objclass.cc objclass/class_api.cc ceph_osd.cc) -if(WITH_JAEGER) - list(APPEND ceph_osd_srcs common/tracer.cc) -endif() + add_executable(ceph-osd ${ceph_osd_srcs}) add_dependencies(ceph-osd erasure_code_plugins) target_link_libraries(ceph-osd osd os global-static common diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index 1d61c800dcd..223bd574500 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -6084,3 +6084,12 @@ options: desc: How long cleaner should sleep before re-checking utilization default: 5 with_legacy: true +- name: jaeger_tracing_enable + type: bool + level: advanced + desc: Ceph should use jaeger tracing system + default: false + services: + - rgw + - osd + with_legacy: true \ No newline at end of file diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index c58d4e837b4..079609acb59 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -804,14 +804,6 @@ options: services: - rgw with_legacy: true -- name: rgw_jaeger_enable - type: bool - level: advanced - desc: should RGW use jaeger tracing system - default: false - services: - - rgw - with_legacy: true - name: rgw_s3_auth_use_keystone type: bool level: advanced diff --git a/src/common/tracer.cc b/src/common/tracer.cc index d3a1afffa91..ee2961ef77b 100644 --- a/src/common/tracer.cc +++ b/src/common/tracer.cc @@ -1,86 +1,60 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab +#include "common/ceph_context.h" +#include "global/global_context.h" #include "tracer.h" -#include -#include -#ifdef __linux__ -#include -#else -typedef int64_t __s64; -#endif -#include "common/debug.h" +#ifdef HAVE_JAEGER -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_osd -#undef dout_prefix -#define dout_prefix *_dout << "jaeger_tracing " +namespace tracing { -namespace jaeger_tracing { +const opentelemetry::nostd::shared_ptr Tracer::noop_tracer = opentelemetry::trace::Provider::GetTracerProvider()->GetTracer("no-op", OPENTELEMETRY_SDK_VERSION); - std::shared_ptr tracer = nullptr; +Tracer::Tracer(opentelemetry::nostd::string_view service_name) { + init(service_name); +} - void init_tracer(const char* tracer_name) { - if (!tracer) { - YAML::Node yaml; - try{ - yaml = YAML::LoadFile("../src/jaegertracing/config.yml"); - dout(3) << "yaml loaded" << yaml << dendl; - } - catch(std::exception &e) { - dout(3) << "failed to load yaml file using default config" << dendl; - auto yaml_config = R"cfg( -disabled: false -reporter: - logSpans: false - queueSize: 100 - bufferFlushInterval: 10 -sampler: - type: const - param: 1 -headers: - jaegerDebugHeader: debug-id - jaegerBaggageHeader: baggage - TraceContextHeaderName: trace-id -baggage_restrictions: - denyBaggageOnInitializationFailure: false - refreshInterval: 60 -)cfg"; - yaml = YAML::Load(yaml_config); - dout(3) << "yaml loaded" << yaml << dendl; - } - static auto configuration = jaegertracing::Config::parse(yaml); - tracer = jaegertracing::Tracer::make( tracer_name, configuration, - jaegertracing::logging::consoleLogger()); - dout(3) << "tracer_jaeger init successful" << dendl; - } - //incase of stale tracer, configure with a new global tracer - if (opentracing::Tracer::Global() != tracer) { - opentracing::Tracer::InitGlobal( - std::static_pointer_cast(tracer)); - } - } - - jspan new_span(const char* span_name) { - return opentracing::Tracer::Global()->StartSpan(span_name); - } - - jspan child_span(const char* span_name, const jspan& parent_span) { - //no parent check if parent not found span will still be constructed - return opentracing::Tracer::Global()->StartSpan(span_name, - {opentracing::ChildOf(&parent_span->context())}); - } - - void finish_span(const jspan& span) { - if (span) { - span->Finish(); - } - } - - void set_span_tag(const jspan& span, const char* key, const char* value) { - if (span) { - span->SetTag(key, value); - } +void Tracer::init(opentelemetry::nostd::string_view service_name) { + if (!tracer) { + const opentelemetry::exporter::jaeger::JaegerExporterOptions opts; + auto jaeger_exporter = std::unique_ptr(new opentelemetry::exporter::jaeger::JaegerExporter(opts)); + auto processor = std::unique_ptr(new opentelemetry::sdk::trace::SimpleSpanProcessor(std::move(jaeger_exporter))); + const auto jaeger_resource = opentelemetry::sdk::resource::Resource::Create(std::move(opentelemetry::sdk::resource::ResourceAttributes{{"service.name", service_name}})); + const auto provider = opentelemetry::nostd::shared_ptr(new opentelemetry::sdk::trace::TracerProvider(std::move(processor), jaeger_resource)); + tracer = provider->GetTracer(service_name, OPENTELEMETRY_SDK_VERSION); } } + +void Tracer::shutdown() { + if (tracer) { + tracer->CloseWithMicroseconds(1); + } +} + +jspan Tracer::start_trace(opentelemetry::nostd::string_view trace_name) { + if (is_enabled()) { + return tracer->StartSpan(trace_name); + } + return noop_tracer->StartSpan(trace_name); +} + +jspan Tracer::add_span(opentelemetry::nostd::string_view span_name, jspan& parent_span) { + if (is_enabled() && parent_span) { + const auto parent_ctx = parent_span->GetContext(); + if (parent_ctx.IsValid()) { + opentelemetry::trace::StartSpanOptions span_opts; + span_opts.parent = parent_ctx; + return tracer->StartSpan(span_name, span_opts); + } + } + return noop_tracer->StartSpan(span_name); +} + +bool Tracer::is_enabled() const { + return g_ceph_context->_conf->jaeger_tracing_enable; +} + +} // namespace tracing + +#endif // HAVE_JAEGER diff --git a/src/common/tracer.h b/src/common/tracer.h index 028ea04d831..5e7f4e799be 100644 --- a/src/common/tracer.h +++ b/src/common/tracer.h @@ -1,32 +1,82 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#ifndef TRACER_H_ -#define TRACER_H_ +#pragma once -#define SIGNED_RIGHT_SHIFT_IS 1 -#define ARITHMETIC_RIGHT_SHIFT 1 +#include "acconfig.h" -#include +#ifdef HAVE_JAEGER -typedef std::unique_ptr jspan; +#include "opentelemetry/trace/provider.h" +#include "opentelemetry/exporters/jaeger/jaeger_exporter.h" +#include "opentelemetry/sdk/trace/simple_processor.h" +#include "opentelemetry/sdk/trace/tracer_provider.h" -namespace jaeger_tracing { +using jspan = opentelemetry::nostd::shared_ptr; - extern std::shared_ptr tracer; +namespace tracing { - void init_tracer(const char* tracer_name); +class Tracer { + private: + const static opentelemetry::nostd::shared_ptr noop_tracer; + opentelemetry::nostd::shared_ptr tracer; - //create a root jspan - jspan new_span(const char*); + public: + Tracer() = default; + Tracer(opentelemetry::nostd::string_view service_name); - //create a child_span used given parent_span - jspan child_span(const char*, const jspan&); + void init(opentelemetry::nostd::string_view service_name); + void shutdown(); - //finish tracing of a single jspan - void finish_span(const jspan&); + bool is_enabled() const; + // creates and returns a new span with `trace_name` + // this span represents a trace, since it has no parent. + jspan start_trace(opentelemetry::nostd::string_view trace_name); + // creates and returns a new span with `span_name` which parent span is `parent_span' + jspan add_span(opentelemetry::nostd::string_view span_name, jspan& parent_span); - //setting tags in sundefined reference topans - void set_span_tag(const jspan&, const char*, const char*); +}; + + +} // namespace tracing + + +#else // !HAVE_JAEGER + +#include + +class Value { + public: + template Value(T val) {} +}; + +struct span_stub { + template + void SetAttribute(std::string_view key, const T& value) const noexcept {} + void AddEvent(std::string_view, std::initializer_list> fields) {} +}; + +class jspan { + span_stub span; + public: + span_stub& operator*() { return span; } + const span_stub& operator*() const { return span; } + + span_stub* operator->() { return &span; } + const span_stub* operator->() const { return &span; } + + operator bool() const { return false; } +}; + +namespace tracing { + +struct Tracer { + bool is_enabled() const { return false; } + jspan start_trace(std::string_view) { return {}; } + jspan add_span(std::string_view, const jspan&) { return {}; } + void init(std::string_view service_name) {} + void shutdown() {} +}; } -#endif // TRACER_H_ + +#endif // !HAVE_JAEGER diff --git a/src/osd/CMakeLists.txt b/src/osd/CMakeLists.txt index 26a0dd519fc..d95efc35070 100644 --- a/src/osd/CMakeLists.txt +++ b/src/osd/CMakeLists.txt @@ -43,7 +43,8 @@ set(osd_srcs ${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc ${CMAKE_SOURCE_DIR}/src/mgr/OSDPerfMetricTypes.cc ${osd_cyg_functions_src} - ${osdc_osd_srcs}) + ${osdc_osd_srcs} + osd_tracer.cc) if(HAS_VTA) set_source_files_properties(osdcap.cc PROPERTIES COMPILE_FLAGS -fno-var-tracking-assignments) diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index b08be6e6f70..b061e0f69ca 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -25,6 +25,7 @@ #include "ECMsgTypes.h" #include "PrimaryLogPG.h" +#include "osd_tracer.h" #define dout_context cct #define dout_subsys ceph_subsys_osd @@ -947,14 +948,13 @@ void ECBackend::handle_sub_write( ECSubWrite &op, const ZTracer::Trace &trace) { - if (msg) + jspan span; + if (msg) { msg->mark_event("sub_op_started"); - trace.event("handle_sub_write"); -#ifdef HAVE_JAEGER - if (msg && msg->osd_parent_span) { - auto ec_sub_trans = jaeger_tracing::child_span(__func__, msg->osd_parent_span); + span = tracing::osd::tracer.add_span(__func__, msg->osd_parent_span); } -#endif + trace.event("handle_sub_write"); + if (!get_parent()->pgb_is_primary()) get_parent()->update_stats(op.stats); ObjectStore::Transaction localt; @@ -1550,14 +1550,11 @@ void ECBackend::submit_transaction( op->tid = tid; op->reqid = reqid; op->client_op = client_op; - if (client_op) + jspan span; + if (client_op) { op->trace = client_op->pg_trace; - -#ifdef HAVE_JAEGER - if (client_op && client_op->osd_parent_span) { - auto ec_sub_trans = jaeger_tracing::child_span("ECBackend::submit_transaction", client_op->osd_parent_span); + span = tracing::osd::tracer.add_span("ECBackend::submit_transaction", client_op->osd_parent_span); } -#endif dout(10) << __func__ << ": op " << *op << " starting" << dendl; start_rmw(op, std::move(t)); } @@ -2129,12 +2126,11 @@ bool ECBackend::try_reads_to_commit() messages.push_back(std::make_pair(i->osd, r)); } } + jspan span; + if (op->client_op) { + span = tracing::osd::tracer.add_span("EC sub write", op->client_op->osd_parent_span); + } -#ifdef HAVE_JAEGER - if (op->client_op && op->client_op->osd_parent_span) { - auto sub_write_span = jaeger_tracing::child_span("EC sub write", op->client_op->osd_parent_span); - } -#endif if (!messages.empty()) { get_parent()->send_message_osd_cluster(messages, get_osdmap_epoch()); } diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index ca1dc472b21..5b73d77d006 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -174,9 +174,9 @@ #else #define tracepoint(...) #endif -#ifdef HAVE_JAEGER -#include "common/tracer.h" -#endif + +#include "osd_tracer.h" + #define dout_context cct #define dout_subsys ceph_subsys_osd @@ -217,6 +217,7 @@ static ostream& _prefix(std::ostream* _dout, int whoami, epoch_t epoch) { return *_dout << "osd." << whoami << " " << epoch << " "; } + //Initial features in new superblock. //Features here are also automatically upgraded CompatSet OSD::get_osd_initial_compat_set() { @@ -3455,7 +3456,7 @@ int OSD::init() std::lock_guard lock(osd_lock); if (is_stopping()) return 0; - + tracing::osd::tracer.init("osd"); tick_timer.init(); tick_timer_without_osd_lock.init(); service.recovery_request_timer.init(); @@ -4435,6 +4436,8 @@ int OSD::shutdown() hb_front_server_messenger->shutdown(); hb_back_server_messenger->shutdown(); + tracing::osd::tracer.shutdown(); + return r; } @@ -7188,18 +7191,12 @@ void OSD::dispatch_session_waiting(const ceph::ref_t& session, OSDMapRe void OSD::ms_fast_dispatch(Message *m) { - -#ifdef HAVE_JAEGER - jaeger_tracing::init_tracer("osd-services-reinit"); - dout(10) << "jaeger tracer after " << opentracing::Tracer::Global() << dendl; - auto dispatch_span = jaeger_tracing::new_span(__func__); -#endif + auto dispatch_span = tracing::osd::tracer.start_trace(__func__); FUNCTRACE(cct); if (service.is_stopping()) { m->put(); return; } - // peering event? switch (m->get_type()) { case CEPH_MSG_PING: @@ -7250,13 +7247,8 @@ void OSD::ms_fast_dispatch(Message *m) tracepoint(osd, ms_fast_dispatch, reqid.name._type, reqid.name._num, reqid.tid, reqid.inc); } -#ifdef HAVE_JAEGER - op->set_osd_parent_span(dispatch_span); - if (op->osd_parent_span) { - auto op_req_span = jaeger_tracing::child_span("op-request-created", op->osd_parent_span); - op->set_osd_parent_span(op_req_span); - } -#endif + op->osd_parent_span = tracing::osd::tracer.add_span("op-request-created", dispatch_span); + if (m->trace) op->osd_trace.init("osd op", &trace_endpoint, &m->trace); @@ -9649,18 +9641,16 @@ void OSD::enqueue_op(spg_t pg, OpRequestRef&& op, epoch_t epoch) op->osd_trace.event("enqueue op"); op->osd_trace.keyval("priority", priority); op->osd_trace.keyval("cost", cost); -#ifdef HAVE_JAEGER - if (op->osd_parent_span) { - auto enqueue_span = jaeger_tracing::child_span(__func__, op->osd_parent_span); - enqueue_span->Log({ - {"priority", priority}, - {"cost", cost}, - {"epoch", epoch}, - {"owner", owner}, - {"type", type} - }); - } -#endif + + auto enqueue_span = tracing::osd::tracer.add_span(__func__, op->osd_parent_span); + enqueue_span->AddEvent(__func__, { + {"priority", priority}, + {"cost", cost}, + {"epoch", epoch}, + {"owner", owner}, + {"type", type} + }); + op->mark_queued_for_pg(); logger->tinc(l_osd_op_before_queue_op_lat, latency); if (type == MSG_OSD_PG_PUSH || diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 0a03a85ac27..79aa21cd392 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -17,10 +17,7 @@ #include "osd/osd_op_util.h" #include "osd/osd_types.h" #include "common/TrackedOp.h" -#ifdef HAVE_JAEGER #include "common/tracer.h" -#endif - /** * The OpRequest takes in a Message* and takes over a single reference * to it, which it puts() when destroyed. @@ -91,17 +88,8 @@ public: epoch_t min_epoch = 0; ///< min epoch needed to handle this msg bool hitset_inserted; -#ifdef HAVE_JAEGER - jspan osd_parent_span = nullptr; - void set_osd_parent_span(jspan& span) { - if(osd_parent_span){ - jaeger_tracing::finish_span(osd_parent_span); - } - osd_parent_span = move(span); - } -#else - void set_osd_parent_span(...) {} -#endif + jspan osd_parent_span; + template const T* get_req() const { return static_cast(request); } diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 298716a1a0c..88f074123b6 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -81,9 +81,7 @@ #undef dout_prefix #define dout_prefix _prefix(_dout, this) -#ifdef HAVE_JAEGER -#include "common/tracer.h" -#endif +#include "osd_tracer.h" MEMPOOL_DEFINE_OBJECT_FACTORY(PrimaryLogPG, replicatedpg, osd); @@ -1799,11 +1797,9 @@ void PrimaryLogPG::do_request( op->pg_trace.init("pg op", &trace_endpoint, &op->osd_trace); op->pg_trace.event("do request"); } -#ifdef HAVE_JAEGER - if (op->osd_parent_span) { - auto do_req_span = jaeger_tracing::child_span(__func__, op->osd_parent_span); - } -#endif + + [[maybe_unused]] auto span = tracing::osd::tracer.add_span(__func__, op->osd_parent_span); + // make sure we have a new enough map auto p = waiting_for_map.find(op->get_source()); if (p != waiting_for_map.end()) { @@ -2175,11 +2171,8 @@ void PrimaryLogPG::do_op(OpRequestRef& op) << " flags " << ceph_osd_flag_string(m->get_flags()) << dendl; -#ifdef HAVE_JAEGER - if (op->osd_parent_span) { - auto do_op_span = jaeger_tracing::child_span(__func__, op->osd_parent_span); - } -#endif + [[maybe_unused]] auto span = tracing::osd::tracer.add_span(__func__, op->osd_parent_span); + // missing object? if (is_unreadable_object(head)) { if (!is_primary()) { @@ -4190,11 +4183,8 @@ void PrimaryLogPG::execute_ctx(OpContext *ctx) tracepoint(osd, prepare_tx_enter, reqid.name._type, reqid.name._num, reqid.tid, reqid.inc); } -#ifdef HAVE_JAEGER - if (ctx->op->osd_parent_span) { - auto execute_span = jaeger_tracing::child_span(__func__, ctx->op->osd_parent_span); - } -#endif + + [[maybe_unused]] auto span = tracing::osd::tracer.add_span(__func__, ctx->op->osd_parent_span); int result = prepare_transaction(ctx); @@ -5956,12 +5946,11 @@ int PrimaryLogPG::do_osd_ops(OpContext *ctx, vector& ops) PGTransaction* t = ctx->op_t.get(); dout(10) << "do_osd_op " << soid << " " << ops << dendl; -#ifdef HAVE_JAEGER - if (ctx->op->osd_parent_span) { - auto do_osd_op_span = jaeger_tracing::child_span(__func__, ctx->op->osd_parent_span); - } -#endif + jspan span; + if (ctx->op) { + span = tracing::osd::tracer.add_span(__func__, ctx->op->osd_parent_span); + } ctx->current_osd_subop_num = 0; for (auto p = ops.begin(); p != ops.end(); ++p, ctx->current_osd_subop_num++, ctx->processed_subop_count++) { OSDOp& osd_op = *p; @@ -8914,11 +8903,11 @@ void PrimaryLogPG::finish_ctx(OpContext *ctx, int log_op_type, int result) << dendl; utime_t now = ceph_clock_now(); -#ifdef HAVE_JAEGER - if (ctx->op->osd_parent_span) { - auto finish_ctx_span = jaeger_tracing::child_span(__func__, ctx->op->osd_parent_span); + jspan span; + if (ctx->op) { + span = tracing::osd::tracer.add_span(__func__, ctx->op->osd_parent_span); } -#endif + // Drop the reference if deduped chunk is modified if (ctx->new_obs.oi.is_dirty() && (ctx->obs->oi.has_manifest() && ctx->obs->oi.manifest.is_chunked()) && @@ -11302,11 +11291,10 @@ void PrimaryLogPG::op_applied(const eversion_t &applied_version) void PrimaryLogPG::eval_repop(RepGather *repop) { - #ifdef HAVE_JAEGER - if (repop->op->osd_parent_span) { - auto eval_span = jaeger_tracing::child_span(__func__, repop->op->osd_parent_span); + jspan span; + if (repop->op) { + span = tracing::osd::tracer.add_span(__func__, repop->op->osd_parent_span); } - #endif dout(10) << "eval_repop " << *repop << (repop->op && repop->op->get_req() ? "" : " (no op)") << dendl; @@ -11361,11 +11349,11 @@ void PrimaryLogPG::issue_repop(RepGather *repop, OpContext *ctx) dout(7) << "issue_repop rep_tid " << repop->rep_tid << " o " << soid << dendl; -#ifdef HAVE_JAEGER - if (ctx->op->osd_parent_span) { - auto issue_repop_span = jaeger_tracing::child_span(__func__, ctx->op->osd_parent_span); + + jspan span; + if (ctx->op) { + span = tracing::osd::tracer.add_span(__func__, ctx->op->osd_parent_span); } -#endif repop->v = ctx->at_version; diff --git a/src/osd/ReplicatedBackend.cc b/src/osd/ReplicatedBackend.cc index b0715e2decb..50a6c60d7f0 100644 --- a/src/osd/ReplicatedBackend.cc +++ b/src/osd/ReplicatedBackend.cc @@ -23,6 +23,7 @@ #include "include/random.h" #include "include/util.h" #include "OSD.h" +#include "osd_tracer.h" #define dout_context cct #define dout_subsys ceph_subsys_osd @@ -503,9 +504,11 @@ void ReplicatedBackend::submit_transaction( ceph_assert(insert_res.second); InProgressOp &op = *insert_res.first->second; -#ifdef HAVE_JAEGER - auto rep_sub_trans = jaeger_tracing::child_span("ReplicatedBackend::submit_transaction", orig_op->osd_parent_span); -#endif + jspan span; + if (orig_op) { + span = tracing::osd::tracer.add_span("ReplicatedBackend::submit_transaction", orig_op->osd_parent_span); + } + op.waiting_for_commit.insert( parent->get_acting_recovery_backfill_shards().begin(), parent->get_acting_recovery_backfill_shards().end()); @@ -1059,9 +1062,10 @@ void ReplicatedBackend::do_repop(OpRequestRef op) << " " << m->logbl.length() << dendl; -#ifdef HAVE_JAEGER - auto do_repop_span = jaeger_tracing::child_span(__func__, op->osd_parent_span); -#endif + jspan span; + if (op) { + span = tracing::osd::tracer.add_span(__func__, op->osd_parent_span); + } // sanity checks ceph_assert(m->map_epoch >= get_info().history.same_interval_since); diff --git a/src/osd/osd_tracer.cc b/src/osd/osd_tracer.cc new file mode 100644 index 00000000000..06217926cfb --- /dev/null +++ b/src/osd/osd_tracer.cc @@ -0,0 +1,12 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "osd_tracer.h" + +namespace tracing { +namespace osd { + +tracing::Tracer tracer; + +} // namespace osd +} // namespace tracing diff --git a/src/osd/osd_tracer.h b/src/osd/osd_tracer.h new file mode 100644 index 00000000000..1490b435106 --- /dev/null +++ b/src/osd/osd_tracer.h @@ -0,0 +1,13 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once + +#include "common/tracer.h" + +namespace tracing { +namespace osd { + +extern tracing::Tracer tracer; + +} // namespace osd +} // namespace tracing diff --git a/src/osd/scheduler/OpSchedulerItem.cc b/src/osd/scheduler/OpSchedulerItem.cc index 27db1dfa37b..cced6c9d777 100644 --- a/src/osd/scheduler/OpSchedulerItem.cc +++ b/src/osd/scheduler/OpSchedulerItem.cc @@ -14,9 +14,8 @@ #include "osd/scheduler/OpSchedulerItem.h" #include "osd/OSD.h" -#ifdef HAVE_JAEGER -#include "common/tracer.h" -#endif +#include "osd/osd_tracer.h" + namespace ceph::osd::scheduler { @@ -26,9 +25,7 @@ void PGOpItem::run( PGRef& pg, ThreadPool::TPHandle &handle) { -#ifdef HAVE_JAEGER - auto PGOpItem_span = jaeger_tracing::child_span("PGOpItem::run", op->osd_parent_span); -#endif + [[maybe_unused]] auto span = tracing::osd::tracer.add_span("PGOpItem::run", op->osd_parent_span); osd->dequeue_op(pg, op, handle); pg->unlock(); } diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 383c8074ccd..c86d3dd793d 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -38,7 +38,7 @@ #include "cls/rgw/cls_rgw_types.h" #include "include/rados/librados.hpp" #include "rgw_public_access.h" -#include "rgw_tracer.h" +#include "common/tracer.h" namespace ceph { class Formatter; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 14672bd9e34..b2354f3140f 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -21,6 +21,7 @@ #include "common/utf8.h" #include "common/ceph_json.h" #include "common/static_ptr.h" +#include "rgw_tracer.h" #include "rgw_rados.h" #include "rgw_zone.h" @@ -51,7 +52,6 @@ #include "rgw_notify_event_type.h" #include "rgw_sal.h" #include "rgw_sal_rados.h" -#include "rgw_tracer.h" #include "services/svc_zone.h" #include "services/svc_quota.h" @@ -3923,7 +3923,7 @@ void RGWPutObj::execute(optional_yield y) rgw_placement_rule *pdest_placement = &s->dest_placement; if (multipart) { - s->trace->SetTag(tracing::UPLOAD_ID, multipart_upload_id); + s->trace->SetAttribute(tracing::rgw::UPLOAD_ID, multipart_upload_id); std::unique_ptr upload; upload = s->bucket->get_multipart_upload(s->object->get_name(), multipart_upload_id); @@ -6157,7 +6157,7 @@ void RGWInitMultipart::execute(optional_yield y) if (op_ret == 0) { upload_id = upload->get_upload_id(); } - s->trace->SetTag(tracing::UPLOAD_ID, upload_id); + s->trace->SetAttribute(tracing::rgw::UPLOAD_ID, upload_id); } @@ -6279,7 +6279,7 @@ void RGWCompleteMultipart::execute(optional_yield y) upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); - s->trace->SetTag(tracing::UPLOAD_ID, upload_id); + s->trace->SetAttribute(tracing::rgw::UPLOAD_ID, upload_id); RGWCompressionInfo cs_info; bool compressed = false; diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 3f8c9c33efe..80338086c75 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -17,7 +17,7 @@ #include "rgw_perf_counters.h" #include "rgw_lua.h" #include "rgw_lua_request.h" - +#include "rgw_tracer.h" #include "services/svc_zone_utils.h" #define dout_subsys ceph_subsys_rgw @@ -145,7 +145,7 @@ int rgw_process_authenticated(RGWHandler_REST * const handler, ldpp_dout(op, 2) << "verifying op permissions" << dendl; { - auto span = rgw_tracer.start_span("verify_permission", s->trace); + auto span = tracing::rgw::tracer.add_span("verify_permission", s->trace); std::swap(span, s->trace); ret = op->verify_permission(y); std::swap(span, s->trace); @@ -171,7 +171,7 @@ int rgw_process_authenticated(RGWHandler_REST * const handler, ldpp_dout(op, 2) << "executing" << dendl; { - auto span = rgw_tracer.start_span("execute", s->trace); + auto span = tracing::rgw::tracer.add_span("execute", s->trace); std::swap(span, s->trace); op->execute(y); std::swap(span, s->trace); @@ -275,7 +275,6 @@ int process_request(rgw::sal::Store* const store, } req->op = op; ldpp_dout(op, 10) << "op=" << typeid(*op).name() << dendl; - s->op_type = op->get_type(); try { @@ -308,9 +307,9 @@ int process_request(rgw::sal::Store* const store, } const auto trace_name = std::string(op->name()) + " " + s->trans_id; - s->trace = rgw_tracer.start_trace(trace_name); - s->trace->SetTag(tracing::OP, op->name()); - s->trace->SetTag(tracing::TYPE, tracing::REQUEST); + s->trace = tracing::rgw::tracer.start_trace(trace_name); + s->trace->SetAttribute(tracing::rgw::OP, op->name()); + s->trace->SetAttribute(tracing::rgw::TYPE, tracing::rgw::REQUEST); ret = rgw_process_authenticated(handler, op, req, s, yield); if (ret < 0) { @@ -324,15 +323,15 @@ int process_request(rgw::sal::Store* const store, done: if (op) { - s->trace->SetTag(tracing::RETURN, op->get_ret()); + s->trace->SetAttribute(tracing::rgw::RETURN, op->get_ret()); if (s->user) { - s->trace->SetTag(tracing::USER_ID, s->user->get_id().id); + s->trace->SetAttribute(tracing::rgw::USER_ID, s->user->get_id().id); } if (s->bucket) { - s->trace->SetTag(tracing::BUCKET_NAME, s->bucket->get_name()); + s->trace->SetAttribute(tracing::rgw::BUCKET_NAME, s->bucket->get_name()); } if (s->object) { - s->trace->SetTag(tracing::OBJECT_NAME, s->object->get_name()); + s->trace->SetAttribute(tracing::rgw::OBJECT_NAME, s->object->get_name()); } std::string script; auto rc = rgw::lua::read_script(s, store, s->bucket_tenant, s->yield, rgw::lua::context::postRequest, script); diff --git a/src/rgw/rgw_tracer.cc b/src/rgw/rgw_tracer.cc index b0b25172857..2d6720f591b 100644 --- a/src/rgw/rgw_tracer.cc +++ b/src/rgw/rgw_tracer.cc @@ -1,55 +1,17 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp -#include "common/ceph_context.h" -#include "global/global_context.h" +// vim: ts=8 sw=2 smarttab + +#include #include "rgw_tracer.h" -#ifdef HAVE_JAEGER - -thread_local tracing::Tracer rgw_tracer(jaeger_configuration::jaeger_default_config); - namespace tracing { +namespace rgw { -const std::shared_ptr Tracer::noop_tracer = opentracing::MakeNoopTracer(); - -Tracer::Tracer(jaegertracing::Config& conf):open_tracer(jaegertracing::Tracer::make(conf)) {} - -std::unique_ptr Tracer::start_trace(opentracing::string_view trace_name) { - if(is_enabled()) { - return open_tracer->StartSpan(trace_name); - } - return noop_tracer->StartSpan(trace_name); -} - -std::unique_ptr Tracer::start_span(opentracing::string_view span_name, std::unique_ptr& parent_span) { - if(is_enabled()) { - return open_tracer->StartSpan(span_name, { opentracing::ChildOf(&parent_span->context()) }); - } - return noop_tracer->StartSpan(span_name); -} - -bool Tracer::is_enabled() const { - return g_ceph_context->_conf->rgw_jaeger_enable; -} -} // namespace tracing - -namespace jaeger_configuration { - -jaegertracing::samplers::Config const_sampler("const", 1, "", 0, jaegertracing::samplers::Config::defaultSamplingRefreshInterval()); - -jaegertracing::reporters::Config reporter_default_config(jaegertracing::reporters::Config::kDefaultQueueSize, jaegertracing::reporters::Config::defaultBufferFlushInterval(), true, jaegertracing::reporters::Config::kDefaultLocalAgentHostPort, ""); - -jaegertracing::propagation::HeadersConfig headers_config("","","",""); - -jaegertracing::baggage::RestrictionsConfig baggage_config(false, "", std::chrono::steady_clock::duration()); - -jaegertracing::Config jaeger_default_config(false, const_sampler, reporter_default_config, headers_config, baggage_config, "rgw", std::vector()); - -} - +#ifdef HAVE_JAEGER +thread_local tracing::Tracer tracer("rgw"); #else // !HAVE_JAEGER - -tracing::Tracer rgw_tracer; - +tracing::Tracer tracer; #endif - \ No newline at end of file + +} // namespace rgw +} // namespace tracing diff --git a/src/rgw/rgw_tracer.h b/src/rgw/rgw_tracer.h index 5c70bc7e55a..4c50b4ea00b 100644 --- a/src/rgw/rgw_tracer.h +++ b/src/rgw/rgw_tracer.h @@ -1,38 +1,10 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#ifndef RGW_TRACER_H -#define RGW_TRACER_H - -#ifdef HAVE_JAEGER - -#define SIGNED_RIGHT_SHIFT_IS 1 -#define ARITHMETIC_RIGHT_SHIFT 1 -#include - -typedef std::unique_ptr jspan; - -#else // !HAVE_JAEGER - -struct span_stub { - template - void SetTag(std::string_view key, const T& value) const noexcept {} - void Log(std::initializer_list> fields) {} -}; - -class jspan { - span_stub span; - public: - span_stub& operator*() { return span; } - const span_stub& operator*() const { return span; } - - span_stub* operator->() { return &span; } - const span_stub* operator->() const { return &span; } -}; - -#endif // !HAVE_JAEGER +// vim: ts=8 sw=2 smarttab +#pragma once +#include "common/tracer.h" namespace tracing { +namespace rgw { const auto OP = "op"; const auto BUCKET_NAME = "bucket_name"; @@ -44,50 +16,11 @@ const auto TYPE = "type"; const auto REQUEST = "request"; #ifdef HAVE_JAEGER - -class Tracer { -private: - const static std::shared_ptr noop_tracer; - std::shared_ptr open_tracer; - -public: - Tracer(jaegertracing::Config& conf); - bool is_enabled() const; - // creates and returns a new span with `trace_name` - // this span represents a trace, since it has no parent. - jspan start_trace(opentracing::string_view trace_name); - // creates and returns a new span with `trace_name` which parent span is `parent_span' - jspan start_span(opentracing::string_view span_name, jspan& parent_span); -}; - -#else // !HAVE_JAEGER - -struct Tracer { - bool is_enabled() const { return false; } - jspan start_trace(std::string_view) { return {}; } - jspan start_span(std::string_view, const jspan&) { return {}; } -}; - +extern thread_local tracing::Tracer tracer; +#else +extern tracing::Tracer tracer; #endif +} // namespace rgw } // namespace tracing - -#ifdef HAVE_JAEGER - -extern thread_local tracing::Tracer rgw_tracer; - -namespace jaeger_configuration { - -extern jaegertracing::Config jaeger_default_config; - -} - -#else // !HAVE_JAEGER - -extern tracing::Tracer rgw_tracer; - -#endif - -#endif // RGW_TRACER_H - \ No newline at end of file