From 3de10e3b445dc05858e1b37326956d8246416786 Mon Sep 17 00:00:00 2001 From: Goutham Veeramachaneni Date: Thu, 30 Nov 2017 16:05:29 +0530 Subject: [PATCH] Add CleanTombstones API endpoint Signed-off-by: Goutham Veeramachaneni --- documentation/dev/api/swagger.json | 21 ++- prompb/remote.pb.go | 20 ++ prompb/rpc.pb.go | 286 +++++++++++++++++++++++++---- prompb/rpc.pb.gw.go | 83 +++++++-- prompb/rpc.proto | 16 +- prompb/types.pb.go | 33 +++- scripts/genproto.sh | 4 +- web/api/v2/api.go | 19 ++ 8 files changed, 424 insertions(+), 58 deletions(-) diff --git a/documentation/dev/api/swagger.json b/documentation/dev/api/swagger.json index 59c07fb83..6ba0e163a 100644 --- a/documentation/dev/api/swagger.json +++ b/documentation/dev/api/swagger.json @@ -15,6 +15,23 @@ "application/json" ], "paths": { + "/v2/admin/tsdb/clean_tombstones": { + "post": { + "summary": "CleanTombstones removes the deleted data from disk and cleans up the existing tombstones.", + "operationId": "TSDBCleanTombstones", + "responses": { + "200": { + "description": "", + "schema": { + "$ref": "#/definitions/prometheusTSDBCleanTombstonesResponse" + } + } + }, + "tags": [ + "Admin" + ] + } + }, "/v2/admin/tsdb/delete_series": { "post": { "summary": "DeleteSeries deletes data for a selection of series in a time range.", @@ -44,7 +61,7 @@ }, "/v2/admin/tsdb/snapshot": { "post": { - "summary": "Snapshot creates a snapshot of all current data into 'snapshots/\u003cdatetime\u003e-\u003crand\u003e' under\nthe TSDB's data directory.", + "summary": "Snapshot creates a snapshot of all current data into 'snapshots/\u003cdatetime\u003e-\u003crand\u003e' under the TSDB's data directory.", "operationId": "TSDBSnapshot", "responses": { "200": { @@ -108,7 +125,7 @@ "prometheusSeriesDeleteResponse": { "type": "object" }, - "prometheusTSDBSnapshotRequest": { + "prometheusTSDBCleanTombstonesResponse": { "type": "object" }, "prometheusTSDBSnapshotResponse": { diff --git a/prompb/remote.pb.go b/prompb/remote.pb.go index c8570a162..d61048134 100644 --- a/prompb/remote.pb.go +++ b/prompb/remote.pb.go @@ -17,6 +17,8 @@ QueryResult TSDBSnapshotRequest TSDBSnapshotResponse + TSDBCleanTombstonesRequest + TSDBCleanTombstonesResponse SeriesDeleteRequest SeriesDeleteResponse Sample @@ -308,6 +310,24 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Remote(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Remote(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintRemote(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) diff --git a/prompb/rpc.pb.go b/prompb/rpc.pb.go index 00c685ae0..40c3e9436 100644 --- a/prompb/rpc.pb.go +++ b/prompb/rpc.pb.go @@ -8,6 +8,7 @@ import fmt "fmt" import math "math" import _ "google.golang.org/genproto/googleapis/api/annotations" +import _ "github.com/gogo/protobuf/gogoproto" import time "time" @@ -43,6 +44,22 @@ func (m *TSDBSnapshotResponse) String() string { return proto.Compact func (*TSDBSnapshotResponse) ProtoMessage() {} func (*TSDBSnapshotResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{1} } +type TSDBCleanTombstonesRequest struct { +} + +func (m *TSDBCleanTombstonesRequest) Reset() { *m = TSDBCleanTombstonesRequest{} } +func (m *TSDBCleanTombstonesRequest) String() string { return proto.CompactTextString(m) } +func (*TSDBCleanTombstonesRequest) ProtoMessage() {} +func (*TSDBCleanTombstonesRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } + +type TSDBCleanTombstonesResponse struct { +} + +func (m *TSDBCleanTombstonesResponse) Reset() { *m = TSDBCleanTombstonesResponse{} } +func (m *TSDBCleanTombstonesResponse) String() string { return proto.CompactTextString(m) } +func (*TSDBCleanTombstonesResponse) ProtoMessage() {} +func (*TSDBCleanTombstonesResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } + type SeriesDeleteRequest struct { MinTime *time.Time `protobuf:"bytes,1,opt,name=min_time,json=minTime,stdtime" json:"min_time,omitempty"` MaxTime *time.Time `protobuf:"bytes,2,opt,name=max_time,json=maxTime,stdtime" json:"max_time,omitempty"` @@ -52,7 +69,7 @@ type SeriesDeleteRequest struct { func (m *SeriesDeleteRequest) Reset() { *m = SeriesDeleteRequest{} } func (m *SeriesDeleteRequest) String() string { return proto.CompactTextString(m) } func (*SeriesDeleteRequest) ProtoMessage() {} -func (*SeriesDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{2} } +func (*SeriesDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{4} } type SeriesDeleteResponse struct { } @@ -60,11 +77,13 @@ type SeriesDeleteResponse struct { func (m *SeriesDeleteResponse) Reset() { *m = SeriesDeleteResponse{} } func (m *SeriesDeleteResponse) String() string { return proto.CompactTextString(m) } func (*SeriesDeleteResponse) ProtoMessage() {} -func (*SeriesDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{3} } +func (*SeriesDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptorRpc, []int{5} } func init() { proto.RegisterType((*TSDBSnapshotRequest)(nil), "prometheus.TSDBSnapshotRequest") proto.RegisterType((*TSDBSnapshotResponse)(nil), "prometheus.TSDBSnapshotResponse") + proto.RegisterType((*TSDBCleanTombstonesRequest)(nil), "prometheus.TSDBCleanTombstonesRequest") + proto.RegisterType((*TSDBCleanTombstonesResponse)(nil), "prometheus.TSDBCleanTombstonesResponse") proto.RegisterType((*SeriesDeleteRequest)(nil), "prometheus.SeriesDeleteRequest") proto.RegisterType((*SeriesDeleteResponse)(nil), "prometheus.SeriesDeleteResponse") } @@ -80,9 +99,10 @@ const _ = grpc.SupportPackageIsVersion4 // Client API for Admin service type AdminClient interface { - // Snapshot creates a snapshot of all current data into 'snapshots/-' under - // the TSDB's data directory. + // Snapshot creates a snapshot of all current data into 'snapshots/-' under the TSDB's data directory. TSDBSnapshot(ctx context.Context, in *TSDBSnapshotRequest, opts ...grpc.CallOption) (*TSDBSnapshotResponse, error) + // CleanTombstones removes the deleted data from disk and cleans up the existing tombstones. + TSDBCleanTombstones(ctx context.Context, in *TSDBCleanTombstonesRequest, opts ...grpc.CallOption) (*TSDBCleanTombstonesResponse, error) // DeleteSeries deletes data for a selection of series in a time range. DeleteSeries(ctx context.Context, in *SeriesDeleteRequest, opts ...grpc.CallOption) (*SeriesDeleteResponse, error) } @@ -104,6 +124,15 @@ func (c *adminClient) TSDBSnapshot(ctx context.Context, in *TSDBSnapshotRequest, return out, nil } +func (c *adminClient) TSDBCleanTombstones(ctx context.Context, in *TSDBCleanTombstonesRequest, opts ...grpc.CallOption) (*TSDBCleanTombstonesResponse, error) { + out := new(TSDBCleanTombstonesResponse) + err := grpc.Invoke(ctx, "/prometheus.Admin/TSDBCleanTombstones", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *adminClient) DeleteSeries(ctx context.Context, in *SeriesDeleteRequest, opts ...grpc.CallOption) (*SeriesDeleteResponse, error) { out := new(SeriesDeleteResponse) err := grpc.Invoke(ctx, "/prometheus.Admin/DeleteSeries", in, out, c.cc, opts...) @@ -116,9 +145,10 @@ func (c *adminClient) DeleteSeries(ctx context.Context, in *SeriesDeleteRequest, // Server API for Admin service type AdminServer interface { - // Snapshot creates a snapshot of all current data into 'snapshots/-' under - // the TSDB's data directory. + // Snapshot creates a snapshot of all current data into 'snapshots/-' under the TSDB's data directory. TSDBSnapshot(context.Context, *TSDBSnapshotRequest) (*TSDBSnapshotResponse, error) + // CleanTombstones removes the deleted data from disk and cleans up the existing tombstones. + TSDBCleanTombstones(context.Context, *TSDBCleanTombstonesRequest) (*TSDBCleanTombstonesResponse, error) // DeleteSeries deletes data for a selection of series in a time range. DeleteSeries(context.Context, *SeriesDeleteRequest) (*SeriesDeleteResponse, error) } @@ -145,6 +175,24 @@ func _Admin_TSDBSnapshot_Handler(srv interface{}, ctx context.Context, dec func( return interceptor(ctx, in, info, handler) } +func _Admin_TSDBCleanTombstones_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TSDBCleanTombstonesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).TSDBCleanTombstones(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/prometheus.Admin/TSDBCleanTombstones", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).TSDBCleanTombstones(ctx, req.(*TSDBCleanTombstonesRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Admin_DeleteSeries_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SeriesDeleteRequest) if err := dec(in); err != nil { @@ -171,6 +219,10 @@ var _Admin_serviceDesc = grpc.ServiceDesc{ MethodName: "TSDBSnapshot", Handler: _Admin_TSDBSnapshot_Handler, }, + { + MethodName: "TSDBCleanTombstones", + Handler: _Admin_TSDBCleanTombstones_Handler, + }, { MethodName: "DeleteSeries", Handler: _Admin_DeleteSeries_Handler, @@ -222,6 +274,42 @@ func (m *TSDBSnapshotResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *TSDBCleanTombstonesRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TSDBCleanTombstonesRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *TSDBCleanTombstonesResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TSDBCleanTombstonesResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + func (m *SeriesDeleteRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -290,6 +378,24 @@ func (m *SeriesDeleteResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Rpc(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Rpc(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -315,6 +421,18 @@ func (m *TSDBSnapshotResponse) Size() (n int) { return n } +func (m *TSDBCleanTombstonesRequest) Size() (n int) { + var l int + _ = l + return n +} + +func (m *TSDBCleanTombstonesResponse) Size() (n int) { + var l int + _ = l + return n +} + func (m *SeriesDeleteRequest) Size() (n int) { var l int _ = l @@ -483,6 +601,106 @@ func (m *TSDBSnapshotResponse) Unmarshal(dAtA []byte) error { } return nil } +func (m *TSDBCleanTombstonesRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TSDBCleanTombstonesRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TSDBCleanTombstonesRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *TSDBCleanTombstonesResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TSDBCleanTombstonesResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TSDBCleanTombstonesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipRpc(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRpc + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -788,30 +1006,34 @@ var ( func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 393 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x51, 0xcd, 0xae, 0xd2, 0x40, - 0x14, 0xbe, 0x73, 0x2f, 0xa2, 0x0c, 0xac, 0x0a, 0x6a, 0x6d, 0x48, 0x8b, 0xb3, 0x91, 0xb0, 0xe8, - 0x24, 0x75, 0x87, 0x2b, 0x1b, 0x96, 0xba, 0x29, 0xac, 0xdc, 0x90, 0x29, 0x1c, 0x4b, 0x13, 0x3a, - 0x33, 0x76, 0x06, 0x83, 0x2e, 0x7d, 0x02, 0x13, 0x5f, 0x8a, 0xc4, 0x8d, 0x89, 0x7b, 0x7f, 0x88, - 0xef, 0xe0, 0xd6, 0x74, 0x86, 0x2a, 0x35, 0x98, 0xb8, 0x3b, 0x3d, 0xfd, 0xbe, 0xf3, 0xfd, 0x0c, - 0xee, 0x94, 0x72, 0x15, 0xca, 0x52, 0x68, 0xe1, 0x60, 0x59, 0x8a, 0x02, 0xf4, 0x06, 0x76, 0xca, - 0xeb, 0xea, 0x37, 0x12, 0x94, 0xfd, 0xe1, 0x05, 0x99, 0x10, 0xd9, 0x16, 0xa8, 0xf9, 0x4a, 0x77, - 0x2f, 0xa9, 0xce, 0x0b, 0x50, 0x9a, 0x15, 0xf2, 0x04, 0x18, 0x9e, 0x00, 0x4c, 0xe6, 0x94, 0x71, - 0x2e, 0x34, 0xd3, 0xb9, 0xe0, 0x35, 0x7d, 0x90, 0x89, 0x4c, 0x98, 0x91, 0x56, 0x93, 0xdd, 0x92, - 0xbb, 0xb8, 0xbf, 0x98, 0xcf, 0xe2, 0x39, 0x67, 0x52, 0x6d, 0x84, 0x4e, 0xe0, 0xd5, 0x0e, 0x94, - 0x26, 0x13, 0x3c, 0x68, 0xae, 0x95, 0x14, 0x5c, 0x81, 0xe3, 0xe0, 0x16, 0x67, 0x05, 0xb8, 0x68, - 0x84, 0xc6, 0x9d, 0xc4, 0xcc, 0xe4, 0x23, 0xc2, 0xfd, 0x39, 0x94, 0x39, 0xa8, 0x19, 0x6c, 0x41, - 0xc3, 0xe9, 0x86, 0xf3, 0x04, 0xdf, 0x29, 0x72, 0xbe, 0xac, 0x5c, 0x1a, 0x7c, 0x37, 0xf2, 0x42, - 0xeb, 0x30, 0xac, 0x23, 0x84, 0x8b, 0x3a, 0x42, 0xdc, 0x7a, 0xff, 0x35, 0x40, 0xc9, 0xed, 0x22, - 0xe7, 0xd5, 0xce, 0x90, 0xd9, 0xde, 0x92, 0xaf, 0xff, 0x9b, 0xcc, 0xf6, 0x86, 0x3c, 0xad, 0xc8, - 0x7a, 0xb5, 0x81, 0x52, 0xb9, 0x37, 0xa3, 0x9b, 0x71, 0x37, 0x72, 0xc3, 0x3f, 0xad, 0x86, 0xcf, - 0x58, 0x0a, 0xdb, 0xe7, 0x16, 0x10, 0xb7, 0x0e, 0x5f, 0x82, 0xab, 0xe4, 0x37, 0x9e, 0xdc, 0xc3, - 0x83, 0x66, 0x18, 0x9b, 0x3c, 0xfa, 0x89, 0xf0, 0xad, 0xa7, 0xeb, 0x22, 0xe7, 0x4e, 0x89, 0x7b, - 0xe7, 0xdd, 0x38, 0xc1, 0xf9, 0xed, 0x0b, 0x65, 0x7a, 0xa3, 0x7f, 0x03, 0xec, 0x71, 0x12, 0xbc, - 0xfb, 0xfc, 0xe3, 0xc3, 0xf5, 0x03, 0x72, 0x9f, 0xbe, 0x8e, 0x28, 0xab, 0x54, 0xa8, 0x56, 0xeb, - 0x94, 0xaa, 0x5a, 0xe3, 0x2d, 0xee, 0x59, 0x3f, 0xd6, 0x5b, 0x53, 0xf3, 0x42, 0xf9, 0x4d, 0xcd, - 0x4b, 0x81, 0xc8, 0x23, 0xa3, 0xf9, 0x90, 0x0c, 0xff, 0xd2, 0x5c, 0x1b, 0xd8, 0x52, 0x19, 0xce, - 0x14, 0x4d, 0x62, 0xf7, 0xf0, 0xdd, 0xbf, 0x3a, 0x1c, 0x7d, 0xf4, 0xe9, 0xe8, 0xa3, 0x6f, 0x47, - 0x1f, 0xbd, 0x68, 0x57, 0xb7, 0x65, 0x9a, 0xb6, 0xcd, 0x53, 0x3c, 0xfe, 0x15, 0x00, 0x00, 0xff, - 0xff, 0xeb, 0x59, 0x61, 0x63, 0xbe, 0x02, 0x00, 0x00, + // 451 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x3d, 0x8f, 0xd3, 0x40, + 0x10, 0xbd, 0xbd, 0x84, 0x83, 0xdb, 0x5c, 0xe5, 0x0b, 0x10, 0x4c, 0x88, 0x83, 0x0b, 0xee, 0x74, + 0x85, 0x57, 0x0a, 0xdd, 0x51, 0x11, 0xae, 0x84, 0x26, 0x49, 0x45, 0x13, 0xad, 0x93, 0xc1, 0xb1, + 0xe4, 0xfd, 0xc0, 0xbb, 0x41, 0x07, 0x25, 0x1d, 0x15, 0x48, 0xfc, 0xa9, 0x48, 0x34, 0x48, 0xf4, + 0x7c, 0x44, 0xfc, 0x10, 0xb4, 0xbb, 0xf6, 0x5d, 0x6c, 0x19, 0x41, 0x37, 0x1e, 0xbf, 0x37, 0x6f, + 0xde, 0x9b, 0xc5, 0x87, 0xb9, 0x5c, 0x44, 0x32, 0x17, 0x5a, 0x78, 0x58, 0xe6, 0x82, 0x81, 0x5e, + 0xc1, 0x5a, 0xf9, 0x1d, 0xfd, 0x56, 0x82, 0x72, 0x3f, 0xfc, 0x20, 0x11, 0x22, 0xc9, 0x80, 0xd8, + 0xaf, 0x78, 0xfd, 0x8a, 0xe8, 0x94, 0x81, 0xd2, 0x94, 0xc9, 0x02, 0xd0, 0x2f, 0x00, 0x54, 0xa6, + 0x84, 0x72, 0x2e, 0x34, 0xd5, 0xa9, 0xe0, 0x25, 0xbd, 0x9b, 0x88, 0x44, 0xd8, 0x92, 0x98, 0xca, + 0x75, 0xc3, 0xdb, 0xf8, 0x78, 0x36, 0xbd, 0x18, 0x4f, 0x39, 0x95, 0x6a, 0x25, 0xf4, 0x04, 0x5e, + 0xaf, 0x41, 0xe9, 0xf0, 0x0c, 0x77, 0xab, 0x6d, 0x25, 0x05, 0x57, 0xe0, 0x79, 0xb8, 0xcd, 0x29, + 0x83, 0x1e, 0x1a, 0xa2, 0xd3, 0xc3, 0x89, 0xad, 0xc3, 0x3e, 0xf6, 0x0d, 0xf6, 0x59, 0x06, 0x94, + 0xcf, 0x04, 0x8b, 0x95, 0x16, 0x1c, 0x54, 0x39, 0xe9, 0x01, 0xbe, 0xdf, 0xf8, 0xd7, 0x0d, 0x0c, + 0xbf, 0x20, 0x7c, 0x3c, 0x85, 0x3c, 0x05, 0x75, 0x01, 0x19, 0x68, 0x28, 0x68, 0xde, 0x13, 0x7c, + 0x8b, 0xa5, 0x7c, 0x6e, 0x2c, 0x5a, 0xb1, 0xce, 0xc8, 0x8f, 0x9c, 0xbd, 0xa8, 0xf4, 0x1f, 0xcd, + 0x4a, 0xff, 0xe3, 0xf6, 0xa7, 0x1f, 0x01, 0x9a, 0xdc, 0x64, 0x29, 0x37, 0x3d, 0x4b, 0xa6, 0x97, + 0x8e, 0xbc, 0xff, 0xdf, 0x64, 0x7a, 0x69, 0xc9, 0xe7, 0x86, 0xac, 0x17, 0x2b, 0xc8, 0x55, 0xaf, + 0x35, 0x6c, 0x9d, 0x76, 0x46, 0xbd, 0xe8, 0xfa, 0x24, 0xd1, 0x73, 0x1a, 0x43, 0xf6, 0xc2, 0x01, + 0xc6, 0xed, 0xcd, 0xf7, 0x60, 0x6f, 0x72, 0x85, 0x0f, 0xef, 0xe0, 0x6e, 0xd5, 0x8c, 0x73, 0x39, + 0xfa, 0xd0, 0xc2, 0x37, 0x9e, 0x2e, 0x59, 0xca, 0xbd, 0x1c, 0x1f, 0xed, 0x06, 0xeb, 0x05, 0xbb, + 0xb3, 0x1b, 0x2e, 0xe1, 0x0f, 0xff, 0x0e, 0x28, 0x22, 0x0c, 0xde, 0x7f, 0xfb, 0xfd, 0x79, 0xff, + 0x5e, 0x78, 0x97, 0xbc, 0x19, 0x11, 0x6a, 0x54, 0x88, 0x56, 0xcb, 0x98, 0xa8, 0x52, 0xe3, 0x23, + 0x72, 0x47, 0xae, 0xdd, 0xc0, 0x7b, 0x54, 0x1f, 0xdd, 0x7c, 0x42, 0xff, 0xe4, 0x9f, 0xb8, 0x62, + 0x93, 0x13, 0xbb, 0xc9, 0xc3, 0x30, 0xa8, 0x6d, 0xb2, 0x30, 0xf8, 0xb9, 0xbe, 0x56, 0x7e, 0x87, + 0x8f, 0x5c, 0x42, 0x2e, 0xad, 0x6a, 0x0a, 0x0d, 0xcf, 0xa1, 0x9a, 0x42, 0x53, 0xc4, 0x57, 0xda, + 0xfd, 0x9a, 0xf6, 0xd2, 0xc2, 0xe6, 0xca, 0x72, 0xce, 0xd1, 0xd9, 0xb8, 0xb7, 0xf9, 0x35, 0xd8, + 0xdb, 0x6c, 0x07, 0xe8, 0xeb, 0x76, 0x80, 0x7e, 0x6e, 0x07, 0xe8, 0xe5, 0x81, 0x99, 0x2d, 0xe3, + 0xf8, 0xc0, 0x3e, 0x8e, 0xc7, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x6d, 0x79, 0xf1, 0x8d, + 0x03, 0x00, 0x00, } diff --git a/prompb/rpc.pb.gw.go b/prompb/rpc.pb.gw.go index 51c84721f..177ea4c10 100644 --- a/prompb/rpc.pb.gw.go +++ b/prompb/rpc.pb.gw.go @@ -1,6 +1,5 @@ -// Code generated by protoc-gen-grpc-gateway +// Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. // source: rpc.proto -// DO NOT EDIT! /* Package prompb is a reverse proxy. @@ -20,10 +19,12 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" ) var _ codes.Code var _ io.Reader +var _ status.Status var _ = runtime.String var _ = utilities.NewDoubleArray @@ -36,12 +37,21 @@ func request_Admin_TSDBSnapshot_0(ctx context.Context, marshaler runtime.Marshal } +func request_Admin_TSDBCleanTombstones_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { + var protoReq TSDBCleanTombstonesRequest + var metadata runtime.ServerMetadata + + msg, err := client.TSDBCleanTombstones(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) + return msg, metadata, err + +} + func request_Admin_DeleteSeries_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq SeriesDeleteRequest var metadata runtime.ServerMetadata if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil { - return nil, metadata, grpc.Errorf(codes.InvalidArgument, "%v", err) + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } msg, err := client.DeleteSeries(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) @@ -77,10 +87,18 @@ func RegisterAdminHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux // RegisterAdminHandler registers the http handlers for service Admin to "mux". // The handlers forward requests to the grpc endpoint over "conn". func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { - client := NewAdminClient(conn) + return RegisterAdminHandlerClient(ctx, mux, NewAdminClient(conn)) +} + +// RegisterAdminHandler registers the http handlers for service Admin to "mux". +// The handlers forward requests to the grpc endpoint over the given implementation of "AdminClient". +// Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "AdminClient" +// doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in +// "AdminClient" to call the correct interceptors. +func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, client AdminClient) error { mux.Handle("POST", pattern_Admin_TSDBSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(ctx) + ctx, cancel := context.WithCancel(req.Context()) defer cancel() if cn, ok := w.(http.CloseNotifier); ok { go func(done <-chan struct{}, closed <-chan bool) { @@ -92,23 +110,24 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc }(ctx.Done(), cn.CloseNotify()) } inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) - rctx, err := runtime.AnnotateContext(ctx, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) if err != nil { - runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return } resp, md, err := request_Admin_TSDBSnapshot_0(rctx, inboundMarshaler, client, req, pathParams) ctx = runtime.NewServerMetadataContext(ctx, md) if err != nil { - runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } - forward_Admin_TSDBSnapshot_0(ctx, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + forward_Admin_TSDBSnapshot_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) - mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(ctx) + mux.Handle("POST", pattern_Admin_TSDBCleanTombstones_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) defer cancel() if cn, ok := w.(http.CloseNotifier); ok { go func(done <-chan struct{}, closed <-chan bool) { @@ -120,18 +139,48 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc }(ctx.Done(), cn.CloseNotify()) } inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) - rctx, err := runtime.AnnotateContext(ctx, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) if err != nil { - runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + resp, md, err := request_Admin_TSDBCleanTombstones_0(rctx, inboundMarshaler, client, req, pathParams) + ctx = runtime.NewServerMetadataContext(ctx, md) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return + } + + forward_Admin_TSDBCleanTombstones_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + + }) + + mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { + ctx, cancel := context.WithCancel(req.Context()) + defer cancel() + if cn, ok := w.(http.CloseNotifier); ok { + go func(done <-chan struct{}, closed <-chan bool) { + select { + case <-done: + case <-closed: + cancel() + } + }(ctx.Done(), cn.CloseNotify()) + } + inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) + rctx, err := runtime.AnnotateContext(ctx, mux, req) + if err != nil { + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) + return } resp, md, err := request_Admin_DeleteSeries_0(rctx, inboundMarshaler, client, req, pathParams) ctx = runtime.NewServerMetadataContext(ctx, md) if err != nil { - runtime.HTTPError(ctx, outboundMarshaler, w, req, err) + runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) return } - forward_Admin_DeleteSeries_0(ctx, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) + forward_Admin_DeleteSeries_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) }) @@ -141,11 +190,15 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc var ( pattern_Admin_TSDBSnapshot_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v2", "admin", "tsdb", "snapshot"}, "")) + pattern_Admin_TSDBCleanTombstones_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v2", "admin", "tsdb", "clean_tombstones"}, "")) + pattern_Admin_DeleteSeries_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 2, 3}, []string{"v2", "admin", "tsdb", "delete_series"}, "")) ) var ( forward_Admin_TSDBSnapshot_0 = runtime.ForwardResponseMessage + forward_Admin_TSDBCleanTombstones_0 = runtime.ForwardResponseMessage + forward_Admin_DeleteSeries_0 = runtime.ForwardResponseMessage ) diff --git a/prompb/rpc.proto b/prompb/rpc.proto index ec72430f0..a6463fbfb 100644 --- a/prompb/rpc.proto +++ b/prompb/rpc.proto @@ -28,14 +28,20 @@ option (gogoproto.goproto_getters_all) = false; service Admin { - // Snapshot creates a snapshot of all current data into 'snapshots/-' under - // the TSDB's data directory. + // Snapshot creates a snapshot of all current data into 'snapshots/-' under the TSDB's data directory. rpc TSDBSnapshot(TSDBSnapshotRequest) returns (TSDBSnapshotResponse) { option (google.api.http) = { post: "/v2/admin/tsdb/snapshot" }; } + // CleanTombstones removes the deleted data from disk and cleans up the existing tombstones. + rpc TSDBCleanTombstones(TSDBCleanTombstonesRequest) returns (TSDBCleanTombstonesResponse) { + option (google.api.http) = { + post: "/v2/admin/tsdb/clean_tombstones" + }; + } + // DeleteSeries deletes data for a selection of series in a time range. rpc DeleteSeries(SeriesDeleteRequest) returns (SeriesDeleteResponse) { option (google.api.http) = { @@ -53,6 +59,12 @@ message TSDBSnapshotResponse { string name = 1; } +message TSDBCleanTombstonesRequest { +} + +message TSDBCleanTombstonesResponse { +} + message SeriesDeleteRequest { google.protobuf.Timestamp min_time = 1 [(gogoproto.stdtime) = true]; google.protobuf.Timestamp max_time = 2 [(gogoproto.stdtime) = true]; diff --git a/prompb/types.pb.go b/prompb/types.pb.go index 35ebf008b..3edff74f9 100644 --- a/prompb/types.pb.go +++ b/prompb/types.pb.go @@ -6,8 +6,7 @@ package prompb import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" - -import encoding_binary "encoding/binary" +import _ "github.com/gogo/protobuf/gogoproto" import io "io" @@ -190,8 +189,7 @@ func (m *Sample) MarshalTo(dAtA []byte) (int, error) { if m.Value != 0 { dAtA[i] = 0x9 i++ - encoding_binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value)))) - i += 8 + i = encodeFixed64Types(dAtA, i, uint64(math.Float64bits(float64(m.Value)))) } if m.Timestamp != 0 { dAtA[i] = 0x10 @@ -338,6 +336,24 @@ func (m *LabelMatcher) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func encodeFixed64Types(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Types(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -470,8 +486,15 @@ func (m *Sample) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 + v = uint64(dAtA[iNdEx-8]) + v |= uint64(dAtA[iNdEx-7]) << 8 + v |= uint64(dAtA[iNdEx-6]) << 16 + v |= uint64(dAtA[iNdEx-5]) << 24 + v |= uint64(dAtA[iNdEx-4]) << 32 + v |= uint64(dAtA[iNdEx-3]) << 40 + v |= uint64(dAtA[iNdEx-2]) << 48 + v |= uint64(dAtA[iNdEx-1]) << 56 m.Value = float64(math.Float64frombits(v)) case 2: if wireType != 0 { diff --git a/scripts/genproto.sh b/scripts/genproto.sh index 190118af7..846f23c1e 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -10,8 +10,8 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then exit 255 fi -if ! [[ $(protoc --version) =~ "3.2.0" ]]; then - echo "could not find protoc 3.2.0, is it installed + in PATH?" +if ! [[ $(protoc --version) =~ "3.4" ]]; then + echo "could not find protoc 3.4.x, is it installed + in PATH?" exit 255 fi diff --git a/web/api/v2/api.go b/web/api/v2/api.go index 415f5746e..508bdc156 100644 --- a/web/api/v2/api.go +++ b/web/api/v2/api.go @@ -144,6 +144,11 @@ func (s *adminDisabled) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotReques return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") } +// TSDBCleanTombstones implements pb.AdminServer. +func (s *adminDisabled) TSDBCleanTombstones(_ old_ctx.Context, _ *pb.TSDBCleanTombstonesRequest) (*pb.TSDBCleanTombstonesResponse, error) { + return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") +} + // DeleteSeries imeplements pb.AdminServer. func (s *adminDisabled) DeleteSeries(_ old_ctx.Context, r *pb.SeriesDeleteRequest) (*pb.SeriesDeleteResponse, error) { return nil, status.Error(codes.Unavailable, "Admin APIs are disabled") @@ -181,6 +186,20 @@ func (s *Admin) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb. return &pb.TSDBSnapshotResponse{Name: name}, nil } +// TSDBCleanTombstones implements pb.AdminServer. +func (s *Admin) TSDBCleanTombstones(_ old_ctx.Context, _ *pb.TSDBCleanTombstonesRequest) (*pb.TSDBCleanTombstonesResponse, error) { + db := s.db() + if db == nil { + return nil, status.Errorf(codes.Unavailable, "TSDB not ready") + } + + if err := db.CleanTombstones(); err != nil { + return nil, status.Errorf(codes.Internal, "clean tombstones: %s", err) + } + + return &pb.TSDBCleanTombstonesResponse{}, nil +} + // DeleteSeries implements pb.AdminServer. func (s *Admin) DeleteSeries(_ old_ctx.Context, r *pb.SeriesDeleteRequest) (*pb.SeriesDeleteResponse, error) { mint, maxt, err := extractTimeRange(r.MinTime, r.MaxTime)