From c3434a39b761cff8bb6e444fecaf312c5df2bd68 Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Wed, 5 Sep 2018 20:04:44 +0100 Subject: [PATCH] proto: Allow reproducible proto generation. Current way of setting prometheus project to generate repo is tedious. If you just go install packages described in README.md it ends up with those (non buildable code) ``` ../../../prompb/remote.pb.go:61:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:109:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:164:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:214:27: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:284:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ``` The issue is that the guide presented in proto folder is asking to install some GO packages. It does not mention from what version and there is strong mismatch between those used to generate proto and those that are vendored and used to run the code later... The particular outdated packages in vendor are: `github.com/gogo/protobuf` and `google.golang.org/grpc`. Just updating vendor would be not enough, as it would only tmp fix. I propose adding script that will parse version from vendor.json and install correct version. Changes: - Added script that uses packages in *exactly* the same version the Go code is build against later. - Regenerated proto with those pinned deps. - Updated REAMDE.md - Added `make proto`Current way of setting prometheus project to generate code from proto files is tedious. If you just go install packages described in README.md, it ends up with this (non buildable code) ``` ../../../prompb/remote.pb.go:61:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:109:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:164:34: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:214:27: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ../../../prompb/remote.pb.go:284:33: undefined: "github.com/prometheus/prometheus/vendor/github.com/gogo/protobuf/proto".InternalMessageInfo ``` The issue is that the guide presented in proto folder is asking to install some GO packages. It does not mention from what version and there is strong mismatch between those used to generate proto and those that are vendored and used to run the code later... The particular outdated packages in vendor are: `github.com/gogo/protobuf` and `google.golang.org/grpc`. Just updating vendor would be not enough, as it would only tmp fix. I propose adding script that will parse version from vendor.json and install correct version. Changes: - Added script that uses packages in *exactly* the same version the Go code is build against later. - Regenerated proto with those pinned deps. - Updated REAMDE.md - Added `make proto` Signed-off-by: Bartek Plotka --- Makefile.common | 5 +++++ prompb/README.md | 11 +++-------- prompb/remote.pb.go | 18 ------------------ prompb/rpc.pb.go | 42 +++++++++++------------------------------- prompb/rpc.pb.gw.go | 2 +- prompb/types.pb.go | 32 +++++--------------------------- scripts/genproto.sh | 26 +++++++++++++++++++++++--- 7 files changed, 48 insertions(+), 88 deletions(-) diff --git a/Makefile.common b/Makefile.common index c9d832373..f19ed6389 100644 --- a/Makefile.common +++ b/Makefile.common @@ -123,6 +123,11 @@ common-docker-tag-latest: promu: GOOS= GOARCH= $(GO) get -u github.com/prometheus/promu +.PHONY: proto +proto: + @echo ">> generating code from proto files" + @./scripts/genproto.sh + .PHONY: $(STATICCHECK) $(STATICCHECK): GOOS= GOARCH= $(GO) get -u honnef.co/go/tools/cmd/staticcheck diff --git a/prompb/README.md b/prompb/README.md index d2aa933ef..7f2439166 100644 --- a/prompb/README.md +++ b/prompb/README.md @@ -2,13 +2,8 @@ The compiled protobufs are version controlled and you won't normally need to re-compile them when building Prometheus. If however you have modified the defs and do need to re-compile, run -`./scripts/genproto.sh` from the parent dir. +`make proto` from the parent dir. -In order for the script to run, you'll need `protoc` (version 3.5) in your -PATH, and the following Go packages installed: +In order for the script to run, you'll need `protoc` (version 3.5.1) in your +PATH. -- github.com/gogo/protobuf -- github.com/gogo/protobuf/protoc-gen-gogofast -- github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/ -- github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger -- golang.org/x/tools/cmd/goimports diff --git a/prompb/remote.pb.go b/prompb/remote.pb.go index 08ceed01e..5ed2d06c9 100644 --- a/prompb/remote.pb.go +++ b/prompb/remote.pb.go @@ -330,24 +330,6 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Remote(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Remote(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintRemote(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) diff --git a/prompb/rpc.pb.go b/prompb/rpc.pb.go index 4da57ee9d..59291b66a 100644 --- a/prompb/rpc.pb.go +++ b/prompb/rpc.pb.go @@ -11,12 +11,10 @@ import _ "google.golang.org/genproto/googleapis/api/annotations" import time "time" -import ( - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" -) +import context "golang.org/x/net/context" +import grpc "google.golang.org/grpc" -import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" +import types "github.com/gogo/protobuf/types" import io "io" @@ -338,8 +336,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) { if m.MinTime != nil { dAtA[i] = 0xa i++ - i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime))) - n1, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MinTime, dAtA[i:]) + i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MinTime))) + n1, err := types.StdTimeMarshalTo(*m.MinTime, dAtA[i:]) if err != nil { return 0, err } @@ -348,8 +346,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) { if m.MaxTime != nil { dAtA[i] = 0x12 i++ - i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime))) - n2, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:]) + i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MaxTime))) + n2, err := types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:]) if err != nil { return 0, err } @@ -388,24 +386,6 @@ func (m *SeriesDeleteResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Rpc(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Rpc(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -450,11 +430,11 @@ func (m *SeriesDeleteRequest) Size() (n int) { var l int _ = l if m.MinTime != nil { - l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime) + l = types.SizeOfStdTime(*m.MinTime) n += 1 + l + sovRpc(uint64(l)) } if m.MaxTime != nil { - l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime) + l = types.SizeOfStdTime(*m.MaxTime) n += 1 + l + sovRpc(uint64(l)) } if len(m.Matchers) > 0 { @@ -792,7 +772,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error { if m.MinTime == nil { m.MinTime = new(time.Time) } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil { + if err := types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -825,7 +805,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error { if m.MaxTime == nil { m.MaxTime = new(time.Time) } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil { + if err := types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/prompb/rpc.pb.gw.go b/prompb/rpc.pb.gw.go index fd1526d83..8ea7d1b67 100644 --- a/prompb/rpc.pb.gw.go +++ b/prompb/rpc.pb.gw.go @@ -58,7 +58,7 @@ func request_Admin_DeleteSeries_0(ctx context.Context, marshaler runtime.Marshal var protoReq SeriesDeleteRequest var metadata runtime.ServerMetadata - if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil && err != io.EOF { + if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) } diff --git a/prompb/types.pb.go b/prompb/types.pb.go index 75b017879..1280cfe0b 100644 --- a/prompb/types.pb.go +++ b/prompb/types.pb.go @@ -7,6 +7,8 @@ import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" +import binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -229,7 +231,8 @@ func (m *Sample) MarshalTo(dAtA []byte) (int, error) { if m.Value != 0 { dAtA[i] = 0x9 i++ - i = encodeFixed64Types(dAtA, i, uint64(math.Float64bits(float64(m.Value)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value)))) + i += 8 } if m.Timestamp != 0 { dAtA[i] = 0x10 @@ -415,24 +418,6 @@ func (m *ReadHints) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Types(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Types(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -584,15 +569,8 @@ func (m *Sample) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.Value = float64(math.Float64frombits(v)) case 2: if wireType != 0 { diff --git a/scripts/genproto.sh b/scripts/genproto.sh index a46eedc76..8d5c1e51d 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -10,11 +10,31 @@ if ! [[ "$0" =~ "scripts/genproto.sh" ]]; then exit 255 fi -if ! [[ $(protoc --version) =~ "3.5" ]]; then - echo "could not find protoc 3.5.x, is it installed + in PATH?" +if ! [[ $(protoc --version) =~ "3.5.1" ]]; then + echo "could not find protoc 3.5.1, is it installed + in PATH?" exit 255 fi +# Exact version of plugins to build. +PROTOC_GEN_GOGOFAST_SHA="971cbfd2e72b513a28c74af7462aee0800248d69" +PROTOC_GEN_GRPC_ECOSYSTEM_SHA="e4b8a938efae14de11fd97311e873e989896348c" + +echo "installing plugins" +go install "golang.org/x/tools/cmd/goimports" +go get -d -u "github.com/gogo/protobuf/protoc-gen-gogo" +pushd ${GOPATH}/src/github.com/gogo/protobuf + git reset --hard "${PROTOC_GEN_GOGOFAST_SHA}" + go install "github.com/gogo/protobuf/protoc-gen-gogofast" +popd + +go get -d -u "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway" +go get -d -u "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger" +pushd ${GOPATH}/src/github.com/grpc-ecosystem/grpc-gateway + git reset --hard "${PROTOC_GEN_GRPC_ECOSYSTEM_SHA}" + go install "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway" + go install "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger" +popd + PROM_ROOT="${GOPATH}/src/github.com/prometheus/prometheus" PROM_PATH="${PROM_ROOT}/prompb" GOGOPROTO_ROOT="${GOPATH}/src/github.com/gogo/protobuf" @@ -40,7 +60,7 @@ for dir in ${DIRS}; do rpc.proto mv ../documentation/dev/api/rpc.swagger.json ../documentation/dev/api/swagger.json - sed -i.bak -E 's/import _ \"gogoproto\"//g' *.pb.go + sed -i.bak -E 's/import _ \"github.com\/gogo\/protobuf\/gogoproto\"//g' *.pb.go sed -i.bak -E 's/import _ \"google\/protobuf\"//g' *.pb.go sed -i.bak -E 's/golang\/protobuf/gogo\/protobuf/g' *.go rm -f *.bak