RepoMirrors
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #273 from Gouthamve/large-idx-reader 

Fix reader for large index files.
This commit is contained in:
Fabian Reinartz 2018-02-15 12:29:47 +01:00 committed by GitHub
parent cc673dc167 bb0e74b343
commit 494acd3070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
index/index.go
View File

@ -144,7 +144,11 @@ func NewWriter(fn string) (*Writer, error) {
if err != nil {
return nil, err
}
defer df.Close() // close for flatform windows
defer df.Close() // Close for platform windows.
if err := os.RemoveAll(fn); err != nil {
return nil, errors.Wrap(err, "remove any existing index at path")
}
f, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
@ -535,8 +539,8 @@ type Reader struct {
c io.Closer
// Cached hashmaps of section offsets.
labels map[string]uint32
postings map[labels.Label]uint32
labels map[string]uint64
postings map[labels.Label]uint64
// Cache of read symbols. Strings that are returned when reading from the
// block are always backed by true strings held in here rather than
// strings that are backed by byte slices from the mmap'd index file. This
@ -597,8 +601,8 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
b: b,
c: c,
symbols: map[uint32]string{},
labels: map[string]uint32{},
postings: map[labels.Label]uint32{},
labels: map[string]uint64{},
postings: map[labels.Label]uint64{},
crc32: newCRC32(),
}
@ -623,7 +627,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
}
var err error
err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint32) error {
err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint64) error {
if len(key) != 1 {
return errors.Errorf("unexpected key length %d", len(key))
}
@ -633,7 +637,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
if err != nil {
return nil, errors.Wrap(err, "read label index table")
}
err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint32) error {
err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint64) error {
if len(key) != 2 {
return errors.Errorf("unexpected key length %d", len(key))
}
@ -786,7 +790,7 @@ func (r *Reader) readSymbols(off int) error {
// readOffsetTable reads an offset table at the given position calls f for each
// found entry.f
// If f returns an error it stops decoding and returns the received error,
func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) error {
func (r *Reader) readOffsetTable(off uint64, f func([]string, uint64) error) error {
d := r.decbufAt(int(off))
cnt := d.be32()
@ -797,7 +801,7 @@ func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) err
for i := 0; i < keyCount; i++ {
keys = append(keys, d.uvarintStr())
}
o := uint32(d.uvarint())
o := d.uvarint64()
if d.err() != nil {
break
}