RepoMirrors
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #273 from Gouthamve/large-idx-reader 

Fix reader for large index files.
This commit is contained in:
Fabian Reinartz 2018-02-15 12:29:47 +01:00 committed by GitHub
parent cc673dc167 bb0e74b343
commit 494acd3070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 13 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
index/index.go
View File

@ -144,7 +144,11 @@ func NewWriter(fn string) (*Writer, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer df.Close() // close for flatform windows defer df.Close() // Close for platform windows.
if err := os.RemoveAll(fn); err != nil {
return nil, errors.Wrap(err, "remove any existing index at path")
}
f, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, 0666) f, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, 0666)
if err != nil { if err != nil {
@ -535,8 +539,8 @@ type Reader struct {
c io.Closer c io.Closer
// Cached hashmaps of section offsets. // Cached hashmaps of section offsets.
labels map[string]uint32 labels map[string]uint64
postings map[labels.Label]uint32 postings map[labels.Label]uint64
// Cache of read symbols. Strings that are returned when reading from the // Cache of read symbols. Strings that are returned when reading from the
// block are always backed by true strings held in here rather than // block are always backed by true strings held in here rather than
// strings that are backed by byte slices from the mmap'd index file. This // strings that are backed by byte slices from the mmap'd index file. This
@ -597,8 +601,8 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
b: b, b: b,
c: c, c: c,
symbols: map[uint32]string{}, symbols: map[uint32]string{},
labels: map[string]uint32{}, labels: map[string]uint64{},
postings: map[labels.Label]uint32{}, postings: map[labels.Label]uint64{},
crc32: newCRC32(), crc32: newCRC32(),
} }
@ -623,7 +627,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
} }
var err error var err error
err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint32) error { err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint64) error {
if len(key) != 1 { if len(key) != 1 {
return errors.Errorf("unexpected key length %d", len(key)) return errors.Errorf("unexpected key length %d", len(key))
} }
@ -633,7 +637,7 @@ func newReader(b ByteSlice, c io.Closer) (*Reader, error) {
if err != nil { if err != nil {
return nil, errors.Wrap(err, "read label index table") return nil, errors.Wrap(err, "read label index table")
} }
err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint32) error { err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint64) error {
if len(key) != 2 { if len(key) != 2 {
return errors.Errorf("unexpected key length %d", len(key)) return errors.Errorf("unexpected key length %d", len(key))
} }
@ -786,7 +790,7 @@ func (r *Reader) readSymbols(off int) error {
// readOffsetTable reads an offset table at the given position calls f for each // readOffsetTable reads an offset table at the given position calls f for each
// found entry.f // found entry.f
// If f returns an error it stops decoding and returns the received error, // If f returns an error it stops decoding and returns the received error,
func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) error { func (r *Reader) readOffsetTable(off uint64, f func([]string, uint64) error) error {
d := r.decbufAt(int(off)) d := r.decbufAt(int(off))
cnt := d.be32() cnt := d.be32()
@ -797,7 +801,7 @@ func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) err
for i := 0; i < keyCount; i++ { for i := 0; i < keyCount; i++ {
keys = append(keys, d.uvarintStr()) keys = append(keys, d.uvarintStr())
} }
o := uint32(d.uvarint()) o := d.uvarint64()
if d.err() != nil { if d.err() != nil {
break break
} }