diff --git a/model/histogram/float_histogram.go b/model/histogram/float_histogram.go
index b941bfd08..beddf8a3e 100644
--- a/model/histogram/float_histogram.go
+++ b/model/histogram/float_histogram.go
@@ -305,14 +305,164 @@ func addBucket(
 
 // Compact eliminates empty buckets at the beginning and end of each span, then
 // merges spans that are consecutive or at most maxEmptyBuckets apart, and
-// finally splits spans that contain more than maxEmptyBuckets. The compaction
-// happens "in place" in the receiving histogram, but a pointer to it is
-// returned for convenience.
+// finally splits spans that contain more consecutive empty buckets than
+// maxEmptyBuckets. (The actual implementation might do something more efficient
+// but with the same result.)  The compaction happens "in place" in the
+// receiving histogram, but a pointer to it is returned for convenience.
 func (h *FloatHistogram) Compact(maxEmptyBuckets int) *FloatHistogram {
-	// TODO(beorn7): Implement.
+	h.PositiveBuckets, h.PositiveSpans = compactBuckets(
+		h.PositiveBuckets, h.PositiveSpans, maxEmptyBuckets,
+	)
+	h.NegativeBuckets, h.NegativeSpans = compactBuckets(
+		h.NegativeBuckets, h.NegativeSpans, maxEmptyBuckets,
+	)
 	return h
 }
 
+func compactBuckets(buckets []float64, spans []Span, maxEmptyBuckets int) ([]float64, []Span) {
+	if len(buckets) == 0 {
+		return buckets, spans
+	}
+
+	var iBucket, iSpan int
+	var posInSpan uint32
+
+	// Helper function.
+	emptyBucketsHere := func() int {
+		i := 0
+		for i+iBucket < len(buckets) &&
+			uint32(i)+posInSpan < spans[iSpan].Length &&
+			buckets[i+iBucket] == 0 {
+			i++
+		}
+		return i
+	}
+
+	// Merge spans with zero-offset to avoid special cases later.
+	if len(spans) > 1 {
+		for i, span := range spans[1:] {
+			if span.Offset == 0 {
+				spans[iSpan].Length += span.Length
+				continue
+			}
+			iSpan++
+			if i+1 != iSpan {
+				spans[iSpan] = span
+			}
+		}
+		spans = spans[:iSpan+1]
+		iSpan = 0
+	}
+
+	// Merge spans with zero-length to avoid special cases later.
+	for i, span := range spans {
+		if span.Length == 0 {
+			if i+1 < len(spans) {
+				spans[i+1].Offset += span.Offset
+			}
+			continue
+		}
+		if i != iSpan {
+			spans[iSpan] = span
+		}
+		iSpan++
+	}
+	spans = spans[:iSpan]
+	iSpan = 0
+
+	// Cut out empty buckets from start and end of spans, no matter
+	// what. Also cut out empty buckets from the middle of a span but only
+	// if there are more than maxEmptyBuckets consecutive empty buckets.
+	for iBucket < len(buckets) {
+		if nEmpty := emptyBucketsHere(); nEmpty > 0 {
+			if posInSpan > 0 &&
+				nEmpty < int(spans[iSpan].Length-posInSpan) &&
+				nEmpty <= maxEmptyBuckets {
+				// The empty buckets are in the middle of a
+				// span, and there are few enough to not bother.
+				// Just fast-forward.
+				iBucket += nEmpty
+				posInSpan += uint32(nEmpty)
+				continue
+			}
+			// In all other cases, we cut out the empty buckets.
+			buckets = append(buckets[:iBucket], buckets[iBucket+nEmpty:]...)
+			if posInSpan == 0 {
+				// Start of span.
+				if nEmpty == int(spans[iSpan].Length) {
+					// The whole span is empty.
+					spans = append(spans[:iSpan], spans[iSpan+1:]...)
+					continue
+				}
+				spans[iSpan].Length -= uint32(nEmpty)
+				spans[iSpan].Offset += int32(nEmpty)
+				continue
+			}
+			// It's in the middle or in the end of the span.
+			// Split the current span.
+			newSpan := Span{
+				Offset: int32(nEmpty),
+				Length: spans[iSpan].Length - posInSpan - uint32(nEmpty),
+			}
+			spans[iSpan].Length = posInSpan
+			// In any case, we have to split to the next span.
+			iSpan++
+			posInSpan = 0
+			if newSpan.Length == 0 {
+				// The span is empty, so we were already at the end of a span.
+				// We don't have to insert the new span, just adjust the next
+				// span's offset, if there is one.
+				if iSpan < len(spans) {
+					spans[iSpan].Offset += int32(nEmpty)
+				}
+				continue
+			}
+			// Insert the new span.
+			spans = append(spans, Span{})
+			if iSpan+1 < len(spans) {
+				copy(spans[iSpan+1:], spans[iSpan:])
+			}
+			spans[iSpan] = newSpan
+			continue
+		}
+		iBucket++
+		posInSpan++
+		if posInSpan >= spans[iSpan].Length {
+			posInSpan = 0
+			iSpan++
+		}
+	}
+	if maxEmptyBuckets == 0 {
+		return buckets, spans
+	}
+
+	// Finally, check if any offsets between spans are small enough to merge
+	// the spans.
+	iBucket = int(spans[0].Length)
+	iSpan = 1
+	for iSpan < len(spans) {
+		if int(spans[iSpan].Offset) > maxEmptyBuckets {
+			iBucket += int(spans[iSpan].Length)
+			iSpan++
+			continue
+		}
+		// Merge span with previous one and insert empty buckets.
+		offset := int(spans[iSpan].Offset)
+		spans[iSpan-1].Length += uint32(offset) + spans[iSpan].Length
+		spans = append(spans[:iSpan], spans[iSpan+1:]...)
+		newBuckets := make([]float64, len(buckets)+offset)
+		copy(newBuckets, buckets[:iBucket])
+		copy(newBuckets[iBucket+offset:], buckets[iBucket:])
+		iBucket += offset
+		buckets = newBuckets
+		// Note that with many merges, it would be more efficient to
+		// first record all the chunks of empty buckets to insert and
+		// then do it in one go through all the buckets.
+	}
+
+	return buckets, spans
+}
+
 // DetectReset returns true if the receiving histogram is missing any buckets
 // that have a non-zero population in the provided previous histogram. It also
 // returns true if any count (in any bucket, in the zero count, or in the count
diff --git a/model/histogram/float_histogram_test.go b/model/histogram/float_histogram_test.go
index 6d43ea429..39f428f1f 100644
--- a/model/histogram/float_histogram_test.go
+++ b/model/histogram/float_histogram_test.go
@@ -495,7 +495,200 @@ func TestFloatHistogramCompact(t *testing.T) {
 		maxEmptyBuckets int
 		expected        *FloatHistogram
 	}{
-		// TODO(beorn7): Add test cases.
+		{
+			"empty histogram",
+			&FloatHistogram{},
+			0,
+			&FloatHistogram{},
+		},
+		{
+			"nothing should happen",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}, {2, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
+				NegativeSpans:   []Span{{3, 2}, {3, 2}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}, {2, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1},
+				NegativeSpans:   []Span{{3, 2}, {3, 2}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000},
+			},
+		},
+		{
+			"eliminate zero offsets",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}, {0, 3}, {0, 1}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {0, 2}, {2, 1}, {0, 1}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 5}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 4}, {2, 2}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"eliminate zero length",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {2, 0}, {3, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {0, 0}, {2, 0}, {1, 4}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 4}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"eliminate multiple zero length spans",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {2, 0}, {2, 0}, {2, 0}, {3, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {9, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+			},
+		},
+		{
+			"cut empty buckets at start or end",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 4}, {5, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4, 0},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 4}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"cut empty buckets at start and end",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 4}, {5, 6}},
+				PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3, 0, 0, 0},
+				NegativeSpans:   []Span{{-2, 4}, {3, 5}},
+				NegativeBuckets: []float64{0, 0, 3.1, 3, 1.234e5, 1000, 3, 4, 0},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 4}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"cut empty buckets at start or end of chunks, even in the middle",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 6}, {3, 6}},
+				PositiveBuckets: []float64{0, 0, 1, 3.3, 0, 0, 4.2, 0.1, 3.3, 0, 0, 0},
+				NegativeSpans:   []Span{{0, 2}, {2, 6}},
+				NegativeBuckets: []float64{3.1, 3, 0, 1.234e5, 1000, 3, 4, 0},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 2}, {5, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 4}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"cut empty buckets at start or end but merge spans due to maxEmptyBuckets",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 4}, {5, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4, 0},
+			},
+			10,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 10}},
+				PositiveBuckets: []float64{1, 3.3, 0, 0, 0, 0, 0, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 9}},
+				NegativeBuckets: []float64{3.1, 3, 0, 0, 0, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"cut empty buckets from the middle of a chunk",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 6}, {3, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+			0,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}, {2, 1}, {3, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 2}, {1, 2}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 3, 4},
+			},
+		},
+		{
+			"cut empty buckets from the middle of a chunk, avoiding some due to maxEmptyBuckets",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 6}, {3, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+			1,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 1}, {2, 1}, {3, 3}},
+				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+		},
+		{
+			"avoiding all cutting of empty buckets from the middle of a chunk due to maxEmptyBuckets",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 6}, {3, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+			2,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 4}, {3, 3}},
+				PositiveBuckets: []float64{1, 0, 0, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+		},
+		{
+			"everything merged into one span due to maxEmptyBuckets",
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-4, 6}, {3, 3}},
+				PositiveBuckets: []float64{0, 0, 1, 0, 0, 3.3, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 2}, {3, 5}},
+				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000, 0, 3, 4},
+			},
+			3,
+			&FloatHistogram{
+				PositiveSpans:   []Span{{-2, 10}},
+				PositiveBuckets: []float64{1, 0, 0, 3.3, 0, 0, 0, 4.2, 0.1, 3.3},
+				NegativeSpans:   []Span{{0, 10}},
+				NegativeBuckets: []float64{3.1, 3, 0, 0, 0, 1.234e5, 1000, 0, 3, 4},
+			},
+		},
 	}
 
 	for _, c := range cases {
diff --git a/promql/engine_test.go b/promql/engine_test.go
index 02baca9b4..0e642acc3 100644
--- a/promql/engine_test.go
+++ b/promql/engine_test.go
@@ -2650,16 +2650,13 @@ func TestSparseHistogramRate(t *testing.T) {
 	require.Len(t, vector, 1)
 	actualHistogram := vector[0].H
 	expectedHistogram := &histogram.FloatHistogram{
-		Schema:        1,
-		ZeroThreshold: 0.001,
-		ZeroCount:     1. / 15.,
-		Count:         4. / 15.,
-		Sum:           1.226666666666667,
-		PositiveSpans: []histogram.Span{
-			{Offset: 0, Length: 2},
-			{Offset: 1, Length: 2},
-		},
-		PositiveBuckets: []float64{1. / 15., 1. / 15., 1. / 15., 1. / 15.},
+		Schema:          1,
+		ZeroThreshold:   0.001,
+		ZeroCount:       1. / 15.,
+		Count:           4. / 15.,
+		Sum:             1.226666666666667,
+		PositiveSpans:   []histogram.Span{{Offset: 0, Length: 5}},
+		PositiveBuckets: []float64{1. / 15., 1. / 15., 0, 1. / 15., 1. / 15.},
 	}
 	require.Equal(t, expectedHistogram, actualHistogram)
 }