Implementation of HFSC

Testing and functionality for the use of HFSC has been implemented.
The use of service curves is implenented closely as to how they behave
with the TC implementation.
Automated checks and testing were succesful.
This commit is contained in:
Francis Begyn 2018-04-09 22:40:06 +02:00 committed by Alessandro Boch
parent d85e18ed5b
commit 1006cf4f24
7 changed files with 467 additions and 10 deletions

View File

@ -9,5 +9,6 @@ before_script:
- sudo modprobe nf_conntrack_netlink
- sudo modprobe nf_conntrack_ipv4
- sudo modprobe nf_conntrack_ipv6
- sudo modprobe sch_hfsc
install:
- go get github.com/vishvananda/netns

View File

@ -4,6 +4,7 @@ import (
"fmt"
)
// Class interfaces for all classes
type Class interface {
Attrs() *ClassAttrs
Type() string
@ -13,25 +14,25 @@ type Class interface {
// This file contains "gnet_" prefixed structs and relevant functions.
// See Documentation/networking/getn_stats.txt in Linux source code for more details.
// Ref: struct gnet_stats_basic { ... }
// GnetStatsBasic Ref: struct gnet_stats_basic { ... }
type GnetStatsBasic struct {
Bytes uint64 // number of seen bytes
Packets uint32 // number of seen packets
}
// Ref: struct gnet_stats_rate_est { ... }
// GnetStatsRateEst Ref: struct gnet_stats_rate_est { ... }
type GnetStatsRateEst struct {
Bps uint32 // current byte rate
Pps uint32 // current packet rate
}
// Ref: struct gnet_stats_rate_est64 { ... }
// GnetStatsRateEst64 Ref: struct gnet_stats_rate_est64 { ... }
type GnetStatsRateEst64 struct {
Bps uint64 // current byte rate
Pps uint64 // current packet rate
}
// Ref: struct gnet_stats_queue { ... }
// GnetStatsQueue Ref: struct gnet_stats_queue { ... }
type GnetStatsQueue struct {
Qlen uint32 // queue length
Backlog uint32 // backlog size of queue
@ -40,7 +41,7 @@ type GnetStatsQueue struct {
Overlimits uint32 // number of enqueues over the limit
}
// Statistics representaion based on generic networking statisticsfor netlink.
// ClassStatistics representaion based on generic networking statisticsfor netlink.
// See Documentation/networking/gen_stats.txt in Linux source code for more details.
type ClassStatistics struct {
Basic *GnetStatsBasic
@ -48,7 +49,7 @@ type ClassStatistics struct {
RateEst *GnetStatsRateEst
}
// Construct a ClassStatistics struct which fields are all initialized by 0.
// NewClassStatistics Construct a ClassStatistics struct which fields are all initialized by 0.
func NewClassStatistics() *ClassStatistics {
return &ClassStatistics{
Basic: &GnetStatsBasic{},
@ -72,6 +73,7 @@ func (q ClassAttrs) String() string {
return fmt.Sprintf("{LinkIndex: %d, Handle: %s, Parent: %s, Leaf: %d}", q.LinkIndex, HandleStr(q.Handle), HandleStr(q.Parent), q.Leaf)
}
// HtbClassAttrs stores the attributes of HTB class
type HtbClassAttrs struct {
// TODO handle all attributes
Rate uint64
@ -103,10 +105,12 @@ func (q HtbClass) String() string {
return fmt.Sprintf("{Rate: %d, Ceil: %d, Buffer: %d, Cbuffer: %d}", q.Rate, q.Ceil, q.Buffer, q.Cbuffer)
}
// Attrs returns the class attributes
func (q *HtbClass) Attrs() *ClassAttrs {
return &q.ClassAttrs
}
// Type return the class type
func (q *HtbClass) Type() string {
return "htb"
}
@ -118,10 +122,90 @@ type GenericClass struct {
ClassType string
}
// Attrs return the class attributes
func (class *GenericClass) Attrs() *ClassAttrs {
return &class.ClassAttrs
}
// Type retrun the class type
func (class *GenericClass) Type() string {
return class.ClassType
}
// ServiceCurve is the way the HFSC curve are represented
type ServiceCurve struct {
m1 uint32
d uint32
m2 uint32
}
// Attrs return the parameters of the service curve
func (c *ServiceCurve) Attrs() (uint32, uint32, uint32) {
return c.m1, c.d, c.m2
}
// HfscClass is a representation of the HFSC class
type HfscClass struct {
ClassAttrs
Rsc ServiceCurve
Fsc ServiceCurve
Usc ServiceCurve
}
// SetUsc sets the Usc curve
func (hfsc *HfscClass) SetUsc(m1 uint32, d uint32, m2 uint32) {
hfsc.Usc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// SetFsc sets the Fsc curve
func (hfsc *HfscClass) SetFsc(m1 uint32, d uint32, m2 uint32) {
hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// SetRsc sets the Rsc curve
func (hfsc *HfscClass) SetRsc(m1 uint32, d uint32, m2 uint32) {
hfsc.Rsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// SetSC implements the SC from the tc CLI
func (hfsc *HfscClass) SetSC(m1 uint32, d uint32, m2 uint32) {
hfsc.Rsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// SetUL implements the UL from the tc CLI
func (hfsc *HfscClass) SetUL(m1 uint32, d uint32, m2 uint32) {
hfsc.Usc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// SetLS implemtens the LS from the tc CLI
func (hfsc *HfscClass) SetLS(m1 uint32, d uint32, m2 uint32) {
hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8}
}
// NewHfscClass returns a new HFSC struct with the set parameters
func NewHfscClass(attrs ClassAttrs) *HfscClass {
return &HfscClass{
ClassAttrs: attrs,
Rsc: ServiceCurve{},
Fsc: ServiceCurve{},
Usc: ServiceCurve{},
}
}
func (hfsc *HfscClass) String() string {
return fmt.Sprintf(
"{%s -- {RSC: {m1=%d d=%d m2=%d}} {FSC: {m1=%d d=%d m2=%d}} {USC: {m1=%d d=%d m2=%d}}}",
hfsc.Attrs(), hfsc.Rsc.m1*8, hfsc.Rsc.d, hfsc.Rsc.m2*8, hfsc.Fsc.m1*8, hfsc.Fsc.d, hfsc.Fsc.m2*8, hfsc.Usc.m1*8, hfsc.Usc.d, hfsc.Usc.m2*8,
)
}
// Attrs return the Hfsc parameters
func (hfsc *HfscClass) Attrs() *ClassAttrs {
return &hfsc.ClassAttrs
}
// Type return the type of the class
func (hfsc *HfscClass) Type() string {
return "hfsc"
}

View File

@ -28,7 +28,7 @@ type tcStats struct {
Backlog uint32
}
// NOTE: function is in here because it uses other linux functions
// NewHtbClass NOTE: function is in here because it uses other linux functions
func NewHtbClass(attrs ClassAttrs, cattrs HtbClassAttrs) *HtbClass {
mtu := 1600
rate := cattrs.Rate / 8
@ -146,7 +146,9 @@ func classPayload(req *nl.NetlinkRequest, class Class) error {
req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(class.Type())))
options := nl.NewRtAttr(nl.TCA_OPTIONS, nil)
if htb, ok := class.(*HtbClass); ok {
switch class.Type() {
case "htb":
htb := class.(*HtbClass)
opt := nl.TcHtbCopt{}
opt.Buffer = htb.Buffer
opt.Cbuffer = htb.Cbuffer
@ -174,6 +176,15 @@ func classPayload(req *nl.NetlinkRequest, class Class) error {
nl.NewRtAttrChild(options, nl.TCA_HTB_PARMS, opt.Serialize())
nl.NewRtAttrChild(options, nl.TCA_HTB_RTAB, SerializeRtab(rtab))
nl.NewRtAttrChild(options, nl.TCA_HTB_CTAB, SerializeRtab(ctab))
case "hfsc":
hfsc := class.(*HfscClass)
opt := nl.HfscCopt{}
opt.Rsc.Set(hfsc.Rsc.Attrs())
opt.Fsc.Set(hfsc.Fsc.Attrs())
opt.Usc.Set(hfsc.Usc.Attrs())
nl.NewRtAttrChild(options, nl.TCA_HFSC_RSC, nl.SerializeHfscCurve(&opt.Rsc))
nl.NewRtAttrChild(options, nl.TCA_HFSC_FSC, nl.SerializeHfscCurve(&opt.Fsc))
nl.NewRtAttrChild(options, nl.TCA_HFSC_USC, nl.SerializeHfscCurve(&opt.Usc))
}
req.AddData(options)
return nil
@ -232,6 +243,8 @@ func (h *Handle) ClassList(link Link, parent uint32) ([]Class, error) {
switch classType {
case "htb":
class = &HtbClass{}
case "hfsc":
class = &HfscClass{}
default:
class = &GenericClass{ClassType: classType}
}
@ -246,6 +259,15 @@ func (h *Handle) ClassList(link Link, parent uint32) ([]Class, error) {
if err != nil {
return nil, err
}
case "hfsc":
data, err := nl.ParseRouteAttr(attr.Value)
if err != nil {
return nil, err
}
_, err = parseHfscClassData(class, data)
if err != nil {
return nil, err
}
}
// For backward compatibility.
case nl.TCA_STATS:
@ -286,6 +308,23 @@ func parseHtbClassData(class Class, data []syscall.NetlinkRouteAttr) (bool, erro
return detailed, nil
}
func parseHfscClassData(class Class, data []syscall.NetlinkRouteAttr) (bool, error) {
hfsc := class.(*HfscClass)
detailed := false
for _, datum := range data {
m1, d, m2 := nl.DeserializeHfscCurve(datum.Value).Attrs()
switch datum.Attr.Type {
case nl.TCA_HFSC_RSC:
hfsc.Rsc = ServiceCurve{m1: m1, d: d, m2: m2}
case nl.TCA_HFSC_FSC:
hfsc.Fsc = ServiceCurve{m1: m1, d: d, m2: m2}
case nl.TCA_HFSC_USC:
hfsc.Usc = ServiceCurve{m1: m1, d: d, m2: m2}
}
}
return detailed, nil
}
func parseTcStats(data []byte) (*ClassStatistics, error) {
buf := &bytes.Buffer{}
buf.Write(data)

View File

@ -24,6 +24,26 @@ func SafeQdiscList(link Link) ([]Qdisc, error) {
return result, nil
}
func SafeClassList(link Link, handle uint32) ([]Class, error) {
classes, err := ClassList(link, handle)
if err != nil {
return nil, err
}
result := []Class{}
for ind := range classes {
double := false
for _, class2 := range classes[ind+1:] {
if classes[ind].Attrs().Handle == class2.Attrs().Handle {
double = true
}
}
if !double {
result = append(result, classes[ind])
}
}
return result, nil
}
func testClassStats(this, that *ClassStatistics, t *testing.T) {
ok := reflect.DeepEqual(this, that)
if !ok {
@ -435,3 +455,213 @@ func TestHtbClassAddHtbClassChangeDel(t *testing.T) {
t.Fatal("Failed to remove qdisc")
}
}
func TestClassHfsc(t *testing.T) {
// New network namespace for tests
tearDown := setUpNetlinkTestWithKModule(t, "sch_hfsc")
defer tearDown()
// Set up testing link and check if succeeded
if err := LinkAdd(&Ifb{LinkAttrs{Name: "foo"}}); err != nil {
t.Fatal(err)
}
link, err := LinkByName("foo")
if err != nil {
t.Fatal(err)
}
if err := LinkSetUp(link); err != nil {
t.Fatal(err)
}
// Adding HFSC qdisc
qdiscAttrs := QdiscAttrs{
LinkIndex: link.Attrs().Index,
Handle: MakeHandle(1, 0),
Parent: HANDLE_ROOT,
}
hfscQdisc := NewHfsc(qdiscAttrs)
hfscQdisc.Defcls = 2
err = QdiscAdd(hfscQdisc)
if err != nil {
t.Fatal(err)
}
qdiscs, err := SafeQdiscList(link)
if err != nil {
t.Fatal(err)
}
if len(qdiscs) != 1 {
t.Fatal("Failed to add qdisc")
}
_, ok := qdiscs[0].(*Hfsc)
if !ok {
t.Fatal("Qdisc is the wrong type")
}
// Adding some HFSC classes
classAttrs := ClassAttrs{
LinkIndex: link.Attrs().Index,
Parent: MakeHandle(1, 0),
Handle: MakeHandle(1, 1),
}
hfscClass := NewHfscClass(classAttrs)
hfscClass.SetLS(5e6, 10, 5e6)
err = ClassAdd(hfscClass)
if err != nil {
t.Fatal(err)
}
hfscClass2 := hfscClass
hfscClass2.SetLS(0, 0, 0)
hfscClass2.Attrs().Parent = MakeHandle(1, 1)
hfscClass2.Attrs().Handle = MakeHandle(1, 2)
hfscClass2.SetRsc(0, 0, 2e6)
err = ClassAdd(hfscClass2)
if err != nil {
t.Fatal(err)
}
hfscClass3 := hfscClass
hfscClass3.SetLS(0, 0, 0)
hfscClass3.Attrs().Parent = MakeHandle(1, 1)
hfscClass3.Attrs().Handle = MakeHandle(1, 3)
err = ClassAdd(hfscClass3)
if err != nil {
t.Fatal(err)
}
// Check the classes
classes, err := SafeClassList(link, MakeHandle(1, 0))
if err != nil {
t.Fatal(err)
}
if len(classes) != 4 {
t.Fatal("Failed to add classes")
}
for _, c := range classes {
class, ok := c.(*HfscClass)
if !ok {
t.Fatal("Wrong type of class")
}
if class.ClassAttrs.Handle == hfscClass.ClassAttrs.Handle {
if class.Fsc != hfscClass.Fsc {
t.Fatal("HfscClass FSC don't match")
}
if class.Usc != hfscClass.Usc {
t.Fatal("HfscClass USC don't match")
}
if class.Rsc != hfscClass.Rsc {
t.Fatal("HfscClass RSC don't match")
}
}
if class.ClassAttrs.Handle == hfscClass2.ClassAttrs.Handle {
if class.Fsc != hfscClass2.Fsc {
t.Fatal("HfscClass2 FSC don't match")
}
if class.Usc != hfscClass2.Usc {
t.Fatal("HfscClass2 USC don't match")
}
if class.Rsc != hfscClass2.Rsc {
t.Fatal("HfscClass2 RSC don't match")
}
}
if class.ClassAttrs.Handle == hfscClass3.ClassAttrs.Handle {
if class.Fsc != hfscClass3.Fsc {
t.Fatal("HfscClass3 FSC don't match")
}
if class.Usc != hfscClass3.Usc {
t.Fatal("HfscClass3 USC don't match")
}
if class.Rsc != hfscClass3.Rsc {
t.Fatal("HfscClass3 RSC don't match")
}
}
}
// Terminating the leafs with fq_codel qdiscs
fqcodelAttrs := QdiscAttrs{
LinkIndex: link.Attrs().Index,
Parent: MakeHandle(1, 2),
Handle: MakeHandle(2, 0),
}
fqcodel1 := NewFqCodel(fqcodelAttrs)
fqcodel1.ECN = 0
fqcodel1.Limit = 1200
fqcodel1.Flows = 65535
fqcodel1.Target = 5
err = QdiscAdd(fqcodel1)
if err != nil {
t.Fatal(err)
}
fqcodel2 := fqcodel1
fqcodel2.Attrs().Handle = MakeHandle(3, 0)
fqcodel2.Attrs().Parent = MakeHandle(1, 3)
err = QdiscAdd(fqcodel2)
if err != nil {
t.Fatal(err)
}
// Check the amount of qdiscs
qdiscs, err = SafeQdiscList(link)
if len(qdiscs) != 3 {
t.Fatal("Failed to add qdisc")
}
for _, q := range qdiscs[1:] {
_, ok = q.(*FqCodel)
if !ok {
t.Fatal("Qdisc is the wrong type")
}
}
// removing a class
if err := ClassDel(hfscClass3); err != nil {
t.Fatal(err)
}
// Check the classes
classes, err = SafeClassList(link, MakeHandle(1, 0))
if err != nil {
t.Fatal(err)
}
if len(classes) != 3 {
t.Fatal("Failed to delete classes")
}
// Check qdisc
qdiscs, err = SafeQdiscList(link)
if len(qdiscs) != 2 {
t.Fatal("Failed to delete qdisc")
}
// Changing a class
hfscClass2.SetRsc(0, 0, 0)
hfscClass2.SetSC(5e6, 100, 1e6)
hfscClass2.SetUL(6e6, 50, 2e6)
hfscClass2.Attrs().Handle = MakeHandle(1, 8)
if err := ClassChange(hfscClass2); err == nil {
t.Fatal("Class change shouldn't work with a different handle")
}
hfscClass2.Attrs().Handle = MakeHandle(1, 2)
if err := ClassChange(hfscClass2); err != nil {
t.Fatal(err)
}
// Replacing a class
// If the handle doesn't exist, create it
hfscClass2.SetSC(6e6, 100, 2e6)
hfscClass2.SetUL(8e6, 500, 4e6)
hfscClass2.Attrs().Handle = MakeHandle(1, 8)
if err := ClassReplace(hfscClass2); err != nil {
t.Fatal(err)
}
// If the handle exists, replace it
hfscClass.SetLS(5e6, 200, 1e6)
if err := ClassChange(hfscClass); err != nil {
t.Fatal(err)
}
}

View File

@ -1,6 +1,7 @@
package nl
import (
"encoding/binary"
"unsafe"
)
@ -421,6 +422,57 @@ func (x *TcHtbGlob) Serialize() []byte {
return (*(*[SizeofTcHtbGlob]byte)(unsafe.Pointer(x)))[:]
}
// HFSC
type Curve struct {
m1 uint32
d uint32
m2 uint32
}
type HfscCopt struct {
Rsc Curve
Fsc Curve
Usc Curve
}
func (c *Curve) Attrs() (uint32, uint32, uint32) {
return c.m1, c.d, c.m2
}
func (c *Curve) Set(m1 uint32, d uint32, m2 uint32) {
c.m1 = m1
c.d = d
c.m2 = m2
}
func DeserializeHfscCurve(b []byte) *Curve {
return &Curve{
m1: binary.LittleEndian.Uint32(b[0:4]),
d: binary.LittleEndian.Uint32(b[4:8]),
m2: binary.LittleEndian.Uint32(b[8:12]),
}
}
func SerializeHfscCurve(c *Curve) (b []byte) {
t := make([]byte, binary.MaxVarintLen32)
binary.LittleEndian.PutUint32(t, c.m1)
b = append(b, t[:4]...)
binary.LittleEndian.PutUint32(t, c.d)
b = append(b, t[:4]...)
binary.LittleEndian.PutUint32(t, c.m2)
b = append(b, t[:4]...)
return b
}
type TcHfscOpt struct {
Defcls uint16
}
func (x *TcHfscOpt) Serialize() []byte {
return (*(*[2]byte)(unsafe.Pointer(x)))[:]
}
const (
TCA_U32_UNSPEC = iota
TCA_U32_CLASSID
@ -717,3 +769,10 @@ const (
TCA_FQ_CODEL_DROP_BATCH_SIZE
TCA_FQ_CODEL_MEMORY_LIMIT
)
const (
TCA_HFSC_UNSPEC = iota
TCA_HFSC_RSC
TCA_HFSC_FSC
TCA_HFSC_USC
)

View File

@ -238,6 +238,33 @@ func (qdisc *GenericQdisc) Type() string {
return qdisc.QdiscType
}
type Hfsc struct {
QdiscAttrs
Defcls uint16
}
func NewHfsc(attrs QdiscAttrs) *Hfsc {
return &Hfsc{
QdiscAttrs: attrs,
Defcls: 1,
}
}
func (hfsc *Hfsc) Attrs() *QdiscAttrs {
return &hfsc.QdiscAttrs
}
func (hfsc *Hfsc) Type() string {
return "hfsc"
}
func (hfsc *Hfsc) String() string {
return fmt.Sprintf(
"{%v -- default: %d}",
hfsc.Attrs(), hfsc.Defcls,
)
}
// Fq is a classless packet scheduler meant to be mostly used for locally generated traffic.
type Fq struct {
QdiscAttrs
@ -292,8 +319,8 @@ type FqCodel struct {
func (fqcodel *FqCodel) String() string {
return fmt.Sprintf(
"{Target: %v, Limit: %v, Interval: %v, ECM: %v, Flows: %v, Quantum: %v}",
fqcodel.Target, fqcodel.Limit, fqcodel.Interval, fqcodel.ECN, fqcodel.Flows, fqcodel.Quantum,
"{%v -- Target: %v, Limit: %v, Interval: %v, ECM: %v, Flows: %v, Quantum: %v}",
fqcodel.Attrs(), fqcodel.Target, fqcodel.Limit, fqcodel.Interval, fqcodel.ECN, fqcodel.Flows, fqcodel.Quantum,
)
}

View File

@ -195,6 +195,10 @@ func qdiscPayload(req *nl.NetlinkRequest, qdisc Qdisc) error {
opt.DirectPkts = qdisc.DirectPkts
nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize())
// nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize())
case *Hfsc:
opt := nl.TcHfscOpt{}
opt.Defcls = qdisc.Defcls
options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize())
case *Netem:
opt := nl.TcNetemQopt{}
opt.Latency = qdisc.Latency
@ -348,6 +352,8 @@ func (h *Handle) QdiscList(link Link) ([]Qdisc, error) {
qdisc = &Htb{}
case "fq":
qdisc = &Fq{}
case "hfsc":
qdisc = &Hfsc{}
case "fq_codel":
qdisc = &FqCodel{}
case "netem":
@ -375,6 +381,10 @@ func (h *Handle) QdiscList(link Link) ([]Qdisc, error) {
if err := parseTbfData(qdisc, data); err != nil {
return nil, err
}
case "hfsc":
if err := parseHfscData(qdisc, attr.Value); err != nil {
return nil, err
}
case "htb":
data, err := nl.ParseRouteAttr(attr.Value)
if err != nil {
@ -474,6 +484,13 @@ func parseFqCodelData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error {
return nil
}
func parseHfscData(qdisc Qdisc, data []byte) error {
Hfsc := qdisc.(*Hfsc)
native = nl.NativeEndian()
Hfsc.Defcls = native.Uint16(data)
return nil
}
func parseFqData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error {
native = nl.NativeEndian()
fq := qdisc.(*Fq)