From 1006cf4f24de813313f439c91657b122566897e5 Mon Sep 17 00:00:00 2001 From: Francis Begyn Date: Mon, 9 Apr 2018 22:40:06 +0200 Subject: [PATCH] Implementation of HFSC Testing and functionality for the use of HFSC has been implemented. The use of service curves is implenented closely as to how they behave with the TC implementation. Automated checks and testing were succesful. --- .travis.yml | 1 + class.go | 96 +++++++++++++++++++-- class_linux.go | 43 ++++++++- class_test.go | 230 +++++++++++++++++++++++++++++++++++++++++++++++++ nl/tc_linux.go | 59 +++++++++++++ qdisc.go | 31 ++++++- qdisc_linux.go | 17 ++++ 7 files changed, 467 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index f5c0b3e..f9e04c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,6 @@ before_script: - sudo modprobe nf_conntrack_netlink - sudo modprobe nf_conntrack_ipv4 - sudo modprobe nf_conntrack_ipv6 + - sudo modprobe sch_hfsc install: - go get github.com/vishvananda/netns diff --git a/class.go b/class.go index ab41ed5..bd9bbb9 100644 --- a/class.go +++ b/class.go @@ -4,6 +4,7 @@ import ( "fmt" ) +// Class interfaces for all classes type Class interface { Attrs() *ClassAttrs Type() string @@ -13,25 +14,25 @@ type Class interface { // This file contains "gnet_" prefixed structs and relevant functions. // See Documentation/networking/getn_stats.txt in Linux source code for more details. -// Ref: struct gnet_stats_basic { ... } +// GnetStatsBasic Ref: struct gnet_stats_basic { ... } type GnetStatsBasic struct { Bytes uint64 // number of seen bytes Packets uint32 // number of seen packets } -// Ref: struct gnet_stats_rate_est { ... } +// GnetStatsRateEst Ref: struct gnet_stats_rate_est { ... } type GnetStatsRateEst struct { Bps uint32 // current byte rate Pps uint32 // current packet rate } -// Ref: struct gnet_stats_rate_est64 { ... } +// GnetStatsRateEst64 Ref: struct gnet_stats_rate_est64 { ... } type GnetStatsRateEst64 struct { Bps uint64 // current byte rate Pps uint64 // current packet rate } -// Ref: struct gnet_stats_queue { ... } +// GnetStatsQueue Ref: struct gnet_stats_queue { ... } type GnetStatsQueue struct { Qlen uint32 // queue length Backlog uint32 // backlog size of queue @@ -40,7 +41,7 @@ type GnetStatsQueue struct { Overlimits uint32 // number of enqueues over the limit } -// Statistics representaion based on generic networking statisticsfor netlink. +// ClassStatistics representaion based on generic networking statisticsfor netlink. // See Documentation/networking/gen_stats.txt in Linux source code for more details. type ClassStatistics struct { Basic *GnetStatsBasic @@ -48,7 +49,7 @@ type ClassStatistics struct { RateEst *GnetStatsRateEst } -// Construct a ClassStatistics struct which fields are all initialized by 0. +// NewClassStatistics Construct a ClassStatistics struct which fields are all initialized by 0. func NewClassStatistics() *ClassStatistics { return &ClassStatistics{ Basic: &GnetStatsBasic{}, @@ -72,6 +73,7 @@ func (q ClassAttrs) String() string { return fmt.Sprintf("{LinkIndex: %d, Handle: %s, Parent: %s, Leaf: %d}", q.LinkIndex, HandleStr(q.Handle), HandleStr(q.Parent), q.Leaf) } +// HtbClassAttrs stores the attributes of HTB class type HtbClassAttrs struct { // TODO handle all attributes Rate uint64 @@ -103,10 +105,12 @@ func (q HtbClass) String() string { return fmt.Sprintf("{Rate: %d, Ceil: %d, Buffer: %d, Cbuffer: %d}", q.Rate, q.Ceil, q.Buffer, q.Cbuffer) } +// Attrs returns the class attributes func (q *HtbClass) Attrs() *ClassAttrs { return &q.ClassAttrs } +// Type return the class type func (q *HtbClass) Type() string { return "htb" } @@ -118,10 +122,90 @@ type GenericClass struct { ClassType string } +// Attrs return the class attributes func (class *GenericClass) Attrs() *ClassAttrs { return &class.ClassAttrs } +// Type retrun the class type func (class *GenericClass) Type() string { return class.ClassType } + +// ServiceCurve is the way the HFSC curve are represented +type ServiceCurve struct { + m1 uint32 + d uint32 + m2 uint32 +} + +// Attrs return the parameters of the service curve +func (c *ServiceCurve) Attrs() (uint32, uint32, uint32) { + return c.m1, c.d, c.m2 +} + +// HfscClass is a representation of the HFSC class +type HfscClass struct { + ClassAttrs + Rsc ServiceCurve + Fsc ServiceCurve + Usc ServiceCurve +} + +// SetUsc sets the Usc curve +func (hfsc *HfscClass) SetUsc(m1 uint32, d uint32, m2 uint32) { + hfsc.Usc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// SetFsc sets the Fsc curve +func (hfsc *HfscClass) SetFsc(m1 uint32, d uint32, m2 uint32) { + hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// SetRsc sets the Rsc curve +func (hfsc *HfscClass) SetRsc(m1 uint32, d uint32, m2 uint32) { + hfsc.Rsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// SetSC implements the SC from the tc CLI +func (hfsc *HfscClass) SetSC(m1 uint32, d uint32, m2 uint32) { + hfsc.Rsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} + hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// SetUL implements the UL from the tc CLI +func (hfsc *HfscClass) SetUL(m1 uint32, d uint32, m2 uint32) { + hfsc.Usc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// SetLS implemtens the LS from the tc CLI +func (hfsc *HfscClass) SetLS(m1 uint32, d uint32, m2 uint32) { + hfsc.Fsc = ServiceCurve{m1: m1 / 8, d: d, m2: m2 / 8} +} + +// NewHfscClass returns a new HFSC struct with the set parameters +func NewHfscClass(attrs ClassAttrs) *HfscClass { + return &HfscClass{ + ClassAttrs: attrs, + Rsc: ServiceCurve{}, + Fsc: ServiceCurve{}, + Usc: ServiceCurve{}, + } +} + +func (hfsc *HfscClass) String() string { + return fmt.Sprintf( + "{%s -- {RSC: {m1=%d d=%d m2=%d}} {FSC: {m1=%d d=%d m2=%d}} {USC: {m1=%d d=%d m2=%d}}}", + hfsc.Attrs(), hfsc.Rsc.m1*8, hfsc.Rsc.d, hfsc.Rsc.m2*8, hfsc.Fsc.m1*8, hfsc.Fsc.d, hfsc.Fsc.m2*8, hfsc.Usc.m1*8, hfsc.Usc.d, hfsc.Usc.m2*8, + ) +} + +// Attrs return the Hfsc parameters +func (hfsc *HfscClass) Attrs() *ClassAttrs { + return &hfsc.ClassAttrs +} + +// Type return the type of the class +func (hfsc *HfscClass) Type() string { + return "hfsc" +} diff --git a/class_linux.go b/class_linux.go index a3c1584..8800b99 100644 --- a/class_linux.go +++ b/class_linux.go @@ -28,7 +28,7 @@ type tcStats struct { Backlog uint32 } -// NOTE: function is in here because it uses other linux functions +// NewHtbClass NOTE: function is in here because it uses other linux functions func NewHtbClass(attrs ClassAttrs, cattrs HtbClassAttrs) *HtbClass { mtu := 1600 rate := cattrs.Rate / 8 @@ -146,7 +146,9 @@ func classPayload(req *nl.NetlinkRequest, class Class) error { req.AddData(nl.NewRtAttr(nl.TCA_KIND, nl.ZeroTerminated(class.Type()))) options := nl.NewRtAttr(nl.TCA_OPTIONS, nil) - if htb, ok := class.(*HtbClass); ok { + switch class.Type() { + case "htb": + htb := class.(*HtbClass) opt := nl.TcHtbCopt{} opt.Buffer = htb.Buffer opt.Cbuffer = htb.Cbuffer @@ -174,6 +176,15 @@ func classPayload(req *nl.NetlinkRequest, class Class) error { nl.NewRtAttrChild(options, nl.TCA_HTB_PARMS, opt.Serialize()) nl.NewRtAttrChild(options, nl.TCA_HTB_RTAB, SerializeRtab(rtab)) nl.NewRtAttrChild(options, nl.TCA_HTB_CTAB, SerializeRtab(ctab)) + case "hfsc": + hfsc := class.(*HfscClass) + opt := nl.HfscCopt{} + opt.Rsc.Set(hfsc.Rsc.Attrs()) + opt.Fsc.Set(hfsc.Fsc.Attrs()) + opt.Usc.Set(hfsc.Usc.Attrs()) + nl.NewRtAttrChild(options, nl.TCA_HFSC_RSC, nl.SerializeHfscCurve(&opt.Rsc)) + nl.NewRtAttrChild(options, nl.TCA_HFSC_FSC, nl.SerializeHfscCurve(&opt.Fsc)) + nl.NewRtAttrChild(options, nl.TCA_HFSC_USC, nl.SerializeHfscCurve(&opt.Usc)) } req.AddData(options) return nil @@ -232,6 +243,8 @@ func (h *Handle) ClassList(link Link, parent uint32) ([]Class, error) { switch classType { case "htb": class = &HtbClass{} + case "hfsc": + class = &HfscClass{} default: class = &GenericClass{ClassType: classType} } @@ -246,6 +259,15 @@ func (h *Handle) ClassList(link Link, parent uint32) ([]Class, error) { if err != nil { return nil, err } + case "hfsc": + data, err := nl.ParseRouteAttr(attr.Value) + if err != nil { + return nil, err + } + _, err = parseHfscClassData(class, data) + if err != nil { + return nil, err + } } // For backward compatibility. case nl.TCA_STATS: @@ -286,6 +308,23 @@ func parseHtbClassData(class Class, data []syscall.NetlinkRouteAttr) (bool, erro return detailed, nil } +func parseHfscClassData(class Class, data []syscall.NetlinkRouteAttr) (bool, error) { + hfsc := class.(*HfscClass) + detailed := false + for _, datum := range data { + m1, d, m2 := nl.DeserializeHfscCurve(datum.Value).Attrs() + switch datum.Attr.Type { + case nl.TCA_HFSC_RSC: + hfsc.Rsc = ServiceCurve{m1: m1, d: d, m2: m2} + case nl.TCA_HFSC_FSC: + hfsc.Fsc = ServiceCurve{m1: m1, d: d, m2: m2} + case nl.TCA_HFSC_USC: + hfsc.Usc = ServiceCurve{m1: m1, d: d, m2: m2} + } + } + return detailed, nil +} + func parseTcStats(data []byte) (*ClassStatistics, error) { buf := &bytes.Buffer{} buf.Write(data) diff --git a/class_test.go b/class_test.go index cd72632..e507df2 100644 --- a/class_test.go +++ b/class_test.go @@ -24,6 +24,26 @@ func SafeQdiscList(link Link) ([]Qdisc, error) { return result, nil } +func SafeClassList(link Link, handle uint32) ([]Class, error) { + classes, err := ClassList(link, handle) + if err != nil { + return nil, err + } + result := []Class{} + for ind := range classes { + double := false + for _, class2 := range classes[ind+1:] { + if classes[ind].Attrs().Handle == class2.Attrs().Handle { + double = true + } + } + if !double { + result = append(result, classes[ind]) + } + } + return result, nil +} + func testClassStats(this, that *ClassStatistics, t *testing.T) { ok := reflect.DeepEqual(this, that) if !ok { @@ -435,3 +455,213 @@ func TestHtbClassAddHtbClassChangeDel(t *testing.T) { t.Fatal("Failed to remove qdisc") } } + +func TestClassHfsc(t *testing.T) { + // New network namespace for tests + tearDown := setUpNetlinkTestWithKModule(t, "sch_hfsc") + defer tearDown() + + // Set up testing link and check if succeeded + if err := LinkAdd(&Ifb{LinkAttrs{Name: "foo"}}); err != nil { + t.Fatal(err) + } + link, err := LinkByName("foo") + if err != nil { + t.Fatal(err) + } + if err := LinkSetUp(link); err != nil { + t.Fatal(err) + } + + // Adding HFSC qdisc + qdiscAttrs := QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Handle: MakeHandle(1, 0), + Parent: HANDLE_ROOT, + } + hfscQdisc := NewHfsc(qdiscAttrs) + hfscQdisc.Defcls = 2 + + err = QdiscAdd(hfscQdisc) + if err != nil { + t.Fatal(err) + } + qdiscs, err := SafeQdiscList(link) + if err != nil { + t.Fatal(err) + } + if len(qdiscs) != 1 { + t.Fatal("Failed to add qdisc") + } + _, ok := qdiscs[0].(*Hfsc) + if !ok { + t.Fatal("Qdisc is the wrong type") + } + + // Adding some HFSC classes + classAttrs := ClassAttrs{ + LinkIndex: link.Attrs().Index, + Parent: MakeHandle(1, 0), + Handle: MakeHandle(1, 1), + } + hfscClass := NewHfscClass(classAttrs) + hfscClass.SetLS(5e6, 10, 5e6) + + err = ClassAdd(hfscClass) + if err != nil { + t.Fatal(err) + } + + hfscClass2 := hfscClass + hfscClass2.SetLS(0, 0, 0) + hfscClass2.Attrs().Parent = MakeHandle(1, 1) + hfscClass2.Attrs().Handle = MakeHandle(1, 2) + hfscClass2.SetRsc(0, 0, 2e6) + + err = ClassAdd(hfscClass2) + if err != nil { + t.Fatal(err) + } + + hfscClass3 := hfscClass + hfscClass3.SetLS(0, 0, 0) + hfscClass3.Attrs().Parent = MakeHandle(1, 1) + hfscClass3.Attrs().Handle = MakeHandle(1, 3) + + err = ClassAdd(hfscClass3) + if err != nil { + t.Fatal(err) + } + + // Check the classes + classes, err := SafeClassList(link, MakeHandle(1, 0)) + if err != nil { + t.Fatal(err) + } + if len(classes) != 4 { + t.Fatal("Failed to add classes") + } + for _, c := range classes { + class, ok := c.(*HfscClass) + if !ok { + t.Fatal("Wrong type of class") + } + if class.ClassAttrs.Handle == hfscClass.ClassAttrs.Handle { + if class.Fsc != hfscClass.Fsc { + t.Fatal("HfscClass FSC don't match") + } + if class.Usc != hfscClass.Usc { + t.Fatal("HfscClass USC don't match") + } + if class.Rsc != hfscClass.Rsc { + t.Fatal("HfscClass RSC don't match") + } + } + if class.ClassAttrs.Handle == hfscClass2.ClassAttrs.Handle { + if class.Fsc != hfscClass2.Fsc { + t.Fatal("HfscClass2 FSC don't match") + } + if class.Usc != hfscClass2.Usc { + t.Fatal("HfscClass2 USC don't match") + } + if class.Rsc != hfscClass2.Rsc { + t.Fatal("HfscClass2 RSC don't match") + } + } + if class.ClassAttrs.Handle == hfscClass3.ClassAttrs.Handle { + if class.Fsc != hfscClass3.Fsc { + t.Fatal("HfscClass3 FSC don't match") + } + if class.Usc != hfscClass3.Usc { + t.Fatal("HfscClass3 USC don't match") + } + if class.Rsc != hfscClass3.Rsc { + t.Fatal("HfscClass3 RSC don't match") + } + } + } + + // Terminating the leafs with fq_codel qdiscs + fqcodelAttrs := QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: MakeHandle(1, 2), + Handle: MakeHandle(2, 0), + } + fqcodel1 := NewFqCodel(fqcodelAttrs) + fqcodel1.ECN = 0 + fqcodel1.Limit = 1200 + fqcodel1.Flows = 65535 + fqcodel1.Target = 5 + + err = QdiscAdd(fqcodel1) + if err != nil { + t.Fatal(err) + } + + fqcodel2 := fqcodel1 + fqcodel2.Attrs().Handle = MakeHandle(3, 0) + fqcodel2.Attrs().Parent = MakeHandle(1, 3) + + err = QdiscAdd(fqcodel2) + if err != nil { + t.Fatal(err) + } + + // Check the amount of qdiscs + qdiscs, err = SafeQdiscList(link) + if len(qdiscs) != 3 { + t.Fatal("Failed to add qdisc") + } + for _, q := range qdiscs[1:] { + _, ok = q.(*FqCodel) + if !ok { + t.Fatal("Qdisc is the wrong type") + } + } + + // removing a class + if err := ClassDel(hfscClass3); err != nil { + t.Fatal(err) + } + // Check the classes + classes, err = SafeClassList(link, MakeHandle(1, 0)) + if err != nil { + t.Fatal(err) + } + if len(classes) != 3 { + t.Fatal("Failed to delete classes") + } + // Check qdisc + qdiscs, err = SafeQdiscList(link) + if len(qdiscs) != 2 { + t.Fatal("Failed to delete qdisc") + } + + // Changing a class + hfscClass2.SetRsc(0, 0, 0) + hfscClass2.SetSC(5e6, 100, 1e6) + hfscClass2.SetUL(6e6, 50, 2e6) + hfscClass2.Attrs().Handle = MakeHandle(1, 8) + if err := ClassChange(hfscClass2); err == nil { + t.Fatal("Class change shouldn't work with a different handle") + } + hfscClass2.Attrs().Handle = MakeHandle(1, 2) + if err := ClassChange(hfscClass2); err != nil { + t.Fatal(err) + } + + // Replacing a class + // If the handle doesn't exist, create it + hfscClass2.SetSC(6e6, 100, 2e6) + hfscClass2.SetUL(8e6, 500, 4e6) + hfscClass2.Attrs().Handle = MakeHandle(1, 8) + if err := ClassReplace(hfscClass2); err != nil { + t.Fatal(err) + } + // If the handle exists, replace it + hfscClass.SetLS(5e6, 200, 1e6) + if err := ClassChange(hfscClass); err != nil { + t.Fatal(err) + } + +} diff --git a/nl/tc_linux.go b/nl/tc_linux.go index 1543e7c..6c20b67 100644 --- a/nl/tc_linux.go +++ b/nl/tc_linux.go @@ -1,6 +1,7 @@ package nl import ( + "encoding/binary" "unsafe" ) @@ -421,6 +422,57 @@ func (x *TcHtbGlob) Serialize() []byte { return (*(*[SizeofTcHtbGlob]byte)(unsafe.Pointer(x)))[:] } +// HFSC + +type Curve struct { + m1 uint32 + d uint32 + m2 uint32 +} + +type HfscCopt struct { + Rsc Curve + Fsc Curve + Usc Curve +} + +func (c *Curve) Attrs() (uint32, uint32, uint32) { + return c.m1, c.d, c.m2 +} + +func (c *Curve) Set(m1 uint32, d uint32, m2 uint32) { + c.m1 = m1 + c.d = d + c.m2 = m2 +} + +func DeserializeHfscCurve(b []byte) *Curve { + return &Curve{ + m1: binary.LittleEndian.Uint32(b[0:4]), + d: binary.LittleEndian.Uint32(b[4:8]), + m2: binary.LittleEndian.Uint32(b[8:12]), + } +} + +func SerializeHfscCurve(c *Curve) (b []byte) { + t := make([]byte, binary.MaxVarintLen32) + binary.LittleEndian.PutUint32(t, c.m1) + b = append(b, t[:4]...) + binary.LittleEndian.PutUint32(t, c.d) + b = append(b, t[:4]...) + binary.LittleEndian.PutUint32(t, c.m2) + b = append(b, t[:4]...) + return b +} + +type TcHfscOpt struct { + Defcls uint16 +} + +func (x *TcHfscOpt) Serialize() []byte { + return (*(*[2]byte)(unsafe.Pointer(x)))[:] +} + const ( TCA_U32_UNSPEC = iota TCA_U32_CLASSID @@ -717,3 +769,10 @@ const ( TCA_FQ_CODEL_DROP_BATCH_SIZE TCA_FQ_CODEL_MEMORY_LIMIT ) + +const ( + TCA_HFSC_UNSPEC = iota + TCA_HFSC_RSC + TCA_HFSC_FSC + TCA_HFSC_USC +) diff --git a/qdisc.go b/qdisc.go index 0e94988..3540fb0 100644 --- a/qdisc.go +++ b/qdisc.go @@ -238,6 +238,33 @@ func (qdisc *GenericQdisc) Type() string { return qdisc.QdiscType } +type Hfsc struct { + QdiscAttrs + Defcls uint16 +} + +func NewHfsc(attrs QdiscAttrs) *Hfsc { + return &Hfsc{ + QdiscAttrs: attrs, + Defcls: 1, + } +} + +func (hfsc *Hfsc) Attrs() *QdiscAttrs { + return &hfsc.QdiscAttrs +} + +func (hfsc *Hfsc) Type() string { + return "hfsc" +} + +func (hfsc *Hfsc) String() string { + return fmt.Sprintf( + "{%v -- default: %d}", + hfsc.Attrs(), hfsc.Defcls, + ) +} + // Fq is a classless packet scheduler meant to be mostly used for locally generated traffic. type Fq struct { QdiscAttrs @@ -292,8 +319,8 @@ type FqCodel struct { func (fqcodel *FqCodel) String() string { return fmt.Sprintf( - "{Target: %v, Limit: %v, Interval: %v, ECM: %v, Flows: %v, Quantum: %v}", - fqcodel.Target, fqcodel.Limit, fqcodel.Interval, fqcodel.ECN, fqcodel.Flows, fqcodel.Quantum, + "{%v -- Target: %v, Limit: %v, Interval: %v, ECM: %v, Flows: %v, Quantum: %v}", + fqcodel.Attrs(), fqcodel.Target, fqcodel.Limit, fqcodel.Interval, fqcodel.ECN, fqcodel.Flows, fqcodel.Quantum, ) } diff --git a/qdisc_linux.go b/qdisc_linux.go index 3794ac1..b0fd231 100644 --- a/qdisc_linux.go +++ b/qdisc_linux.go @@ -195,6 +195,10 @@ func qdiscPayload(req *nl.NetlinkRequest, qdisc Qdisc) error { opt.DirectPkts = qdisc.DirectPkts nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize()) // nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize()) + case *Hfsc: + opt := nl.TcHfscOpt{} + opt.Defcls = qdisc.Defcls + options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize()) case *Netem: opt := nl.TcNetemQopt{} opt.Latency = qdisc.Latency @@ -348,6 +352,8 @@ func (h *Handle) QdiscList(link Link) ([]Qdisc, error) { qdisc = &Htb{} case "fq": qdisc = &Fq{} + case "hfsc": + qdisc = &Hfsc{} case "fq_codel": qdisc = &FqCodel{} case "netem": @@ -375,6 +381,10 @@ func (h *Handle) QdiscList(link Link) ([]Qdisc, error) { if err := parseTbfData(qdisc, data); err != nil { return nil, err } + case "hfsc": + if err := parseHfscData(qdisc, attr.Value); err != nil { + return nil, err + } case "htb": data, err := nl.ParseRouteAttr(attr.Value) if err != nil { @@ -474,6 +484,13 @@ func parseFqCodelData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error { return nil } +func parseHfscData(qdisc Qdisc, data []byte) error { + Hfsc := qdisc.(*Hfsc) + native = nl.NativeEndian() + Hfsc.Defcls = native.Uint16(data) + return nil +} + func parseFqData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error { native = nl.NativeEndian() fq := qdisc.(*Fq)