From 9b7c60d6bd29a1ff47b6b348707ae87579c0bdc2 Mon Sep 17 00:00:00 2001 From: chantra Date: Thu, 22 Oct 2015 22:31:36 -0700 Subject: [PATCH 1/2] [netem] minimalist support for netem Support for tc_netem_qopt options, e.g: latency, limit, loss, gap, duplicate and jitter --- class_test.go | 31 ++++++++++++++++++++++ nl/tc_linux.go | 48 +++++++++++++++++++++++++++++++++ qdisc.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ qdisc_linux.go | 28 ++++++++++++++++++++ qdisc_test.go | 8 +----- 5 files changed, 180 insertions(+), 7 deletions(-) diff --git a/class_test.go b/class_test.go index d163f73..0a4c7db 100644 --- a/class_test.go +++ b/class_test.go @@ -79,6 +79,37 @@ func TestClassAddDel(t *testing.T) { if htb.Cbuffer != class.Cbuffer { t.Fatal("Cbuffer doesn't match") } + + qattrs := QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Handle: MakeHandle(0x2, 0), + Parent: MakeHandle(0xffff, 2), + } + nattrs := NetemQdiscAttrs{ + Latency: 20000, + Loss: 23.4, + Duplicate: 14.3, + Jitter: 1000, + } + netem := NewNetem(qattrs, nattrs) + if err := QdiscAdd(netem); err != nil { + t.Fatal(err) + } + + qdiscs, err = QdiscList(link) + if err != nil { + t.Fatal(err) + } + if len(qdiscs) != 2 { + t.Fatal("Failed to add qdisc") + } + _, ok = qdiscs[0].(*Htb) + if !ok { + t.Fatal("Qdisc is the wrong type") + } + + // Deletion + if err := ClassDel(class); err != nil { t.Fatal(err) } diff --git a/nl/tc_linux.go b/nl/tc_linux.go index 4a32055..0482e43 100644 --- a/nl/tc_linux.go +++ b/nl/tc_linux.go @@ -60,6 +60,7 @@ const ( SizeofTcActionMsg = 0x04 SizeofTcPrioMap = 0x14 SizeofTcRateSpec = 0x0c + SizeofTcNetemQopt = 0x18 SizeofTcTbfQopt = 2*SizeofTcRateSpec + 0x0c SizeofTcHtbCopt = 2*SizeofTcRateSpec + 0x14 SizeofTcHtbGlob = 0x14 @@ -191,6 +192,53 @@ func (x *TcRateSpec) Serialize() []byte { return (*(*[SizeofTcRateSpec]byte)(unsafe.Pointer(x)))[:] } +/** +* NETEM + */ + +const ( + TCA_NETEM_UNSPEC = iota + TCA_NETEM_CORR + TCA_NETEM_DELAY_DIST + TCA_NETEM_REORDER + TCA_NETEM_CORRUPT + TCA_NETEM_LOSS + TCA_NETEM_RATE + TCA_NETEM_ECN + TCA_NETEM_RATE64 + TCA_NETEM_MAX = TCA_NETEM_RATE64 +) + +// struct tc_netem_qopt { +// __u32 latency; /* added delay (us) */ +// __u32 limit; /* fifo limit (packets) */ +// __u32 loss; /* random packet loss (0=none ~0=100%) */ +// __u32 gap; /* re-ordering gap (0 for none) */ +// __u32 duplicate; /* random packet dup (0=none ~0=100%) */ +// __u32 jitter; /* random jitter in latency (us) */ +// }; + +type TcNetemQopt struct { + Latency uint32 + Limit uint32 + Loss uint32 + Gap uint32 + Duplicate uint32 + Jitter uint32 +} + +func (msg *TcNetemQopt) Len() int { + return SizeofTcNetemQopt +} + +func DeserializeTcNetemQopt(b []byte) *TcNetemQopt { + return (*TcNetemQopt)(unsafe.Pointer(&b[0:SizeofTcNetemQopt][0])) +} + +func (x *TcNetemQopt) Serialize() []byte { + return (*(*[SizeofTcNetemQopt]byte)(unsafe.Pointer(x)))[:] +} + // struct tc_tbf_qopt { // struct tc_ratespec rate; // struct tc_ratespec peakrate; diff --git a/qdisc.go b/qdisc.go index 41a4aa8..29cb958 100644 --- a/qdisc.go +++ b/qdisc.go @@ -2,6 +2,7 @@ package netlink import ( "fmt" + "math" ) const ( @@ -52,6 +53,10 @@ func HandleStr(handle uint32) string { } } +func Percentage(percentage float32) uint32 { + return uint32(math.MaxUint32 * (percentage / 100)) +} + // PfifoFast is the default qdisc created by the kernel if one has not // been defined for the interface type PfifoFast struct { @@ -120,6 +125,73 @@ func (qdisc *Htb) Type() string { return "htb" } +// Netem is a classless qdisc that rate limits based on tokens + +type NetemQdiscAttrs struct { + Latency uint32 // in us + Limit uint32 + Loss float32 // in % + Gap uint32 + Duplicate float32 // in % + Jitter uint32 // in us +} + +func (q NetemQdiscAttrs) String() string { + return fmt.Sprintf( + "{Latency: %d, Limit: %d, Loss: %d, Gap: %d, Duplicate: %d, Jitter: %d}", + q.Latency, q.Limit, q.Loss, q.Gap, q.Duplicate, q.Jitter, + ) +} + +type Netem struct { + QdiscAttrs + Latency uint32 + Limit uint32 + Loss uint32 + Gap uint32 + Duplicate uint32 + Jitter uint32 +} + +func NewNetem(attrs QdiscAttrs, nattrs NetemQdiscAttrs) *Netem { + var limit uint32 = 1000 + + latency := nattrs.Latency + loss := Percentage(nattrs.Loss) + gap := nattrs.Gap + duplicate := Percentage(nattrs.Duplicate) + jitter := nattrs.Jitter + + // FIXME should validate values(like loss/duplicate are percentages...) + latency = time2Tick(latency) + + if nattrs.Limit != 0 { + limit = nattrs.Limit + } + // Jitter is only value if latency is > 0 + if latency > 0 { + jitter = time2Tick(jitter) + } + + return &Netem{ + QdiscAttrs: attrs, + Latency: latency, + Limit: limit, + Loss: loss, + Gap: gap, + Duplicate: duplicate, + Jitter: jitter, + } +} + +func (qdisc *Netem) Attrs() *QdiscAttrs { + return &qdisc.QdiscAttrs +} + +func (qdisc *Netem) Type() string { + return "netem" +} + // Tbf is a classless qdisc that rate limits based on tokens type Tbf struct { QdiscAttrs diff --git a/qdisc_linux.go b/qdisc_linux.go index a16eb99..3acc363 100644 --- a/qdisc_linux.go +++ b/qdisc_linux.go @@ -65,6 +65,15 @@ func QdiscAdd(qdisc Qdisc) error { opt.DirectPkts = htb.DirectPkts nl.NewRtAttrChild(options, nl.TCA_HTB_INIT, opt.Serialize()) // nl.NewRtAttrChild(options, nl.TCA_HTB_DIRECT_QLEN, opt.Serialize()) + } else if netem, ok := qdisc.(*Netem); ok { + opt := nl.TcNetemQopt{} + opt.Latency = netem.Latency + opt.Limit = netem.Limit + opt.Loss = netem.Loss + opt.Gap = netem.Gap + opt.Duplicate = netem.Duplicate + opt.Jitter = netem.Jitter + options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize()) } else if _, ok := qdisc.(*Ingress); ok { // ingress filters must use the proper handle if msg.Parent != HANDLE_INGRESS { @@ -135,6 +144,8 @@ func QdiscList(link Link) ([]Qdisc, error) { qdisc = &Ingress{} case "htb": qdisc = &Htb{} + case "netem": + qdisc = &Netem{} default: qdisc = &GenericQdisc{QdiscType: qdiscType} } @@ -166,6 +177,10 @@ func QdiscList(link Link) ([]Qdisc, error) { if err := parseHtbData(qdisc, data); err != nil { return nil, err } + case "netem": + if err := parseNetemData(qdisc, attr.Value); err != nil { + return nil, err + } // no options for ingress } @@ -213,6 +228,19 @@ func parseHtbData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error { } return nil } + +func parseNetemData(qdisc Qdisc, value []byte) error { + netem := qdisc.(*Netem) + opt := nl.DeserializeTcNetemQopt(value) + netem.Latency = opt.Latency + netem.Limit = opt.Limit + netem.Loss = opt.Loss + netem.Gap = opt.Gap + netem.Duplicate = opt.Duplicate + netem.Jitter = opt.Jitter + return nil +} + func parseTbfData(qdisc Qdisc, data []syscall.NetlinkRouteAttr) error { native = nl.NativeEndian() tbf := qdisc.(*Tbf) diff --git a/qdisc_test.go b/qdisc_test.go index 1dc92db..89a2532 100644 --- a/qdisc_test.go +++ b/qdisc_test.go @@ -88,13 +88,6 @@ func TestHtbAddDel(t *testing.T) { t.Fatal(err) } - /* - cmd := exec.Command("tc", "qdisc") - out, err := cmd.CombinedOutput() - if err == nil { - fmt.Printf("%s\n", out) - } - */ qdiscs, err := QdiscList(link) if err != nil { t.Fatal(err) @@ -126,6 +119,7 @@ func TestHtbAddDel(t *testing.T) { t.Fatal("Failed to remove qdisc") } } + func TestPrioAddDel(t *testing.T) { tearDown := setUpNetlinkTest(t) defer tearDown() From 322c7826d2ef56e15b18233526be57f38fc74164 Mon Sep 17 00:00:00 2001 From: chantra Date: Sat, 24 Oct 2015 20:36:18 -0700 Subject: [PATCH 2/2] [netem] add support for reordering/corruption/correlation Note: This requires #58 to be merged in order to properly work. --- class_test.go | 42 ++++++++++++++++++---- nl/tc_linux.go | 95 +++++++++++++++++++++++++++++++++++++++++++------- qdisc.go | 95 ++++++++++++++++++++++++++++++++++++++------------ qdisc_linux.go | 45 ++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 41 deletions(-) diff --git a/class_test.go b/class_test.go index 0a4c7db..cdc32b9 100644 --- a/class_test.go +++ b/class_test.go @@ -86,13 +86,18 @@ func TestClassAddDel(t *testing.T) { Parent: MakeHandle(0xffff, 2), } nattrs := NetemQdiscAttrs{ - Latency: 20000, - Loss: 23.4, - Duplicate: 14.3, - Jitter: 1000, + Latency: 20000, + Loss: 23.4, + Duplicate: 14.3, + LossCorr: 8.34, + Jitter: 1000, + DelayCorr: 12.3, + ReorderProb: 23.4, + CorruptProb: 10.0, + CorruptCorr: 10, } - netem := NewNetem(qattrs, nattrs) - if err := QdiscAdd(netem); err != nil { + qdiscnetem := NewNetem(qattrs, nattrs) + if err := QdiscAdd(qdiscnetem); err != nil { t.Fatal(err) } @@ -108,8 +113,31 @@ func TestClassAddDel(t *testing.T) { t.Fatal("Qdisc is the wrong type") } - // Deletion + netem, ok := qdiscs[1].(*Netem) + if !ok { + t.Fatal("Qdisc is the wrong type") + } + // Compare the record we got from the list with the one we created + if netem.Loss != qdiscnetem.Loss { + t.Fatal("Loss does not match") + } + if netem.Latency != qdiscnetem.Latency { + t.Fatal("Latency does not match") + } + if netem.CorruptProb != qdiscnetem.CorruptProb { + t.Fatal("CorruptProb does not match") + } + if netem.Jitter != qdiscnetem.Jitter { + t.Fatal("Jitter does not match") + } + if netem.LossCorr != qdiscnetem.LossCorr { + t.Fatal("Loss does not match") + } + if netem.DuplicateCorr != qdiscnetem.DuplicateCorr { + t.Fatal("DuplicateCorr does not match") + } + // Deletion if err := ClassDel(class); err != nil { t.Fatal(err) } diff --git a/nl/tc_linux.go b/nl/tc_linux.go index 0482e43..aa59005 100644 --- a/nl/tc_linux.go +++ b/nl/tc_linux.go @@ -56,18 +56,21 @@ const ( ) const ( - SizeofTcMsg = 0x14 - SizeofTcActionMsg = 0x04 - SizeofTcPrioMap = 0x14 - SizeofTcRateSpec = 0x0c - SizeofTcNetemQopt = 0x18 - SizeofTcTbfQopt = 2*SizeofTcRateSpec + 0x0c - SizeofTcHtbCopt = 2*SizeofTcRateSpec + 0x14 - SizeofTcHtbGlob = 0x14 - SizeofTcU32Key = 0x10 - SizeofTcU32Sel = 0x10 // without keys - SizeofTcMirred = 0x1c - SizeofTcPolice = 2*SizeofTcRateSpec + 0x20 + SizeofTcMsg = 0x14 + SizeofTcActionMsg = 0x04 + SizeofTcPrioMap = 0x14 + SizeofTcRateSpec = 0x0c + SizeofTcNetemQopt = 0x18 + SizeofTcNetemCorr = 0x0c + SizeofTcNetemReorder = 0x08 + SizeofTcNetemCorrupt = 0x08 + SizeofTcTbfQopt = 2*SizeofTcRateSpec + 0x0c + SizeofTcHtbCopt = 2*SizeofTcRateSpec + 0x14 + SizeofTcHtbGlob = 0x14 + SizeofTcU32Key = 0x10 + SizeofTcU32Sel = 0x10 // without keys + SizeofTcMirred = 0x1c + SizeofTcPolice = 2*SizeofTcRateSpec + 0x20 ) // struct tcmsg { @@ -239,6 +242,74 @@ func (x *TcNetemQopt) Serialize() []byte { return (*(*[SizeofTcNetemQopt]byte)(unsafe.Pointer(x)))[:] } +// struct tc_netem_corr { +// __u32 delay_corr; /* delay correlation */ +// __u32 loss_corr; /* packet loss correlation */ +// __u32 dup_corr; /* duplicate correlation */ +// }; + +type TcNetemCorr struct { + DelayCorr uint32 + LossCorr uint32 + DupCorr uint32 +} + +func (msg *TcNetemCorr) Len() int { + return SizeofTcNetemCorr +} + +func DeserializeTcNetemCorr(b []byte) *TcNetemCorr { + return (*TcNetemCorr)(unsafe.Pointer(&b[0:SizeofTcNetemCorr][0])) +} + +func (x *TcNetemCorr) Serialize() []byte { + return (*(*[SizeofTcNetemCorr]byte)(unsafe.Pointer(x)))[:] +} + +// struct tc_netem_reorder { +// __u32 probability; +// __u32 correlation; +// }; + +type TcNetemReorder struct { + Probability uint32 + Correlation uint32 +} + +func (msg *TcNetemReorder) Len() int { + return SizeofTcNetemReorder +} + +func DeserializeTcNetemReorder(b []byte) *TcNetemReorder { + return (*TcNetemReorder)(unsafe.Pointer(&b[0:SizeofTcNetemReorder][0])) +} + +func (x *TcNetemReorder) Serialize() []byte { + return (*(*[SizeofTcNetemReorder]byte)(unsafe.Pointer(x)))[:] +} + +// struct tc_netem_corrupt { +// __u32 probability; +// __u32 correlation; +// }; + +type TcNetemCorrupt struct { + Probability uint32 + Correlation uint32 +} + +func (msg *TcNetemCorrupt) Len() int { + return SizeofTcNetemCorrupt +} + +func DeserializeTcNetemCorrupt(b []byte) *TcNetemCorrupt { + return (*TcNetemCorrupt)(unsafe.Pointer(&b[0:SizeofTcNetemCorrupt][0])) +} + +func (x *TcNetemCorrupt) Serialize() []byte { + return (*(*[SizeofTcNetemCorrupt]byte)(unsafe.Pointer(x)))[:] +} + // struct tc_tbf_qopt { // struct tc_ratespec rate; // struct tc_ratespec peakrate; diff --git a/qdisc.go b/qdisc.go index 29cb958..48fe7c7 100644 --- a/qdisc.go +++ b/qdisc.go @@ -53,7 +53,11 @@ func HandleStr(handle uint32) string { } } -func Percentage(percentage float32) uint32 { +func Percentage2u32(percentage float32) uint32 { + // FIXME this is most likely not the best way to convert from % to uint32 + if percentage == 100 { + return math.MaxUint32 + } return uint32(math.MaxUint32 * (percentage / 100)) } @@ -128,12 +132,19 @@ func (qdisc *Htb) Type() string { // Netem is a classless qdisc that rate limits based on tokens type NetemQdiscAttrs struct { - Latency uint32 // in us - Limit uint32 - Loss float32 // in % - Gap uint32 - Duplicate float32 // in % - Jitter uint32 // in us + Latency uint32 // in us + DelayCorr float32 // in % + Limit uint32 + Loss float32 // in % + LossCorr float32 // in % + Gap uint32 + Duplicate float32 // in % + DuplicateCorr float32 // in % + Jitter uint32 // in us + ReorderProb float32 // in % + ReorderCorr float32 // in % + CorruptProb float32 // in % + CorruptCorr float32 // in % } func (q NetemQdiscAttrs) String() string { @@ -145,23 +156,43 @@ func (q NetemQdiscAttrs) String() string { type Netem struct { QdiscAttrs - Latency uint32 - Limit uint32 - Loss uint32 - Gap uint32 - Duplicate uint32 - Jitter uint32 + Latency uint32 + DelayCorr uint32 + Limit uint32 + Loss uint32 + LossCorr uint32 + Gap uint32 + Duplicate uint32 + DuplicateCorr uint32 + Jitter uint32 + ReorderProb uint32 + ReorderCorr uint32 + CorruptProb uint32 + CorruptCorr uint32 } func NewNetem(attrs QdiscAttrs, nattrs NetemQdiscAttrs) *Netem { var limit uint32 = 1000 + var loss_corr, delay_corr, duplicate_corr uint32 + var reorder_prob, reorder_corr uint32 + var corrupt_prob, corrupt_corr uint32 latency := nattrs.Latency - loss := Percentage(nattrs.Loss) + loss := Percentage2u32(nattrs.Loss) gap := nattrs.Gap - duplicate := Percentage(nattrs.Duplicate) + duplicate := Percentage2u32(nattrs.Duplicate) jitter := nattrs.Jitter + // Correlation + if latency > 0 && jitter > 0 { + delay_corr = Percentage2u32(nattrs.DelayCorr) + } + if loss > 0 { + loss_corr = Percentage2u32(nattrs.LossCorr) + } + if duplicate > 0 { + duplicate_corr = Percentage2u32(nattrs.DuplicateCorr) + } // FIXME should validate values(like loss/duplicate are percentages...) latency = time2Tick(latency) @@ -173,14 +204,34 @@ func NewNetem(attrs QdiscAttrs, nattrs NetemQdiscAttrs) *Netem { jitter = time2Tick(jitter) } + reorder_prob = Percentage2u32(nattrs.ReorderProb) + reorder_corr = Percentage2u32(nattrs.ReorderCorr) + + if reorder_prob > 0 { + // ERROR if lantency == 0 + if gap == 0 { + gap = 1 + } + } + + corrupt_prob = Percentage2u32(nattrs.CorruptProb) + corrupt_corr = Percentage2u32(nattrs.CorruptCorr) + return &Netem{ - QdiscAttrs: attrs, - Latency: latency, - Limit: limit, - Loss: loss, - Gap: gap, - Duplicate: duplicate, - Jitter: jitter, + QdiscAttrs: attrs, + Latency: latency, + DelayCorr: delay_corr, + Limit: limit, + Loss: loss, + LossCorr: loss_corr, + Gap: gap, + Duplicate: duplicate, + DuplicateCorr: duplicate_corr, + Jitter: jitter, + ReorderProb: reorder_prob, + ReorderCorr: reorder_corr, + CorruptProb: corrupt_prob, + CorruptCorr: corrupt_corr, } } diff --git a/qdisc_linux.go b/qdisc_linux.go index 3acc363..eb253b0 100644 --- a/qdisc_linux.go +++ b/qdisc_linux.go @@ -74,12 +74,36 @@ func QdiscAdd(qdisc Qdisc) error { opt.Duplicate = netem.Duplicate opt.Jitter = netem.Jitter options = nl.NewRtAttr(nl.TCA_OPTIONS, opt.Serialize()) + // Correlation + corr := nl.TcNetemCorr{} + corr.DelayCorr = netem.DelayCorr + corr.LossCorr = netem.LossCorr + corr.DupCorr = netem.DuplicateCorr + + if corr.DelayCorr > 0 || corr.LossCorr > 0 || corr.DupCorr > 0 { + nl.NewRtAttrChild(options, nl.TCA_NETEM_CORR, corr.Serialize()) + } + // Corruption + corruption := nl.TcNetemCorrupt{} + corruption.Probability = netem.CorruptProb + corruption.Correlation = netem.CorruptCorr + if corruption.Probability > 0 { + nl.NewRtAttrChild(options, nl.TCA_NETEM_CORRUPT, corruption.Serialize()) + } + // Reorder + reorder := nl.TcNetemReorder{} + reorder.Probability = netem.ReorderProb + reorder.Correlation = netem.ReorderCorr + if reorder.Probability > 0 { + nl.NewRtAttrChild(options, nl.TCA_NETEM_REORDER, reorder.Serialize()) + } } else if _, ok := qdisc.(*Ingress); ok { // ingress filters must use the proper handle if msg.Parent != HANDLE_INGRESS { return fmt.Errorf("Ingress filters must set Parent to HANDLE_INGRESS") } } + req.AddData(options) _, err := req.Execute(syscall.NETLINK_ROUTE, 0) return err @@ -238,6 +262,27 @@ func parseNetemData(qdisc Qdisc, value []byte) error { netem.Gap = opt.Gap netem.Duplicate = opt.Duplicate netem.Jitter = opt.Jitter + data, err := nl.ParseRouteAttr(value[nl.SizeofTcNetemQopt:]) + if err != nil { + return err + } + for _, datum := range data { + switch datum.Attr.Type { + case nl.TCA_NETEM_CORR: + opt := nl.DeserializeTcNetemCorr(datum.Value) + netem.DelayCorr = opt.DelayCorr + netem.LossCorr = opt.LossCorr + netem.DuplicateCorr = opt.DupCorr + case nl.TCA_NETEM_CORRUPT: + opt := nl.DeserializeTcNetemCorrupt(datum.Value) + netem.CorruptProb = opt.Probability + netem.CorruptCorr = opt.Correlation + case nl.TCA_NETEM_REORDER: + opt := nl.DeserializeTcNetemReorder(datum.Value) + netem.ReorderProb = opt.Probability + netem.ReorderCorr = opt.Correlation + } + } return nil }