diff --git a/bond/bond.go b/bond/bond.go index 1011548..fe2c722 100644 --- a/bond/bond.go +++ b/bond/bond.go @@ -173,8 +173,8 @@ func deattachLinksFromBond(linkObjectsToDeattach []netlink.Link, netNsHandle *ne return nil } -func setLinksinNetNs(bondConf *bondingConfig, nspath string, releaseLinks bool) error { - var netNs, curnetNs ns.NetNS +func setLinksInNetNs(bondConf *bondingConfig, nspath string, releaseLinks bool) error { + var podNs, hostNS ns.NetNS var err error linkNames := []string{} @@ -186,22 +186,27 @@ func setLinksinNetNs(bondConf *bondingConfig, nspath string, releaseLinks bool) linkNames = append(linkNames, s) } - if netNs, err = ns.GetNS(nspath); err != nil { + if podNs, err = ns.GetNS(nspath); err != nil { return fmt.Errorf("failed to open netns %q: %v", nspath, err) } + defer podNs.Close() - if curnetNs, err = ns.GetCurrentNS(); err != nil { + if hostNS, err = ns.GetCurrentNS(); err != nil { return fmt.Errorf("failed to get init netns: %v", err) } if releaseLinks { - if err := netNs.Set(); err != nil { - return fmt.Errorf("failed to enter netns %q: %v", netNs, err) - } + return moveLinksBetweenNs(linkNames, podNs, hostNS, "host") } + return moveLinksBetweenNs(linkNames, hostNS, podNs, "container") +} - if len(linkNames) >= 2 { // currently supporting two or more links to one bond - for _, linkName := range linkNames { +func moveLinksBetweenNs(links []string, from ns.NetNS, to ns.NetNS, toNsName string) error { + return from.Do(func(ns.NetNS) error { + if len(links) < 2 { // currently supporting two or more links to one bond + return fmt.Errorf("Bonding requires at least two links, we have %+v", len(links)) + } + for _, linkName := range links { // get interface link in the network namespace link, err := netlink.LinkByName(linkName) if err != nil { @@ -213,22 +218,13 @@ func setLinksinNetNs(bondConf *bondingConfig, nspath string, releaseLinks bool) return fmt.Errorf("failed to down link interface %q: %v", linkName, err) } - if releaseLinks { // move link inteface to network netns - if err = netlink.LinkSetNsFd(link, int(curnetNs.Fd())); err != nil { - return fmt.Errorf("failed to move link interface to host netns %q: %v", linkName, err) - } - } else { - if err = netlink.LinkSetNsFd(link, int(netNs.Fd())); err != nil { - return fmt.Errorf("failed to move link interface to container netns %q: %v", linkName, err) - } + // move link interface to network netns + if err = netlink.LinkSetNsFd(link, int(to.Fd())); err != nil { + return fmt.Errorf("failed to move link interface to %s netns %q: %v", toNsName, linkName, err) } - } - } else { - return fmt.Errorf("Bonding requires at least two links, we have %+v", len(linkNames)) - } - - return nil + return nil + }) } func createBond(bondName string, bondConf *bondingConfig, nspath string, ns ns.NetNS) (*current.Interface, error) { @@ -249,7 +245,7 @@ func createBond(bondName string, bondConf *bondingConfig, nspath string, ns ns.N defer netNsHandle.Close() if !bondConf.LinksContNs { - if err := setLinksinNetNs(bondConf, nspath, false); err != nil { + if err := setLinksInNetNs(bondConf, nspath, false); err != nil { return nil, fmt.Errorf("Failed to move the links (%+v) in container network namespace, error: %+v", bondConf.Links, err) } } @@ -423,7 +419,7 @@ func cmdDel(args *skel.CmdArgs) error { } if !bondConf.LinksContNs { - if err := setLinksinNetNs(bondConf, args.Netns, true); err != nil { + if err := setLinksInNetNs(bondConf, args.Netns, true); err != nil { return fmt.Errorf("Failed set links (%+v) in host network namespace, error: %+v", bondConf.Links, err) } } diff --git a/bond/bond_test.go b/bond/bond_test.go index 748c447..0fcefbc 100644 --- a/bond/bond_test.go +++ b/bond/bond_test.go @@ -16,6 +16,8 @@ package main import ( "fmt" + "strconv" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" types020 "github.com/containernetworking/cni/pkg/types/020" @@ -39,7 +41,7 @@ const ( "cniVersion": "%s", "mode": "%s", "failOverMac": 1, - "linksInContainer": true, + "linksInContainer": %s, "miimon": "100", "mtu": 1400, "links": [ @@ -55,267 +57,331 @@ var Slaves = []string{Slave1, Slave2} var _ = Describe("bond plugin", func() { var podNS ns.NetNS + var initNS ns.NetNS + var args *skel.CmdArgs + var linksInContainer bool + var linkAttrs = []netlink.LinkAttrs{ + {Name: Slave1}, + {Name: Slave2}, + } - BeforeEach(func() { - var err error - podNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) + AfterEach(func() { + Expect(podNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(podNS)).To(Succeed()) + }) + + When("links are in container`s network namespace at initial state (meaning linksInContainer is true)", func() { + BeforeEach(func() { + var err error + linksInContainer = true + podNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + addLinksInNS(podNS, linkAttrs) + args = &skel.CmdArgs{ + ContainerID: "dummy", + Netns: podNS.Path(), + IfName: IfName, + StdinData: []byte(fmt.Sprintf(Config, "0.3.1", ActiveBackupMode, strconv.FormatBool(true))), + } + }) + + It("verifies a plugin is added and deleted correctly", func() { + By("creating the plugin") + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + By("validationg the returned result is correct") + checkAddReturnResult(&r, IfName) - for _, ifName := range Slaves { err = podNS.Do(func(ns.NetNS) error { defer GinkgoRecover() - err = netlink.LinkAdd(&netlink.Dummy{ - LinkAttrs: netlink.LinkAttrs{ - Name: ifName, - }, - }) - Expect(err).NotTo(HaveOccurred()) - _, err := netlink.LinkByName(ifName) + By("validating the bond interface is configured correctly") + link, err := netlink.LinkByName(IfName) Expect(err).NotTo(HaveOccurred()) + bond := link.(*netlink.Bond) + Expect(bond.Attrs().MTU).To(Equal(1400)) + Expect(bond.Mode.String()).To(Equal(ActiveBackupMode)) + Expect(bond.Miimon).To(Equal(100)) + + By("validating the bond slaves are configured correctly") + for _, slaveName := range Slaves { + slave, err := netlink.LinkByName(slaveName) + Expect(err).NotTo(HaveOccurred()) + Expect(slave.Attrs().Slave).NotTo(BeNil()) + Expect(slave.Attrs().MasterIndex).To(Equal(bond.Attrs().Index)) + } return nil }) - } - Expect(err).NotTo(HaveOccurred()) - }) + Expect(err).NotTo(HaveOccurred()) - AfterEach(func() { - Expect(podNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(podNS)).To(Succeed()) - }) + By("validating the bond interface is deleted correctly") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) + Expect(err).NotTo(HaveOccurred()) - It("verifies a plugin is added and deleted correctly", func() { - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, "0.3.1", ActiveBackupMode)), - } - - By("creating the plugin") - r, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + _, err = netlink.LinkByName(IfName) + Expect(err).To(HaveOccurred()) + return nil + }) + Expect(err).NotTo(HaveOccurred()) }) - Expect(err).NotTo(HaveOccurred()) - By("validationg the returned result is correct") - checkAddReturnResult(&r, IfName) + It("verifies the plugin handles multiple del commands", func() { + By("adding a bond interface") + _, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) - err = podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() - By("validating the bond interface is configured correctly") - link, err := netlink.LinkByName(IfName) + By("deleting the bond interface") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) Expect(err).NotTo(HaveOccurred()) - bond := link.(*netlink.Bond) - Expect(bond.Attrs().MTU).To(Equal(1400)) - Expect(bond.Mode.String()).To(Equal(ActiveBackupMode)) - Expect(bond.Miimon).To(Equal(100)) - - By("validating the bond slaves are configured correctly") - for _, slaveName := range Slaves { - slave, err := netlink.LinkByName(slaveName) - Expect(err).NotTo(HaveOccurred()) - Expect(slave.Attrs().Slave).NotTo(BeNil()) - Expect(slave.Attrs().MasterIndex).To(Equal(bond.Attrs().Index)) - } - return nil - }) - Expect(err).NotTo(HaveOccurred()) - By("validating the bond interface is deleted correctly") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) + By("deleting again the bond interface") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) + Expect(err).NotTo(HaveOccurred()) - err = podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() - _, err = netlink.LinkByName(IfName) - Expect(err).To(HaveOccurred()) - return nil }) - Expect(err).NotTo(HaveOccurred()) - }) - It("verifies the plugin handles multiple del commands", func() { - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, "1.0.0", ActiveBackupMode)), - } - - By("adding a bond interface") - _, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) - }) - Expect(err).NotTo(HaveOccurred()) + It("verifies the del command does not fail when a device (in container) assigned to the bond has been deleted", func() { + By("adding a bond interface") + _, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) - By("deleting the bond interface") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() - By("deleting again the bond interface") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) + By("deleting a slave interface") + slave, err := netlink.LinkByName(Slave1) + Expect(err).NotTo(HaveOccurred()) + err = netlink.LinkDel(slave) + Expect(err).NotTo(HaveOccurred()) + return nil + }) - }) + Expect(err).NotTo(HaveOccurred()) + + By("deleting the bond interface") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) + Expect(err).NotTo(HaveOccurred()) - It("verifies the del command does not fail when a device (in container) assigned to the bond has been deleted", func() { - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, "1.0.0", ActiveBackupMode)), - } - - By("adding a bond interface") - _, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred()) - err = podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() + DescribeTable("verifies the plugin returns correct results for supported tested versions", func(version string) { - By("deleting a slave interface") - slave, err := netlink.LinkByName(Slave1) + args.StdinData = []byte(fmt.Sprintf(Config, version, ActiveBackupMode, strconv.FormatBool(linksInContainer))) + + By(fmt.Sprintf("creating the plugin with config in version %s", version)) + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) Expect(err).NotTo(HaveOccurred()) - err = netlink.LinkDel(slave) + By(fmt.Sprintf("expecting the result version to be %s", version)) + Expect(r.Version()).To(Equal(version)) + checkAddReturnResult(&r, IfName) + + By("deleting plugin") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) Expect(err).NotTo(HaveOccurred()) - return nil - }) + }, + Entry("When Version is 0.3.0", "0.3.0"), + Entry("When Version is 0.3.1", "0.3.1"), + Entry("When Version is 0.4.0", "0.4.0"), + Entry("When Version is 1.0.0", "1.0.0"), + Entry("When Version is 0.2.0", "0.2.0"), + Entry("When Version is 0.1.0", "0.1.0"), + ) + + It("verifies the plugin copes with duplicated macs in balance-tlb mode", func() { + args.StdinData = []byte(fmt.Sprintf(Config, "0.3.1", BalanceTlbMode, strconv.FormatBool(linksInContainer))) + + err := podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() - Expect(err).NotTo(HaveOccurred()) + slave1, err := netlink.LinkByName(Slave1) + Expect(err).NotTo(HaveOccurred()) - By("deleting the bond interface") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) + slave2, err := netlink.LinkByName(Slave2) + Expect(err).NotTo(HaveOccurred()) - }) + err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + By("creating the plugin") + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + + By("checking the bond was created") + checkAddReturnResult(&r, IfName) - DescribeTable("verifies the plugin returns correct results for supported tested versions", func(version string) { - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, version, ActiveBackupMode)), - } - By(fmt.Sprintf("creating the plugin with config in version %s", version)) - r, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) + Expect(err).NotTo(HaveOccurred()) }) - Expect(err).NotTo(HaveOccurred()) - By(fmt.Sprintf("expecting the result version to be %s", version)) - Expect(r.Version()).To(Equal(version)) - checkAddReturnResult(&r, IfName) - By("deleting plugin") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) - }, - Entry("When Version is 0.3.0", "0.3.0"), - Entry("When Version is 0.3.1", "0.3.1"), - Entry("When Version is 0.4.0", "0.4.0"), - Entry("When Version is 1.0.0", "1.0.0"), - Entry("When Version is 0.2.0", "0.2.0"), - Entry("When Version is 0.1.0", "0.1.0"), - ) - - It("verifies the plugin copes with duplicated macs in balance-tlb mode", func() { - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, "0.3.1", BalanceTlbMode)), - } - - err := podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() + It("verifies the plugin handles duplicated macs on delete", func() { + var slave1, slave2 netlink.Link + var err error + + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + slave1, err = netlink.LinkByName(Slave1) + Expect(err).NotTo(HaveOccurred()) - slave1, err := netlink.LinkByName(Slave1) + slave2, err = netlink.LinkByName(Slave2) + Expect(err).NotTo(HaveOccurred()) + + err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + + By("creating the plugin") + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) Expect(err).NotTo(HaveOccurred()) - slave2, err := netlink.LinkByName(Slave2) + By("checking the bond was created") + checkAddReturnResult(&r, IfName) + + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + By("duplicating the macs on the slaves") + err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + Expect(err).NotTo(HaveOccurred()) + return nil + }) + By("deleting the plugin") + err = testutils.CmdDel(podNS.Path(), + args.ContainerID, "", func() error { return cmdDel(args) }) Expect(err).NotTo(HaveOccurred()) - err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + By("validating the macs are not duplicated") + slave1, err = netlink.LinkByName(Slave1) + Expect(err).NotTo(HaveOccurred()) + slave2, err = netlink.LinkByName(Slave2) + Expect(err).NotTo(HaveOccurred()) + Expect(slave1.Attrs().HardwareAddr.String()).NotTo(Equal(slave2.Attrs().HardwareAddr.String())) + return nil + }) Expect(err).NotTo(HaveOccurred()) - return nil }) - Expect(err).NotTo(HaveOccurred()) - - By("creating the plugin") - r, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) + }) + When("links are in the initial network namespace at initial state (meaning linksInContainer is false)", func() { + BeforeEach(func() { + var err error + linksInContainer = false + podNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + initNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + addLinksInNS(initNS, linkAttrs) }) - Expect(err).NotTo(HaveOccurred()) - By("checking the bond was created") - checkAddReturnResult(&r, IfName) - - Expect(err).NotTo(HaveOccurred()) - }) + AfterEach(func() { + Expect(initNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(initNS)).To(Succeed()) + }) + It("verifies a plugin is added and deleted correctly ", func() { + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: podNS.Path(), + IfName: IfName, + StdinData: []byte(fmt.Sprintf(Config, "0.3.1", ActiveBackupMode, strconv.FormatBool(linksInContainer))), + } + err := initNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + By("creating the plugin") + r, _, err := testutils.CmdAddWithArgs(args, func() error { + return cmdAdd(args) + }) + Expect(err).NotTo(HaveOccurred()) + By("validating the returned result is correct") + checkAddReturnResult(&r, IfName) - It("verifies the plugin handles duplicated macs on delete", func() { - var slave1, slave2 netlink.Link - var err error - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: podNS.Path(), - IfName: IfName, - StdinData: []byte(fmt.Sprintf(Config, "0.3.1", ActiveBackupMode)), - } - - err = podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() - slave1, err = netlink.LinkByName(Slave1) + return nil + }) Expect(err).NotTo(HaveOccurred()) - slave2, err = netlink.LinkByName(Slave2) + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + By("validating the bond interface is configured correctly") + link, err := netlink.LinkByName(IfName) + Expect(err).NotTo(HaveOccurred()) + validateBondIFConf(link, 1400, ActiveBackupMode, 100) + + By("validating the bond slaves are configured correctly") + validateBondSlavesConf(link, Slaves) + return nil + }) Expect(err).NotTo(HaveOccurred()) - err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + err = initNS.Do(func(ns.NetNS) error { + By("validating the bond interface is deleted correctly") + err = testutils.CmdDelWithArgs(args, func() error { + return cmdDel(args) + }) + Expect(err).NotTo(HaveOccurred()) + return nil + }) Expect(err).NotTo(HaveOccurred()) - return nil - }) - By("creating the plugin") - r, _, err := testutils.CmdAddWithArgs(args, func() error { - return cmdAdd(args) - }) - Expect(err).NotTo(HaveOccurred()) + By("Checking that links are not in pod namespace") - By("checking the bond was created") - checkAddReturnResult(&r, IfName) + err = podNS.Do(func(ns.NetNS) error { + defer GinkgoRecover() + for _, slaveName := range Slaves { + _, err := netlink.LinkByName(slaveName) + Expect(err).To(HaveOccurred()) + } + return nil + }) + Expect(err).NotTo(HaveOccurred()) - err = podNS.Do(func(ns.NetNS) error { - defer GinkgoRecover() - By("duplicating the macs on the slaves") - err = netlink.LinkSetHardwareAddr(slave2, slave1.Attrs().HardwareAddr) + err = initNS.Do(func(ns.NetNS) error { + By("Checking that links are in initial namespace") + for _, slaveName := range Slaves { + _, err := netlink.LinkByName(slaveName) + Expect(err).NotTo(HaveOccurred()) + } + return nil + }) Expect(err).NotTo(HaveOccurred()) - return nil + }) - By("deleting the plugin") - err = testutils.CmdDel(podNS.Path(), - args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred()) + }) +}) - err = podNS.Do(func(ns.NetNS) error { +func addLinksInNS(initNS ns.NetNS, links []netlink.LinkAttrs) { + for _, link := range links { + var err error + err = initNS.Do(func(ns.NetNS) error { defer GinkgoRecover() - By("validating the macs are not duplicated") - slave1, err = netlink.LinkByName(Slave1) - Expect(err).NotTo(HaveOccurred()) - slave2, err = netlink.LinkByName(Slave2) + err = netlink.LinkAdd(&netlink.Dummy{ + LinkAttrs: link, + }) Expect(err).NotTo(HaveOccurred()) - Expect(slave1.Attrs().HardwareAddr.String()).NotTo(Equal(slave2.Attrs().HardwareAddr.String())) return nil }) Expect(err).NotTo(HaveOccurred()) - }) -}) + } +} func checkAddReturnResult(r *types.Result, bondIfName string) { switch result := (*r).(type) { @@ -332,3 +398,20 @@ func checkAddReturnResult(r *types.Result, bondIfName string) { Fail("Unsupported result type") } } + +func validateBondIFConf(link netlink.Link, expectedMTU int, expectedMode string, expectedMiimon int) { + bond := link.(*netlink.Bond) + Expect(bond.Attrs().MTU).To(Equal(expectedMTU)) + Expect(bond.Mode.String()).To(Equal(expectedMode)) + Expect(bond.Miimon).To(Equal(expectedMiimon)) +} + +func validateBondSlavesConf(link netlink.Link, slaves []string) { + bond := link.(*netlink.Bond) + for _, slaveName := range slaves { + slave, err := netlink.LinkByName(slaveName) + Expect(err).NotTo(HaveOccurred()) + Expect(slave.Attrs().Slave).NotTo(BeNil()) + Expect(slave.Attrs().MasterIndex).To(Equal(bond.Attrs().Index)) + } +}