Skip to content

Commit

Permalink
Merge pull request kubevirt#5514 from rhrazdil/no-fwd-livemigration-p…
Browse files Browse the repository at this point in the history 
…orts

Avoid forwarding local traffic to ports used by live migration
  • Loading branch information
@kubevirt-bot
kubevirt-bot authored May 12, 2021
2 parents 834167e + 14e970b commit ea96e7a
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 37 deletions.
97 changes: 76 additions & 21 deletions pkg/virt-launcher/virtwrap/network/podinterface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ import (

var bridgeFakeIP = "169.254.75.1%d/32"

const (
LibvirtLocalConnectionPort = 22222
LibvirtDirectMigrationPort = 49152
LibvirtBlockMigrationPort = 49153
)

type BindMechanism interface {
discoverPodNetworkInterface() error
preparePodNetworkInterface() error
Expand Down Expand Up @@ -1051,6 +1057,11 @@ func (b *MasqueradeBindMechanism) createNatRulesUsingIptables(protocol iptables.
return err
}

err = b.skipForwardingForReservedPortsUsingIptables(protocol)
if err != nil {
return err
}

if len(b.iface.Ports) == 0 {
err = b.handler.IptablesAppendRule(protocol, "nat", "KUBEVIRT_PREINBOUND",
"-j",
Expand Down Expand Up @@ -1107,28 +1118,20 @@ func (b *MasqueradeBindMechanism) createNatRulesUsingIptables(protocol iptables.
return nil
}

func (b *MasqueradeBindMechanism) getGatewayByProtocol(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return b.gatewayAddr.IP.String()
} else {
return b.gatewayIpv6Addr.IP.String()
}
}

func (b *MasqueradeBindMechanism) getVifIpByProtocol(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return b.vif.IP.IP.String()
} else {
return b.vif.IPv6.IP.String()
}
}

func getLoopbackAdrress(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return "127.0.0.1"
} else {
return "::1"
func (b *MasqueradeBindMechanism) skipForwardingForReservedPortsUsingIptables(protocol iptables.Protocol) error {
chainWhereDnatIsPerformed := "OUTPUT"
chainWhereSnatIsPerformed := "KUBEVIRT_POSTINBOUND"
for _, chain := range []string{chainWhereDnatIsPerformed, chainWhereSnatIsPerformed} {
err := b.handler.IptablesAppendRule(protocol, "nat", chain,
"-p", "tcp", "--match", "multiport",
"--dports", fmt.Sprintf("%s", strings.Join(portsUsedByLiveMigration(), ",")),
"--source", getLoopbackAdrress(protocol),
"-j", "RETURN")
if err != nil {
return err
}
}
return nil
}

func (b *MasqueradeBindMechanism) createNatRulesUsingNftables(proto iptables.Protocol) error {
Expand Down Expand Up @@ -1157,6 +1160,11 @@ func (b *MasqueradeBindMechanism) createNatRulesUsingNftables(proto iptables.Pro
return err
}

err = b.skipForwardingForReservedPortsUsingNftables(proto)
if err != nil {
return err
}

if len(b.iface.Ports) == 0 {
err = b.handler.NftablesAppendRule(proto, "nat", "KUBEVIRT_PREINBOUND",
"counter", "dnat", "to", b.getVifIpByProtocol(proto))
Expand Down Expand Up @@ -1202,6 +1210,53 @@ func (b *MasqueradeBindMechanism) createNatRulesUsingNftables(proto iptables.Pro
return nil
}

func (b *MasqueradeBindMechanism) skipForwardingForReservedPortsUsingNftables(proto iptables.Protocol) error {
chainWhereDnatIsPerformed := "output"
chainWhereSnatIsPerformed := "KUBEVIRT_POSTINBOUND"
for _, chain := range []string{chainWhereDnatIsPerformed, chainWhereSnatIsPerformed} {
err := b.handler.NftablesAppendRule(proto, "nat", chain,
"tcp", "dport", fmt.Sprintf("{ %s }", strings.Join(portsUsedByLiveMigration(), ", ")),
b.handler.GetNFTIPString(proto), "saddr", getLoopbackAdrress(proto),
"counter", "return")
if err != nil {
return err
}
}
return nil
}

func (b *MasqueradeBindMechanism) getGatewayByProtocol(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return b.gatewayAddr.IP.String()
} else {
return b.gatewayIpv6Addr.IP.String()
}
}

func (b *MasqueradeBindMechanism) getVifIpByProtocol(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return b.vif.IP.IP.String()
} else {
return b.vif.IPv6.IP.String()
}
}

func getLoopbackAdrress(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return "127.0.0.1"
} else {
return "::1"
}
}

func portsUsedByLiveMigration() []string {
return []string{
fmt.Sprint(LibvirtLocalConnectionPort),
fmt.Sprint(LibvirtDirectMigrationPort),
fmt.Sprint(LibvirtBlockMigrationPort),
}
}

type SlirpBindMechanism struct {
iface *v1.Interface
domain *api.Domain
Expand Down
12 changes: 12 additions & 0 deletions pkg/virt-launcher/virtwrap/network/podinterface_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"net"
"os"
"runtime"
"strings"

"github.com/coreos/go-iptables/iptables"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -246,6 +247,14 @@ var _ = Describe("Pod Network", func() {
mockNetwork.EXPECT().CreateTapDevice(tapDeviceName, queueNumber, pid, mtu, libvirtUser).Return(nil)
mockNetwork.EXPECT().DisableTXOffloadChecksum(bridgeTest.Name).Return(nil)
// Global nat rules using iptables
for _, proto := range ipProtocols() {
for _, chain := range []string{"OUTPUT", "KUBEVIRT_POSTINBOUND"} {
mockNetwork.EXPECT().IptablesAppendRule(proto, "nat", chain,
"-p", "tcp", "--match", "multiport",
"--dports", fmt.Sprintf("%s", strings.Join(portsUsedByLiveMigration(), ",")),
"--source", getLoopbackAdrress(proto), "-j", "RETURN").Return(nil)
}
}
mockNetwork.EXPECT().ConfigureIpForwarding(iptables.ProtocolIPv4).Return(nil)
mockNetwork.EXPECT().ConfigureIpForwarding(iptables.ProtocolIPv6).Return(nil)
mockNetwork.EXPECT().GetNFTIPString(iptables.ProtocolIPv4).Return("ip").AnyTimes()
Expand Down Expand Up @@ -282,6 +291,9 @@ var _ = Describe("Pod Network", func() {
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat", "postrouting", GetNFTIPString(proto), "saddr", GetMasqueradeVmIp(proto), "counter", "masquerade").Return(nil)
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat", "prerouting", "iifname", "eth0", "counter", "jump", "KUBEVIRT_PREINBOUND").Return(nil)
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat", "postrouting", "oifname", "k6t-eth0", "counter", "jump", "KUBEVIRT_POSTINBOUND").Return(nil)
for _, chain := range []string{"output", "KUBEVIRT_POSTINBOUND"} {
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat", chain, "tcp", "dport", fmt.Sprintf("{ %s }", strings.Join(portsUsedByLiveMigration(), ", ")), GetNFTIPString(proto), "saddr", getLoopbackAdrress(proto), "counter", "return").Return(nil)
}
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat", "KUBEVIRT_PREINBOUND", "counter", "dnat", "to", GetMasqueradeVmIp(proto)).Return(nil)

}
Expand Down
46 changes: 30 additions & 16 deletions tests/network/vmi_networking.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,12 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
var inboundVMIWithCustomMacAddress *v1.VirtualMachineInstance
var outboundVMI *v1.VirtualMachineInstance

const testPort = 1500
const (
testPort = 1500
LibvirtLocalConnectionPort = 22222
LibvirtDirectMigrationPort = 49152
LibvirtBlockMigrationPort = 49153
)

tests.BeforeAll(func() {
virtClient, err = kubecli.GetKubevirtClient()
Expand Down Expand Up @@ -671,6 +676,14 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
return nil
}

portsUsedByLiveMigration := func() []v1.Port {
return []v1.Port{
{Port: LibvirtLocalConnectionPort},
{Port: LibvirtDirectMigrationPort},
{Port: LibvirtBlockMigrationPort},
}
}

Context("[Conformance][test_id:1780][label:masquerade_binding_connectivity]should allow regular network connection", func() {

verifyClientServerConnectivity := func(clientVMI *v1.VirtualMachineInstance, serverVMI *v1.VirtualMachineInstance, tcpPort int, ipFamily k8sv1.IPFamily) error {
Expand All @@ -695,7 +708,7 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
return nil
}

table.DescribeTable("ipv4", func(ports []v1.Port, networkCIDR string) {
table.DescribeTable("ipv4", func(ports []v1.Port, tcpPort int, networkCIDR string) {
var clientVMI *v1.VirtualMachineInstance
var serverVMI *v1.VirtualMachineInstance

Expand All @@ -714,7 +727,6 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
Expect(serverVMI.Status.Interfaces[0].IPs).NotTo(BeEmpty())

By("starting a tcp server")
tcpPort := 8080
tests.StartTCPServer(serverVMI, tcpPort)

if networkCIDR == "" {
Expand All @@ -731,12 +743,13 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]

Expect(verifyClientServerConnectivity(clientVMI, serverVMI, tcpPort, k8sv1.IPv4Protocol)).To(Succeed())
},
table.Entry("with a specific port number [IPv4]", []v1.Port{{Name: "http", Port: 8080}}, ""),
table.Entry("without a specific port number [IPv4]", []v1.Port{}, ""),
table.Entry("with custom CIDR [IPv4]", []v1.Port{}, "10.10.10.0/24"),
table.Entry("with a specific port number [IPv4]", []v1.Port{{Name: "http", Port: 8080}}, 8080, ""),
table.Entry("with a specific port used by live migration", portsUsedByLiveMigration(), LibvirtLocalConnectionPort, ""),
table.Entry("without a specific port number [IPv4]", []v1.Port{}, 8080, ""),
table.Entry("with custom CIDR [IPv4]", []v1.Port{}, 8080, "10.10.10.0/24"),
)

table.DescribeTable("IPv6", func(ports []v1.Port, networkCIDR string) {
table.DescribeTable("IPv6", func(ports []v1.Port, tcpPort int, networkCIDR string) {
libnet.SkipWhenNotDualStackCluster(virtClient)
var serverVMI *v1.VirtualMachineInstance
var clientVMI *v1.VirtualMachineInstance
Expand All @@ -763,7 +776,6 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
Expect(serverVMI.Status.Interfaces[0].IPs).NotTo(BeEmpty())

By("starting a http server")
tcpPort := 8080
tests.StartPythonHttpServer(serverVMI, tcpPort)

assert.XFail("https://github.com/kubevirt/kubevirt/issues/5113", func() {
Expand All @@ -776,9 +788,10 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]

Expect(verifyClientServerConnectivity(clientVMI, serverVMI, tcpPort, k8sv1.IPv6Protocol)).To(Succeed())
},
table.Entry("with a specific port number [IPv6]", []v1.Port{{Name: "http", Port: 8080}}, ""),
table.Entry("without a specific port number [IPv6]", []v1.Port{}, ""),
table.Entry("with custom CIDR [IPv6]", []v1.Port{}, "fd10:10:10::/120"),
table.Entry("with a specific port number [IPv6]", []v1.Port{{Name: "http", Port: 8080}}, 8080, ""),
table.Entry("with a specific port used by live migration", portsUsedByLiveMigration(), LibvirtLocalConnectionPort, ""),
table.Entry("without a specific port number [IPv6]", []v1.Port{}, 8080, ""),
table.Entry("with custom CIDR [IPv6]", []v1.Port{}, 8080, "fd10:10:10::/120"),
)
})

Expand Down Expand Up @@ -837,7 +850,7 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
}
})

table.DescribeTable("[Conformance] preserves connectivity", func(ipFamily k8sv1.IPFamily) {
table.DescribeTable("[Conformance] preserves connectivity", func(ipFamily k8sv1.IPFamily, ports []v1.Port) {
if ipFamily == k8sv1.IPv6Protocol {
libnet.SkipWhenNotDualStackCluster(virtClient)
}
Expand All @@ -847,10 +860,10 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]

By("Create VMI")
if ipFamily == k8sv1.IPv4Protocol {
vmi = masqueradeVMI([]v1.Port{}, "")
vmi = masqueradeVMI(ports, "")
loginMethod = console.LoginToCirros
} else {
vmi, err = fedoraMasqueradeVMI([]v1.Port{}, "")
vmi, err = fedoraMasqueradeVMI(ports, "")
Expect(err).ToNot(HaveOccurred(), "Error creating fedora masquerade vmi")
loginMethod = console.LoginToFedora
}
Expand Down Expand Up @@ -902,8 +915,9 @@ var _ = SIGDescribe("[Serial][rfe_id:694][crit:medium][vendor:[email protected]]
}
Expect(ping(podIP)).To(Succeed())
},
table.Entry("IPv4", k8sv1.IPv4Protocol),
table.Entry("IPv6", k8sv1.IPv6Protocol),
table.Entry("IPv4", k8sv1.IPv4Protocol, []v1.Port{}),
table.Entry("IPv4 with explicit ports used by live migration", k8sv1.IPv4Protocol, portsUsedByLiveMigration()),
table.Entry("IPv6", k8sv1.IPv6Protocol, []v1.Port{}),
)
})

Expand Down

0 comments on commit ea96e7a

Please sign in to comment.