Skip to content

Commit

Permalink
Merge pull request kubevirt#6111 from rhrazdil/istio-pkg
Browse files Browse the repository at this point in the history
network, istio: Introduce pkg/network/istio package
kubevirt-bot authored Jul 28, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents dd5d379 + e00bf8b commit 9981641
Showing 17 changed files with 150 additions and 78 deletions.
2 changes: 1 addition & 1 deletion cmd/virt-launcher/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@ go_library(
"//pkg/hooks:go_default_library",
"//pkg/hotplug-disk:go_default_library",
"//pkg/ignition:go_default_library",
"//pkg/network/infraconfigurators:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/util:go_default_library",
"//pkg/virt-handler/cmd-client:go_default_library",
"//pkg/virt-launcher:go_default_library",
6 changes: 3 additions & 3 deletions cmd/virt-launcher/virt-launcher.go
Original file line number Diff line number Diff line change
@@ -51,7 +51,7 @@ import (
"kubevirt.io/kubevirt/pkg/hooks"
hotplugdisk "kubevirt.io/kubevirt/pkg/hotplug-disk"
"kubevirt.io/kubevirt/pkg/ignition"
"kubevirt.io/kubevirt/pkg/network/infraconfigurators"
"kubevirt.io/kubevirt/pkg/network/istio"
putil "kubevirt.io/kubevirt/pkg/util"
cmdclient "kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
virtlauncher "kubevirt.io/kubevirt/pkg/virt-launcher"
@@ -644,7 +644,7 @@ func terminateIstioProxy() {
return false
}
err := retry.OnError(retry.DefaultBackoff, isRetriable, func() error {
resp, err := httpClient.Post(fmt.Sprintf("http://localhost:%d/quitquitquit", infraconfigurators.EnvoyMergedPrometheusTelemetryPort), "", nil)
resp, err := httpClient.Post(fmt.Sprintf("http://localhost:%d/quitquitquit", istio.EnvoyMergedPrometheusTelemetryPort), "", nil)
if err != nil {
log.Log.Reason(err).Error("failed to request istio-proxy termination, retrying...")
return err
@@ -670,7 +670,7 @@ func istioProxyPresent(httpClient *http.Client) bool {
return false
}
err := retry.OnError(retry.DefaultBackoff, isRetriable, func() error {
resp, err := httpClient.Get(fmt.Sprintf("http://localhost:%d/healthz/ready", infraconfigurators.EnvoyHealthCheckPort))
resp, err := httpClient.Get(fmt.Sprintf("http://localhost:%d/healthz/ready", istio.EnvoyHealthCheckPort))
if err != nil {
log.Log.Reason(err).Error("error when checking for istio-proxy presence")
return err
8 changes: 0 additions & 8 deletions pkg/network/consts/BUILD.bazel

This file was deleted.

5 changes: 0 additions & 5 deletions pkg/network/consts/annotations.go

This file was deleted.

2 changes: 1 addition & 1 deletion pkg/network/infraconfigurators/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -13,8 +13,8 @@ go_library(
deps = [
"//pkg/network:go_default_library",
"//pkg/network/cache:go_default_library",
"//pkg/network/consts:go_default_library",
"//pkg/network/driver:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/network/link:go_default_library",
"//pkg/util:go_default_library",
"//pkg/virt-launcher/virtwrap/api:go_default_library",
50 changes: 10 additions & 40 deletions pkg/network/infraconfigurators/masquerade.go
Original file line number Diff line number Diff line change
@@ -13,23 +13,16 @@ import (
"kubevirt.io/client-go/log"
"kubevirt.io/kubevirt/pkg/network"
"kubevirt.io/kubevirt/pkg/network/cache"
"kubevirt.io/kubevirt/pkg/network/consts"
netdriver "kubevirt.io/kubevirt/pkg/network/driver"
"kubevirt.io/kubevirt/pkg/network/istio"
virtnetlink "kubevirt.io/kubevirt/pkg/network/link"
"kubevirt.io/kubevirt/pkg/util"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

const (
LibvirtDirectMigrationPort = 49152
LibvirtBlockMigrationPort = 49153
EnvoyAdminPort = 15000
EnvoyOutboundPort = 15001
EnvoyInboundPort = 15006
EnvoyTunnelPort = 15008
EnvoyMergedPrometheusTelemetryPort = 15020
EnvoyHealthCheckPort = 15021
EnvoyPrometheusTelemetryPort = 15090
LibvirtDirectMigrationPort = 49152
LibvirtBlockMigrationPort = 49153
)

type MasqueradePodNetworkConfigurator struct {
@@ -197,17 +190,6 @@ func (b *MasqueradePodNetworkConfigurator) createBridge() error {
return nil
}

func hasIstioSidecarInjectionEnabled(vmi *v1.VirtualMachineInstance) bool {
if val, ok := vmi.GetAnnotations()[consts.ISTIO_INJECT_ANNOTATION]; ok {
return strings.ToLower(val) == "true"
}
return false
}

func GetEnvoyLoopbackAddress() string {
return "127.0.0.6"
}

func GetLoopbackAdrress(proto iptables.Protocol) string {
if proto == iptables.ProtocolIPv4 {
return "127.0.0.1"
@@ -399,8 +381,8 @@ func (b *MasqueradePodNetworkConfigurator) createNatRulesUsingNftables(proto ipt
}

if len(b.vmiSpecIface.Ports) == 0 {
if hasIstioSidecarInjectionEnabled(b.vmi) {
err = b.skipForwardingForPortsUsingNftables(proto, PortsUsedByIstio())
if istio.ProxyInjectionEnabled(b.vmi) {
err = b.skipForwardingForPortsUsingNftables(proto, istio.ReservedPorts())
if err != nil {
return err
}
@@ -413,7 +395,7 @@ func (b *MasqueradePodNetworkConfigurator) createNatRulesUsingNftables(proto ipt
return err
}

if !hasIstioSidecarInjectionEnabled(b.vmi) {
if !istio.ProxyInjectionEnabled(b.vmi) {
err = b.handler.NftablesAppendRule(proto, "nat", "KUBEVIRT_PREINBOUND",
"counter", "dnat", "to", b.geVmIfaceIpByProtocol(proto))
if err != nil {
@@ -446,7 +428,7 @@ func (b *MasqueradePodNetworkConfigurator) createNatRulesUsingNftables(proto ipt
return err
}

if !hasIstioSidecarInjectionEnabled(b.vmi) {
if !istio.ProxyInjectionEnabled(b.vmi) {
err = b.handler.NftablesAppendRule(proto, "nat", "KUBEVIRT_PREINBOUND",
strings.ToLower(port.Protocol),
"dport",
@@ -504,15 +486,15 @@ func (b *MasqueradePodNetworkConfigurator) geVmIfaceIpByProtocol(proto iptables.

func (b *MasqueradePodNetworkConfigurator) getSrcAddressesToSnat(proto iptables.Protocol) string {
addresses := []string{getLoopbackAdrress(proto)}
if hasIstioSidecarInjectionEnabled(b.vmi) && proto == iptables.ProtocolIPv4 {
addresses = append(addresses, GetEnvoyLoopbackAddress())
if istio.ProxyInjectionEnabled(b.vmi) && proto == iptables.ProtocolIPv4 {
addresses = append(addresses, istio.GetLoopbackAddress())
}
return fmt.Sprintf("{ %s }", strings.Join(addresses, ", "))
}

func (b *MasqueradePodNetworkConfigurator) getDstAddressesToDnat(proto iptables.Protocol) (string, error) {
addresses := []string{getLoopbackAdrress(proto)}
if hasIstioSidecarInjectionEnabled(b.vmi) && proto == iptables.ProtocolIPv4 {
if istio.ProxyInjectionEnabled(b.vmi) && proto == iptables.ProtocolIPv4 {
ipv4, _, err := b.handler.ReadIPAddressesFromLink(b.podNicLink.Attrs().Name)
if err != nil {
return "", err
@@ -536,15 +518,3 @@ func portsUsedByLiveMigration() []string {
fmt.Sprint(LibvirtBlockMigrationPort),
}
}

func PortsUsedByIstio() []string {
return []string{
fmt.Sprint(EnvoyAdminPort),
fmt.Sprint(EnvoyOutboundPort),
fmt.Sprint(EnvoyInboundPort),
fmt.Sprint(EnvoyTunnelPort),
fmt.Sprint(EnvoyMergedPrometheusTelemetryPort),
fmt.Sprint(EnvoyHealthCheckPort),
fmt.Sprint(EnvoyPrometheusTelemetryPort),
}
}
13 changes: 13 additions & 0 deletions pkg/network/istio/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = [
"annotations.go",
"ports.go",
"proxy.go",
],
importpath = "kubevirt.io/kubevirt/pkg/network/istio",
visibility = ["//visibility:public"],
deps = ["//staging/src/kubevirt.io/client-go/api/v1:go_default_library"],
)
24 changes: 24 additions & 0 deletions pkg/network/istio/annotations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2021 Red Hat, Inc.
*
*/

package istio

const (
ISTIO_INJECT_ANNOTATION = "sidecar.istio.io/inject"
)
44 changes: 44 additions & 0 deletions pkg/network/istio/ports.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2021 Red Hat, Inc.
*
*/

package istio

import "fmt"

const (
EnvoyAdminPort = 15000
EnvoyOutboundPort = 15001
EnvoyInboundPort = 15006
EnvoyTunnelPort = 15008
EnvoyMergedPrometheusTelemetryPort = 15020
EnvoyHealthCheckPort = 15021
EnvoyPrometheusTelemetryPort = 15090
)

func ReservedPorts() []string {
return []string{
fmt.Sprint(EnvoyAdminPort),
fmt.Sprint(EnvoyOutboundPort),
fmt.Sprint(EnvoyInboundPort),
fmt.Sprint(EnvoyTunnelPort),
fmt.Sprint(EnvoyMergedPrometheusTelemetryPort),
fmt.Sprint(EnvoyHealthCheckPort),
fmt.Sprint(EnvoyPrometheusTelemetryPort),
}
}
37 changes: 37 additions & 0 deletions pkg/network/istio/proxy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* This file is part of the KubeVirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2021 Red Hat, Inc.
*
*/

package istio

import (
"strings"

v1 "kubevirt.io/client-go/api/v1"
)

func ProxyInjectionEnabled(vmi *v1.VirtualMachineInstance) bool {
if val, ok := vmi.GetAnnotations()[ISTIO_INJECT_ANNOTATION]; ok {
return strings.ToLower(val) == "true"
}
return false
}

func GetLoopbackAddress() string {
return "127.0.0.6"
}
4 changes: 2 additions & 2 deletions pkg/virt-controller/services/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -14,7 +14,7 @@ go_library(
"//pkg/downwardmetrics:go_default_library",
"//pkg/hooks:go_default_library",
"//pkg/host-disk:go_default_library",
"//pkg/network/consts:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/util:go_default_library",
"//pkg/util/hardware:go_default_library",
"//pkg/util/net/dns:go_default_library",
@@ -46,7 +46,7 @@ go_test(
deps = [
"//pkg/config:go_default_library",
"//pkg/hooks:go_default_library",
"//pkg/network/consts:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/testutils:go_default_library",
"//pkg/util:go_default_library",
"//pkg/virt-config:go_default_library",
4 changes: 2 additions & 2 deletions pkg/virt-controller/services/template.go
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ import (
"kubevirt.io/kubevirt/pkg/config"
containerdisk "kubevirt.io/kubevirt/pkg/container-disk"
"kubevirt.io/kubevirt/pkg/hooks"
"kubevirt.io/kubevirt/pkg/network/consts"
"kubevirt.io/kubevirt/pkg/network/istio"
"kubevirt.io/kubevirt/pkg/util"
"kubevirt.io/kubevirt/pkg/util/hardware"
"kubevirt.io/kubevirt/pkg/util/net/dns"
@@ -1418,7 +1418,7 @@ func (t *templateService) renderLaunchManifest(vmi *v1.VirtualMachineInstance, t
pod.Spec.ServiceAccountName = serviceAccountName
automount := true
pod.Spec.AutomountServiceAccountToken = &automount
} else if val, ok := vmi.GetAnnotations()[consts.ISTIO_INJECT_ANNOTATION]; ok && strings.ToLower(val) == "true" {
} else if istio.ProxyInjectionEnabled(vmi) {
automount := true
pod.Spec.AutomountServiceAccountToken = &automount
} else {
4 changes: 2 additions & 2 deletions pkg/virt-controller/services/template_test.go
Original file line number Diff line number Diff line change
@@ -49,7 +49,7 @@ import (
fakenetworkclient "kubevirt.io/client-go/generated/network-attachment-definition-client/clientset/versioned/fake"
"kubevirt.io/client-go/kubecli"
"kubevirt.io/kubevirt/pkg/hooks"
networkconsts "kubevirt.io/kubevirt/pkg/network/consts"
"kubevirt.io/kubevirt/pkg/network/istio"
"kubevirt.io/kubevirt/pkg/testutils"
"kubevirt.io/kubevirt/pkg/util"
virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
@@ -1001,7 +1001,7 @@ var _ = Describe("Template", func() {
Namespace: "default",
UID: "1234",
Annotations: map[string]string{
networkconsts.ISTIO_INJECT_ANNOTATION: "true",
istio.ISTIO_INJECT_ANNOTATION: "true",
},
},
}
2 changes: 1 addition & 1 deletion pkg/virt-launcher/virtwrap/network/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -37,11 +37,11 @@ go_test(
"//pkg/ephemeral-disk-utils:go_default_library",
"//pkg/network:go_default_library",
"//pkg/network/cache:go_default_library",
"//pkg/network/consts:go_default_library",
"//pkg/network/dhcp:go_default_library",
"//pkg/network/driver:go_default_library",
"//pkg/network/errors:go_default_library",
"//pkg/network/infraconfigurators:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/virt-launcher/virtwrap/api:go_default_library",
"//staging/src/kubevirt.io/client-go/api/v1:go_default_library",
"//staging/src/kubevirt.io/client-go/testutils:go_default_library",
13 changes: 6 additions & 7 deletions pkg/virt-launcher/virtwrap/network/podinterface_test.go
Original file line number Diff line number Diff line change
@@ -27,8 +27,6 @@ import (
"runtime"
"strings"

"kubevirt.io/kubevirt/pkg/network/consts"

"github.com/coreos/go-iptables/iptables"

"github.com/golang/mock/gomock"
@@ -42,6 +40,7 @@ import (
"kubevirt.io/kubevirt/pkg/network/cache"
netdriver "kubevirt.io/kubevirt/pkg/network/driver"
"kubevirt.io/kubevirt/pkg/network/infraconfigurators"
"kubevirt.io/kubevirt/pkg/network/istio"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

@@ -488,7 +487,7 @@ var _ = Describe("Pod Network", func() {

for _, chain := range []string{"output", "KUBEVIRT_POSTINBOUND"} {
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat",
chain, "tcp", "dport", fmt.Sprintf("{ %s }", strings.Join(infraconfigurators.PortsUsedByIstio(), ", ")),
chain, "tcp", "dport", fmt.Sprintf("{ %s }", strings.Join(istio.ReservedPorts(), ", ")),
GetNFTIPString(proto), "saddr", infraconfigurators.GetLoopbackAdrress(proto), "counter", "return").Return(nil)
}

@@ -498,7 +497,7 @@ var _ = Describe("Pod Network", func() {
srcAddressesToSnat := []string{infraconfigurators.GetLoopbackAdrress(proto)}
dstAddressesToDnat := []string{infraconfigurators.GetLoopbackAdrress(proto)}
if proto == iptables.ProtocolIPv4 {
srcAddressesToSnat = append(srcAddressesToSnat, infraconfigurators.GetEnvoyLoopbackAddress())
srcAddressesToSnat = append(srcAddressesToSnat, istio.GetLoopbackAddress())
dstAddressesToDnat = append(dstAddressesToDnat, fakeAddr.IP.String())
}
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat",
@@ -515,7 +514,7 @@ var _ = Describe("Pod Network", func() {
domain := NewDomainWithBridgeInterface()
vm := newVMIMasqueradeInterface("testnamespace", "testVmName", masqueradeCidr, masqueradeIpv6Cidr)
vm.Annotations = map[string]string{
consts.ISTIO_INJECT_ANNOTATION: "true",
istio.ISTIO_INJECT_ANNOTATION: "true",
}

api.NewDefaulter(runtime.GOARCH).SetObjectDefaults_Domain(domain)
@@ -534,7 +533,7 @@ var _ = Describe("Pod Network", func() {
srcAddressesToSnat := []string{infraconfigurators.GetLoopbackAdrress(proto)}
dstAddressesToDnat := []string{infraconfigurators.GetLoopbackAdrress(proto)}
if proto == iptables.ProtocolIPv4 {
srcAddressesToSnat = append(srcAddressesToSnat, infraconfigurators.GetEnvoyLoopbackAddress())
srcAddressesToSnat = append(srcAddressesToSnat, istio.GetLoopbackAddress())
dstAddressesToDnat = append(dstAddressesToDnat, fakeAddr.IP.String())
}
mockNetwork.EXPECT().NftablesAppendRule(proto, "nat",
@@ -557,7 +556,7 @@ var _ = Describe("Pod Network", func() {
vm := newVMIMasqueradeInterface("testnamespace", "testVmName", masqueradeCidr, masqueradeIpv6Cidr)
vm.Spec.Domain.Devices.Interfaces[0].Ports = []v1.Port{{Name: "test", Port: 80, Protocol: "TCP"}}
vm.Annotations = map[string]string{
consts.ISTIO_INJECT_ANNOTATION: "true",
istio.ISTIO_INJECT_ANNOTATION: "true",
}

api.NewDefaulter(runtime.GOARCH).SetObjectDefaults_Domain(domain)
3 changes: 1 addition & 2 deletions tests/network/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -20,8 +20,7 @@ go_library(
importpath = "kubevirt.io/kubevirt/tests/network",
visibility = ["//visibility:public"],
deps = [
"//pkg/network/consts:go_default_library",
"//pkg/network/infraconfigurators:go_default_library",
"//pkg/network/istio:go_default_library",
"//pkg/virt-config:go_default_library",
"//pkg/virt-controller/services:go_default_library",
"//pkg/virt-launcher/virtwrap/api:go_default_library",
7 changes: 3 additions & 4 deletions tests/network/vmi_istio.go
Original file line number Diff line number Diff line change
@@ -42,8 +42,7 @@ import (

v1 "kubevirt.io/client-go/api/v1"
"kubevirt.io/client-go/kubecli"
"kubevirt.io/kubevirt/pkg/network/consts"
"kubevirt.io/kubevirt/pkg/network/infraconfigurators"
"kubevirt.io/kubevirt/pkg/network/istio"
"kubevirt.io/kubevirt/tests"
"kubevirt.io/kubevirt/tests/console"
"kubevirt.io/kubevirt/tests/libnet"
@@ -57,7 +56,7 @@ const (
svcUndeclaredTestPort = 1501
// Istio uses certain ports for it's own purposes, this port server to verify that traffic is not routed
// into the VMI for these ports. https://istio.io/latest/docs/ops/deployment/requirements/
istioRestrictedPort = infraconfigurators.EnvoyTunnelPort
istioRestrictedPort = istio.EnvoyTunnelPort
)

var _ = SIGDescribe("[Serial] Istio", func() {
@@ -376,7 +375,7 @@ func newVMIWithIstioSidecar(ports []v1.Port) *v1.VirtualMachineInstance {
libvmi.WithNetwork(v1.DefaultPodNetwork()),
libvmi.WithInterface(libvmi.InterfaceDeviceWithMasqueradeBinding(ports...)),
libvmi.WithLabel("app", vmiAppSelector),
libvmi.WithAnnotation(consts.ISTIO_INJECT_ANNOTATION, "true"),
libvmi.WithAnnotation(istio.ISTIO_INJECT_ANNOTATION, "true"),
)
return vmi
}

0 comments on commit 9981641

Please sign in to comment.