Skip to content

Commit

Permalink
Merge pull request #19 from UgOrange/tls_support
Browse files Browse the repository at this point in the history
feat: Implement TLS for interactions between agent and manager
  • Loading branch information
Danny-Wei authored Jan 12, 2024
2 parents 6070662 + 185ea55 commit 5d6791c
Show file tree
Hide file tree
Showing 9 changed files with 178 additions and 831 deletions.
3 changes: 2 additions & 1 deletion cmd/varmor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,14 +251,15 @@ func main() {
go webhookServer.Run()

// The service is used for state synchronization. It only works with leader.
// TODO: support HTTPS
statusSvc, err := status.NewStatusService(
managerIP,
config.StatusServicePort,
tlsPair,
debug,
kubeClient.CoreV1(),
kubeClient.AppsV1(),
varmorClient.CrdV1beta1(),
kubeClient.AuthenticationV1(),
statusUpdateCycle,
log.Log.WithName("STATUS-SERVICE"),
)
Expand Down
6 changes: 0 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ require (
github.com/gin-gonic/gin v1.9.1
github.com/go-logr/logr v1.3.0
github.com/hashicorp/go-version v1.6.0
github.com/jinzhu/copier v0.3.5
github.com/julienschmidt/httprouter v1.3.0
github.com/kubearmor/KubeArmor/KubeArmor v0.0.0-20220103065246-e88285448f28
github.com/kyverno/kyverno v1.7.4
github.com/opencontainers/runtime-spec v1.1.0-rc.1
github.com/pkg/errors v0.9.1
Expand Down Expand Up @@ -66,7 +64,6 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/kubearmor/KubeArmor/protobuf v0.0.0-20211217093440-d99a1cb5f908 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
Expand All @@ -89,9 +86,6 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.17.0 // indirect
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect
Expand Down
807 changes: 0 additions & 807 deletions go.sum

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func NewAgent(
stopCh: stopCh,
log: log,
}

varmorutils.InitAndStartTokenRotation(5*time.Minute, log)
// Pre-checks
agent.appArmorSupported, err = isLSMSupported("AppArmor")
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/behavior/modeller.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
varmortracer "github.com/bytedance/vArmor/internal/behavior/tracer"
varmorutils "github.com/bytedance/vArmor/internal/utils"
varmormonitor "github.com/bytedance/vArmor/pkg/runtime"
utils "github.com/bytedance/vArmor/pkg/utils"
"github.com/bytedance/vArmor/pkg/utils"
)

type BehaviorModeller struct {
Expand Down
62 changes: 53 additions & 9 deletions internal/status/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,28 @@ package status

import (
"context"
"crypto/tls"
"fmt"
varmortls "github.com/bytedance/vArmor/internal/tls"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http"
"os"
"time"

"github.com/gin-gonic/gin"
"github.com/go-logr/logr"

appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

varmorconfig "github.com/bytedance/vArmor/internal/config"
statusmanager "github.com/bytedance/vArmor/internal/status/api/v1"
varmorinterface "github.com/bytedance/vArmor/pkg/client/clientset/versioned/typed/varmor/v1beta1"
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
authclientv1 "k8s.io/client-go/kubernetes/typed/authentication/v1"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

const managerAudience = "varmor-manager"

type StatusService struct {
StatusManager *statusmanager.StatusManager
srv *http.Server
Expand All @@ -41,17 +48,46 @@ type StatusService struct {
log logr.Logger
}

func CheckAgentToken(authInterface authclientv1.AuthenticationV1Interface) gin.HandlerFunc {
return func(c *gin.Context) {
token := c.GetHeader("Token")
if token == "" {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
tr := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: token,
Audiences: []string{
managerAudience,
},
},
}
result, err := authInterface.TokenReviews().Create(context.Background(), tr, metav1.CreateOptions{})
if err != nil {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
if !result.Status.Authenticated {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
c.Next()
}
}
func health(c *gin.Context) {
c.JSON(http.StatusOK, "ok")
}

func NewStatusService(
addr string,
port int,
tlsPair *varmortls.PemPair,
debug bool,
coreInterface corev1.CoreV1Interface,
appsInterface appsv1.AppsV1Interface,
varmorInterface varmorinterface.CrdV1beta1Interface,
authInterface authclientv1.AuthenticationV1Interface,
statusUpdateCycle time.Duration,
log logr.Logger) (*StatusService, error) {

Expand All @@ -77,23 +113,31 @@ func NewStatusService(
}
s.router.SetTrustedProxies(nil)

s.router.POST(varmorconfig.StatusSyncPath, statusManager.Status)
s.router.POST(varmorconfig.DataSyncPath, statusManager.Data)
s.router.POST(varmorconfig.StatusSyncPath, CheckAgentToken(authInterface), statusManager.Status)
s.router.POST(varmorconfig.DataSyncPath, CheckAgentToken(authInterface), statusManager.Data)
s.router.GET("/healthz", health)

cert, err := tls.X509KeyPair(tlsPair.Certificate, tlsPair.PrivateKey)
if err != nil {
log.Error(err, "load key pair failed")
os.Exit(1)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
}
s.srv = &http.Server{
Addr: fmt.Sprintf("%s:%d", s.addr, s.port),
Handler: s.router,
Addr: fmt.Sprintf("%s:%d", s.addr, s.port),
Handler: s.router,
TLSConfig: tlsConfig,
}
return &s, nil
}

func (s *StatusService) Run(stopCh <-chan struct{}) {
s.log.Info("starting", "addr", s.srv.Addr)

go s.StatusManager.Run(stopCh)

if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
if err := s.srv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
s.log.Error(err, "s.srv.ListenAndServe() failed")
}
}
Expand Down
67 changes: 67 additions & 0 deletions internal/utils/token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2022-2023 vArmor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"github.com/go-logr/logr"
"os"
"sync"
"time"
)

const BindTokenPath = "/var/run/secrets/tokens"

var (
token string
mu sync.RWMutex
updateChan chan bool
)

func InitAndStartTokenRotation(interval time.Duration, logger logr.Logger) {
updateToken(BindTokenPath, logger)
updateChan = make(chan bool)
go startTokenRotation(BindTokenPath, interval, logger, updateChan)
}

func startTokenRotation(filePath string, interval time.Duration, logger logr.Logger, update chan bool) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
updateToken(filePath, logger)
case <-update:
updateToken(filePath, logger)
}
}
}

func updateToken(filePath string, logger logr.Logger) {
newToken, err := os.ReadFile(filePath)
if err != nil {
logger.Error(err, "update agent bind token error")
os.Exit(1)
}

mu.Lock()
token = string(newToken)
mu.Unlock()
}

func GetToken() string {
mu.RLock()
defer mu.RUnlock()
return token
}
51 changes: 45 additions & 6 deletions internal/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package utils
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"math/rand"
Expand Down Expand Up @@ -44,25 +45,64 @@ import (
const (
httpTimeout = 3 * time.Second
retryTimes = 5
httpsServerURL = "https://%s.%s:%d%s"
httpsDebugURL = "https://%s:%d%s"
serverURL = "http://%s.%s:%d%s"
debugServerURL = "http://%s:%d%s"
)

func httpsPostWithRetryAndToken(reqBody []byte, debug bool, service string, namespace string, address string, port int, path string, retryTimes int) error {
var url string
if debug {
url = fmt.Sprintf(httpsDebugURL, address, port, path)
} else {
url = fmt.Sprintf(httpsServerURL, service, namespace, port, path)
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Timeout: httpTimeout, Transport: tr}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Token", GetToken())
var httpRsp *http.Response

for i := 0; i < retryTimes; i++ {
httpRsp, err = client.Do(httpReq)
if err == nil {
defer httpRsp.Body.Close()
if httpRsp.StatusCode == http.StatusOK {
return nil
} else if httpRsp.StatusCode == http.StatusUnauthorized {
// try update token
updateChan <- true
} else {
err = fmt.Errorf(fmt.Sprintf("http error code %d", httpRsp.StatusCode))
}
}
r := rand.Intn(60) + 20
time.Sleep(time.Duration(r) * time.Millisecond)
}

return err
}

func httpPostWithRetry(reqBody []byte, debug bool, service string, namespace string, address string, port int, path string, retryTimes int) error {
var url string
if debug {
url = fmt.Sprintf(debugServerURL, address, port, path)
} else {
url = fmt.Sprintf(serverURL, service, namespace, port, path)
}

client := &http.Client{Timeout: httpTimeout}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", "application/json")

var httpRsp *http.Response

for i := 0; i < retryTimes; i++ {
Expand All @@ -89,13 +129,12 @@ func httpPostAndGetResponseWithRetry(reqBody []byte, debug bool, service string,
} else {
url = fmt.Sprintf(serverURL, service, namespace, port, path)
}

client := &http.Client{Timeout: httpTimeout}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return nil, err
}

httpReq.Header.Set("Content-Type", "application/json")
var httpRsp *http.Response
for i := 0; i < retryTimes; i++ {
httpRsp, err = client.Do(httpReq)
Expand Down Expand Up @@ -124,11 +163,11 @@ func RequestMLService(reqBody []byte, debug bool, address string, port int) ([]b
}

func PostStatusToStatusService(reqBody []byte, debug bool, address string, port int) error {
return httpPostWithRetry(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.StatusSyncPath, retryTimes)
return httpsPostWithRetryAndToken(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.StatusSyncPath, retryTimes)
}

func PostDataToStatusService(reqBody []byte, debug bool, address string, port int) error {
return httpPostWithRetry(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.DataSyncPath, retryTimes)
return httpsPostWithRetryAndToken(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.DataSyncPath, retryTimes)
}

func modifyDeploymentAnnotationsAndEnv(enforcer string, target varmor.Target, deploy *appsV1.Deployment, profileName string, bpfExclusiveMode bool) {
Expand Down
9 changes: 9 additions & 0 deletions manifests/varmor/templates/daemonsets/agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ spec:
volumeMounts:
- mountPath: /sys/kernel/security
name: securityfs
- mountPath: /var/run/secrets/tokens
name: bound-token
{{- if .Values.appArmorLsmEnforcer.enabled }}
{{- with .Values.agent.appArmorLsmEnforcer.volumeMounts }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -84,6 +86,13 @@ spec:
path: /sys/kernel/security
type: Directory
name: securityfs
- name: bound-token
projected:
sources:
- serviceAccountToken:
path: bound-token
expirationSeconds: 7200
audience: varmor-manager
{{- if .Values.appArmorLsmEnforcer.enabled }}
{{- with .Values.agent.appArmorLsmEnforcer.volumes }}
{{- toYaml . | nindent 6 }}
Expand Down

0 comments on commit 5d6791c

Please sign in to comment.