Skip to content

Commit

Permalink
Merge branch 'main' into seccomp-integration
Browse files Browse the repository at this point in the history
  • Loading branch information
Danny-Wei committed Jan 16, 2024
2 parents 3d1c329 + 5d6791c commit 1faede0
Show file tree
Hide file tree
Showing 8 changed files with 189 additions and 18 deletions.
3 changes: 2 additions & 1 deletion cmd/varmor/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,14 +251,15 @@ func main() {
go webhookServer.Run()

// The service is used for state synchronization. It only works with leader.
// TODO: support HTTPS
statusSvc, err := status.NewStatusService(
managerIP,
config.StatusServicePort,
tlsPair,
debug,
kubeClient.CoreV1(),
kubeClient.AppsV1(),
varmorClient.CrdV1beta1(),
kubeClient.AuthenticationV1(),
statusUpdateCycle,
log.Log.WithName("STATUS-SERVICE"),
)
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,13 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
<<<<<<< HEAD
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
=======
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
>>>>>>> main
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
Expand Down
2 changes: 1 addition & 1 deletion internal/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ func NewAgent(
stopCh: stopCh,
log: log,
}

varmorutils.InitAndStartTokenRotation(5*time.Minute, log)
// Pre-checks
agent.appArmorSupported, err = isLSMSupported("AppArmor")
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/behavior/modeller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
varmortracer "github.com/bytedance/vArmor/internal/behavior/tracer"
varmorutils "github.com/bytedance/vArmor/internal/utils"
varmormonitor "github.com/bytedance/vArmor/pkg/runtime"
utils "github.com/bytedance/vArmor/pkg/utils"
"github.com/bytedance/vArmor/pkg/utils"
)

type BehaviorModeller struct {
Expand Down
62 changes: 53 additions & 9 deletions internal/status/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,28 @@ package status

import (
"context"
"crypto/tls"
"fmt"
varmortls "github.com/bytedance/vArmor/internal/tls"
authv1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"net/http"
"os"
"time"

"github.com/gin-gonic/gin"
"github.com/go-logr/logr"

appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

varmorconfig "github.com/bytedance/vArmor/internal/config"
statusmanager "github.com/bytedance/vArmor/internal/status/api/v1"
varmorinterface "github.com/bytedance/vArmor/pkg/client/clientset/versioned/typed/varmor/v1beta1"
appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
authclientv1 "k8s.io/client-go/kubernetes/typed/authentication/v1"
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
)

const managerAudience = "varmor-manager"

type StatusService struct {
StatusManager *statusmanager.StatusManager
srv *http.Server
Expand All @@ -41,17 +48,46 @@ type StatusService struct {
log logr.Logger
}

func CheckAgentToken(authInterface authclientv1.AuthenticationV1Interface) gin.HandlerFunc {
return func(c *gin.Context) {
token := c.GetHeader("Token")
if token == "" {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
tr := &authv1.TokenReview{
Spec: authv1.TokenReviewSpec{
Token: token,
Audiences: []string{
managerAudience,
},
},
}
result, err := authInterface.TokenReviews().Create(context.Background(), tr, metav1.CreateOptions{})
if err != nil {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
if !result.Status.Authenticated {
c.AbortWithStatus(http.StatusUnauthorized)
return
}
c.Next()
}
}
func health(c *gin.Context) {
c.JSON(http.StatusOK, "ok")
}

func NewStatusService(
addr string,
port int,
tlsPair *varmortls.PemPair,
debug bool,
coreInterface corev1.CoreV1Interface,
appsInterface appsv1.AppsV1Interface,
varmorInterface varmorinterface.CrdV1beta1Interface,
authInterface authclientv1.AuthenticationV1Interface,
statusUpdateCycle time.Duration,
log logr.Logger) (*StatusService, error) {

Expand All @@ -77,23 +113,31 @@ func NewStatusService(
}
s.router.SetTrustedProxies(nil)

s.router.POST(varmorconfig.StatusSyncPath, statusManager.Status)
s.router.POST(varmorconfig.DataSyncPath, statusManager.Data)
s.router.POST(varmorconfig.StatusSyncPath, CheckAgentToken(authInterface), statusManager.Status)
s.router.POST(varmorconfig.DataSyncPath, CheckAgentToken(authInterface), statusManager.Data)
s.router.GET("/healthz", health)

cert, err := tls.X509KeyPair(tlsPair.Certificate, tlsPair.PrivateKey)
if err != nil {
log.Error(err, "load key pair failed")
os.Exit(1)
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
}
s.srv = &http.Server{
Addr: fmt.Sprintf("%s:%d", s.addr, s.port),
Handler: s.router,
Addr: fmt.Sprintf("%s:%d", s.addr, s.port),
Handler: s.router,
TLSConfig: tlsConfig,
}
return &s, nil
}

func (s *StatusService) Run(stopCh <-chan struct{}) {
s.log.Info("starting", "addr", s.srv.Addr)

go s.StatusManager.Run(stopCh)

if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
if err := s.srv.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed {
s.log.Error(err, "s.srv.ListenAndServe() failed")
}
}
Expand Down
67 changes: 67 additions & 0 deletions internal/utils/token.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// Copyright 2022-2023 vArmor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package utils

import (
"github.com/go-logr/logr"
"os"
"sync"
"time"
)

const BindTokenPath = "/var/run/secrets/tokens"

var (
token string
mu sync.RWMutex
updateChan chan bool
)

func InitAndStartTokenRotation(interval time.Duration, logger logr.Logger) {
updateToken(BindTokenPath, logger)
updateChan = make(chan bool)
go startTokenRotation(BindTokenPath, interval, logger, updateChan)
}

func startTokenRotation(filePath string, interval time.Duration, logger logr.Logger, update chan bool) {
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
updateToken(filePath, logger)
case <-update:
updateToken(filePath, logger)
}
}
}

func updateToken(filePath string, logger logr.Logger) {
newToken, err := os.ReadFile(filePath)
if err != nil {
logger.Error(err, "update agent bind token error")
os.Exit(1)
}

mu.Lock()
token = string(newToken)
mu.Unlock()
}

func GetToken() string {
mu.RLock()
defer mu.RUnlock()
return token
}
51 changes: 45 additions & 6 deletions internal/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package utils
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"math/rand"
Expand Down Expand Up @@ -46,25 +47,64 @@ import (
const (
httpTimeout = 3 * time.Second
retryTimes = 5
httpsServerURL = "https://%s.%s:%d%s"
httpsDebugURL = "https://%s:%d%s"
serverURL = "http://%s.%s:%d%s"
debugServerURL = "http://%s:%d%s"
)

func httpsPostWithRetryAndToken(reqBody []byte, debug bool, service string, namespace string, address string, port int, path string, retryTimes int) error {
var url string
if debug {
url = fmt.Sprintf(httpsDebugURL, address, port, path)
} else {
url = fmt.Sprintf(httpsServerURL, service, namespace, port, path)
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Timeout: httpTimeout, Transport: tr}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Token", GetToken())
var httpRsp *http.Response

for i := 0; i < retryTimes; i++ {
httpRsp, err = client.Do(httpReq)
if err == nil {
defer httpRsp.Body.Close()
if httpRsp.StatusCode == http.StatusOK {
return nil
} else if httpRsp.StatusCode == http.StatusUnauthorized {
// try update token
updateChan <- true
} else {
err = fmt.Errorf(fmt.Sprintf("http error code %d", httpRsp.StatusCode))
}
}
r := rand.Intn(60) + 20
time.Sleep(time.Duration(r) * time.Millisecond)
}

return err
}

func httpPostWithRetry(reqBody []byte, debug bool, service string, namespace string, address string, port int, path string, retryTimes int) error {
var url string
if debug {
url = fmt.Sprintf(debugServerURL, address, port, path)
} else {
url = fmt.Sprintf(serverURL, service, namespace, port, path)
}

client := &http.Client{Timeout: httpTimeout}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return err
}
httpReq.Header.Set("Content-Type", "application/json")

var httpRsp *http.Response

for i := 0; i < retryTimes; i++ {
Expand All @@ -91,13 +131,12 @@ func httpPostAndGetResponseWithRetry(reqBody []byte, debug bool, service string,
} else {
url = fmt.Sprintf(serverURL, service, namespace, port, path)
}

client := &http.Client{Timeout: httpTimeout}
httpReq, err := http.NewRequest("POST", url, bytes.NewBuffer(reqBody))
if err != nil {
return nil, err
}

httpReq.Header.Set("Content-Type", "application/json")
var httpRsp *http.Response
for i := 0; i < retryTimes; i++ {
httpRsp, err = client.Do(httpReq)
Expand Down Expand Up @@ -126,11 +165,11 @@ func RequestMLService(reqBody []byte, debug bool, address string, port int) ([]b
}

func PostStatusToStatusService(reqBody []byte, debug bool, address string, port int) error {
return httpPostWithRetry(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.StatusSyncPath, retryTimes)
return httpsPostWithRetryAndToken(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.StatusSyncPath, retryTimes)
}

func PostDataToStatusService(reqBody []byte, debug bool, address string, port int) error {
return httpPostWithRetry(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.DataSyncPath, retryTimes)
return httpsPostWithRetryAndToken(reqBody, debug, varmorconfig.StatusServiceName, varmorconfig.Namespace, address, port, varmorconfig.DataSyncPath, retryTimes)
}

func modifyDeploymentAnnotationsAndEnv(enforcer string, target varmor.Target, deploy *appsV1.Deployment, profileName string, bpfExclusiveMode bool) {
Expand Down
15 changes: 15 additions & 0 deletions manifests/varmor/templates/daemonsets/agent.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,13 @@ spec:
volumeMounts:
- mountPath: /sys/kernel/security
name: securityfs
<<<<<<< HEAD
- mountPath: /var/lib/kubelet/seccomp
name: seccomp-dir
=======
- mountPath: /var/run/secrets/tokens
name: bound-token
>>>>>>> main
{{- if .Values.appArmorLsmEnforcer.enabled }}
{{- with .Values.agent.appArmorLsmEnforcer.volumeMounts }}
{{- toYaml . | nindent 8 }}
Expand Down Expand Up @@ -86,10 +91,20 @@ spec:
path: /sys/kernel/security
type: Directory
name: securityfs
<<<<<<< HEAD
- hostPath:
path: /var/lib/kubelet/seccomp
type: DirectoryOrCreate
name: seccomp-dir
=======
- name: bound-token
projected:
sources:
- serviceAccountToken:
path: bound-token
expirationSeconds: 7200
audience: varmor-manager
>>>>>>> main
{{- if .Values.appArmorLsmEnforcer.enabled }}
{{- with .Values.agent.appArmorLsmEnforcer.volumes }}
{{- toYaml . | nindent 6 }}
Expand Down

0 comments on commit 1faede0

Please sign in to comment.