Skip to content

Commit

Permalink
WIP - implement kubevirt proxy
Browse files Browse the repository at this point in the history
Signed-off-by: Andrej Krejcir <akrejcir@redhat.com>
  • Loading branch information
akrejcir committed Apr 27, 2023
1 parent ce7013f commit c79d26d
Show file tree
Hide file tree
Showing 15 changed files with 904 additions and 44 deletions.
38 changes: 38 additions & 0 deletions manifests/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,48 @@ rules:
- kubevirt.io
resources:
- virtualmachineinstances
- virtualmachines
verbs:
- get
- list
- watch
- apiGroups:
- subresources.kubevirt.io
resources:
- virtualmachineinstances/vnc
verbs:
- get
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- get
- list
- watch
- create
- update
- delete
- patch
- apiGroups:
- ""
resources:
- serviceaccounts/token
verbs:
- create
- apiGroups:
- rbac.authorization.k8s.io
resources:
- roles
- rolebindings
verbs:
- get
- list
- watch
- create
- update
- delete
- patch
- apiGroups:
- authentication.k8s.io
resources:
Expand Down
3 changes: 3 additions & 0 deletions pkg/console/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,15 +160,18 @@ func (c *configWatch) reloadUseSaTokensLocked() {
}
if err != nil {
c.useSaTokensError = fmt.Errorf("error reading use-service-account-tokens file: %w", err)
log.Log.Errorf("Failed to load configuration: %s", c.useSaTokensError)
return
}

useSaTokens, err := strconv.ParseBool(string(useSaTokensData))
if err != nil {
c.useSaTokensError = fmt.Errorf("error parsing use-service-account-tokens: %w", err)
log.Log.Errorf("Failed to load configuration: %s", c.useSaTokensError)
return
}

log.Log.Infof("Loaded configuration.")
c.useSaTokens = useSaTokens
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/console/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ var _ = Describe("Config", func() {
err = os.WriteFile(useSeTokensPath, []byte(strconv.FormatBool(changedVal)), 0666)
Expect(err).ToNot(HaveOccurred())

mockWatch.Trigger(useSeTokensPath)
mockWatch.Trigger(configDir)

changedConfig, err := configObj.GetUseServiceAccountTokens()
Expect(err).ToNot(HaveOccurred())
Expand All @@ -227,7 +227,7 @@ var _ = Describe("Config", func() {
err := os.WriteFile(useSeTokensPath, []byte("definitely-not-a-bool"), 0666)
Expect(err).ToNot(HaveOccurred())

mockWatch.Trigger(useSeTokensPath)
mockWatch.Trigger(configDir)

_, err = configObj.GetUseServiceAccountTokens()
Expect(err).To(MatchError(ContainSubstring("error parsing use-service-account-tokens")))
Expand Down Expand Up @@ -292,7 +292,7 @@ var _ = Describe("Config", func() {
It("should use default use-service-account-tokens if file is deleted", func() {
Expect(os.Remove(useSeTokensPath)).ToNot(HaveOccurred())

mockWatch.Trigger(useSeTokensPath)
mockWatch.Trigger(configDir)

config, err := configObj.GetUseServiceAccountTokens()
Expect(err).ToNot(HaveOccurred())
Expand Down
2 changes: 1 addition & 1 deletion pkg/console/console.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func Run() error {
}
return tokenKey, nil
}),
service.NewKubevirtProxy(),
service.NewKubevirtProxy(cli),
configWatch.GetUseServiceAccountTokens,
)

Expand Down
7 changes: 7 additions & 0 deletions pkg/console/service/constants.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package service

import "time"

const (
defaultTokenDuration = 10 * time.Minute
)
2 changes: 1 addition & 1 deletion pkg/console/service/direct-proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (d *directProxy) TokenHandler(request *restful.Request, response *restful.R
return
}

duration := 10 * time.Minute
duration := defaultTokenDuration
durationParam := request.QueryParameter("duration")
if durationParam != "" {
var err error
Expand Down
244 changes: 235 additions & 9 deletions pkg/console/service/kubevirt-proxy.go
Original file line number Diff line number Diff line change
@@ -1,29 +1,255 @@
package service

import (
"context"
"fmt"
"net/http"
"time"

"github.com/emicklei/go-restful/v3"
authnv1 "k8s.io/api/authentication/v1"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
kubevirt "kubevirt.io/api/core"
kubevirtv1 "kubevirt.io/api/core/v1"
"kubevirt.io/client-go/kubecli"

"github.com/kubevirt/vm-console-proxy/api/v1alpha1"
)

const (
AppKubernetesNameLabel = "app.kubernetes.io/name"
AppKubernetesPartOfLabel = "app.kubernetes.io/part-of"
AppKubernetesVersionLabel = "app.kubernetes.io/version"
AppKubernetesManagedByLabel = "app.kubernetes.io/managed-by"
AppKubernetesComponentLabel = "app.kubernetes.io/component"
)

func NewKubevirtProxy() Service {
return &kubevirtProxy{}
func NewKubevirtProxy(kubevirtClient kubecli.KubevirtClient) Service {
return &kubevirtProxy{
kubevirtClient: kubevirtClient,
}
}

type kubevirtProxy struct {
kubevirtClient kubecli.KubevirtClient
}

var _ Service = &kubevirtProxy{}

func (k *kubevirtProxy) TokenHandler(request *restful.Request, response *restful.Response) {
//TODO: implement
panic("not implemented")
err := ValidateVncRbac(request, k.kubevirtClient)
if err != nil {
_ = response.WriteError(http.StatusUnauthorized, err)
return
}

namespace := request.PathParameter("namespace")
name := request.PathParameter("name")

duration := defaultTokenDuration
durationParam := request.QueryParameter("duration")
if durationParam != "" {
var err error
duration, err = time.ParseDuration(durationParam)
if err != nil {
_ = response.WriteError(http.StatusBadRequest, fmt.Errorf("failed to parse duration: %w", err))
return
}
}

vm, err := k.kubevirtClient.VirtualMachine(namespace).Get(name, &metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
_ = response.WriteError(http.StatusNotFound, fmt.Errorf("VirtualMachine does not exist: %w", err))
return
}
_ = response.WriteError(http.StatusInternalServerError, fmt.Errorf("error getting VirtualMachine: %w", err))
return
}

serviceAccountName := vm.Name + "-vnc-access"
err = k.createResources(request.Request.Context(), serviceAccountName, vm)
if err != nil {
_ = response.WriteError(http.StatusInternalServerError, fmt.Errorf("error creating resources: %w", err))
return
}

durationSeconds := int64(duration.Seconds())
tokenRequest := &authnv1.TokenRequest{
Spec: authnv1.TokenRequestSpec{
Audiences: nil,
ExpirationSeconds: &durationSeconds,
BoundObjectRef: nil,
},
}

tokenRequest, err = k.kubevirtClient.CoreV1().ServiceAccounts(namespace).CreateToken(
request.Request.Context(),
serviceAccountName,
tokenRequest,
metav1.CreateOptions{},
)
if err != nil {
_ = response.WriteError(http.StatusInternalServerError, fmt.Errorf("failed to create token: %w", err))
return
}

_ = response.WriteAsJson(&v1alpha1.TokenResponse{
Token: tokenRequest.Status.Token,
})
}

func (k *kubevirtProxy) VncHandler(request *restful.Request, response *restful.Response) {
//TODO: implement
panic("not implemented")
func (k *kubevirtProxy) VncHandler(_ *restful.Request, response *restful.Response) {
_ = response.WriteError(http.StatusNotFound, fmt.Errorf(
"/vnc endpoint is not available, when the VM console proxy is configured to use kubernetes API to generate tokens",
))
}

func (k *kubevirtProxy) createResources(ctx context.Context, name string, vmMeta metav1.Object) error {
namespace := vmMeta.GetNamespace()
commonLabels := map[string]string{
AppKubernetesNameLabel: "vm-console-proxy",
AppKubernetesPartOfLabel: "vm-console-proxy",
// TODO -- add version vv
//AppKubernetesVersionLabel: "TODO",
AppKubernetesManagedByLabel: "vm-console-proxy",
AppKubernetesComponentLabel: "vm-console-proxy",
}

vmOwnerRef := metav1.OwnerReference{
APIVersion: kubevirt.GroupName,
Kind: "VirtualMachine",
Name: vmMeta.GetName(),
UID: vmMeta.GetUID(),
}

serviceAccount, err := createOrUpdate[*core.ServiceAccount](
ctx,
name,
namespace,
k.kubevirtClient.CoreV1().ServiceAccounts(namespace),
func(foundObj *core.ServiceAccount) {
foundObj.Labels = commonLabels
foundObj.OwnerReferences = []metav1.OwnerReference{vmOwnerRef}
},
)
if err != nil {
return fmt.Errorf("failed to create service account: %w", err)
}

role, err := createOrUpdate[*rbac.Role](
ctx,
name,
namespace,
k.kubevirtClient.RbacV1().Roles(namespace),
func(foundObj *rbac.Role) {
foundObj.Labels = commonLabels
foundObj.OwnerReferences = []metav1.OwnerReference{vmOwnerRef}
foundObj.Rules = []rbac.PolicyRule{{
APIGroups: []string{kubevirtv1.SubresourceGroupName},
Resources: []string{"virtualmachineinstances/vnc"},
ResourceNames: []string{vmMeta.GetName()},
Verbs: []string{"get"},
}}
},
)
if err != nil {
return fmt.Errorf("failed to create role: %w", err)
}

_, err = createOrUpdate[*rbac.RoleBinding](
ctx,
name,
namespace,
k.kubevirtClient.RbacV1().RoleBindings(namespace),
func(foundObj *rbac.RoleBinding) {
foundObj.Labels = commonLabels
foundObj.OwnerReferences = []metav1.OwnerReference{vmOwnerRef}
foundObj.Subjects = []rbac.Subject{{
Kind: "ServiceAccount",
Name: serviceAccount.Name,
}}
foundObj.RoleRef = rbac.RoleRef{
APIGroup: rbac.GroupName,
Kind: "Role",
Name: role.Name,
}
},
)
if err != nil {
return fmt.Errorf("failed to create role binding: %w", err)
}
return nil
}

type clientInterface[PT any] interface {
Create(ctx context.Context, obj PT, opts metav1.CreateOptions) (PT, error)
Update(ctx context.Context, obj PT, opts metav1.UpdateOptions) (PT, error)
Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
Get(ctx context.Context, name string, opts metav1.GetOptions) (PT, error)
}

func createOrUpdate[PT interface {
*T
metav1.Object
runtime.Object
}, T any](ctx context.Context, objName string, objNamespace string, client clientInterface[PT], mutateFn func(PT)) (PT, error) {
return retryOnConflict(ctx, retry.DefaultRetry, func() (PT, error) {
foundObj, err := client.Get(ctx, objName, metav1.GetOptions{})
if errors.IsNotFound(err) {
newObj := PT(new(T))
newObj.SetName(objName)
newObj.SetNamespace(objNamespace)
mutateFn(newObj)
return client.Create(ctx, newObj, metav1.CreateOptions{})
}
if err != nil {
return foundObj, err
}

copyObj := foundObj.DeepCopyObject().(PT)
mutateFn(foundObj)

if equality.Semantic.DeepEqual(foundObj, copyObj) {
return foundObj, nil
}

return client.Update(ctx, foundObj, metav1.UpdateOptions{})
})
}

func retryOnConflict[T any](ctx context.Context, backoff wait.Backoff, fn func() (T, error)) (T, error) {
var result T
var lastErr error
err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) {
var err error
result, err = fn()

switch {
case err == nil:
return true, nil
case errors.IsConflict(err):
lastErr = err
return false, nil
default:
return false, err
}
})
if err == wait.ErrWaitTimeout {
return result, lastErr
}
return result, err
}

func (k *kubevirtProxy) InfoHandler(request *restful.Request, response *restful.Response) {
//TODO implement me
panic("implement me")
_ = response.WriteAsJson(&v1alpha1.InfoResponse{
TokenGeneration: v1alpha1.KubernetesTokenGeneration,
})
}
Loading

0 comments on commit c79d26d

Please sign in to comment.