feat(controller): add gc (goharbor#808)

add gc Signed-off-by: cndoit18 <cndoit18@outlook.com>
thcdrt · Nov 26, 2021 · d7f6e61 · d7f6e61
1 parent 4678dd6
commit d7f6e61
Show file tree

Hide file tree

Showing 8 changed files with 254 additions and 17 deletions.
diff --git a/pkg/controller/common.go b/pkg/controller/common.go
@@ -19,6 +19,7 @@ import (
 	"github.com/pkg/errors"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -35,6 +36,8 @@ type Controller struct {
 	Version        string
 	GitCommit      string
 
+	deletableResources map[schema.GroupVersionKind]struct{}
+
 	ConfigStore *configstore.Store
 	rm          ResourceManager
 	Log         logr.Logger
@@ -52,12 +55,13 @@ func NewController(ctx context.Context, base controllers.Controller, rm Resource
 	}
 
 	return &Controller{
-		BaseController: base,
-		Version:        application.GetVersion(ctx),
-		GitCommit:      gitCommit,
-		rm:             rm,
-		Log:            ctrl.Log.WithName(application.GetName(ctx)).WithName("controller").WithValues(logValues...),
-		ConfigStore:    config,
+		BaseController:     base,
+		Version:            application.GetVersion(ctx),
+		GitCommit:          gitCommit,
+		rm:                 rm,
+		Log:                ctrl.Log.WithName(application.GetName(ctx)).WithName("controller").WithValues(logValues...),
+		deletableResources: application.GetDeletableResources(ctx),
+		ConfigStore:        config,
 	}
 }
 
@@ -186,5 +190,9 @@ func (c *Controller) Run(ctx context.Context, owner resources.Resource) error {
 		return errors.Wrap(err, "cannot prepare owner status")
 	}
 
+	if err := c.Mark(ctx, owner); err != nil {
+		return errors.Wrap(err, "cannot mark resources")
+	}
+
 	return sgraph.Get(ctx).Run(ctx)
 }
diff --git a/pkg/controller/context.go b/pkg/controller/context.go
@@ -6,14 +6,15 @@ import (
 	sgraph "github.com/goharbor/harbor-operator/pkg/controller/internal/graph"
 	"github.com/goharbor/harbor-operator/pkg/factories/application"
 	"github.com/goharbor/harbor-operator/pkg/factories/logger"
+	"github.com/goharbor/harbor-operator/pkg/graph"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 func (c *Controller) PopulateContext(ctx context.Context, req ctrl.Request) context.Context {
 	application.SetName(&ctx, c.GetName())
 	application.SetVersion(&ctx, c.GetVersion())
 	application.SetGitCommit(&ctx, c.GetGitCommit())
-	ctx = sgraph.WithGraph(ctx)
+	sgraph.SetGraph(&ctx, graph.NewResourceManager())
 
 	logger.Set(&ctx, c.Log.WithValues("request", req))
 

diff --git a/pkg/controller/garbagecollector.go b/pkg/controller/garbagecollector.go
@@ -0,0 +1,136 @@
+package controller
+
+import (
+	"context"
+
+	sgraph "github.com/goharbor/harbor-operator/pkg/controller/internal/graph"
+	"github.com/goharbor/harbor-operator/pkg/graph"
+	"github.com/opentracing/opentracing-go"
+	"github.com/pkg/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func (c *Controller) Sweep(ctx context.Context, resource graph.Resource) error {
+	obj, ok := resource.(client.Object)
+	if !ok {
+		return nil
+	}
+
+	span, ctx := opentracing.StartSpanFromContext(ctx, "sweep")
+	defer span.Finish()
+
+	namespace, name := obj.GetNamespace(), obj.GetName()
+	gvk := c.AddGVKToSpan(ctx, span, obj)
+
+	span.
+		SetTag("resource.name", name).
+		SetTag("resource.namespace", namespace)
+
+	err := c.Delete(ctx, obj)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return errors.Wrapf(err, "sweep %s (%s/%s)", gvk, namespace, name)
+	}
+
+	return nil
+}
+
+func (c *Controller) Mark(ctx context.Context, owner client.Object) error { // nolint: funlen
+	span, ctx := opentracing.StartSpanFromContext(ctx, "mark")
+	defer span.Finish()
+
+	g := sgraph.Get(ctx)
+	if g == nil {
+		return errors.Errorf("no graph in current context")
+	}
+
+	gvk := c.AddGVKToSpan(ctx, span, owner)
+	reference := metav1.OwnerReference{
+		APIVersion: gvk.GroupVersion().String(),
+		Kind:       gvk.Kind,
+		Name:       owner.GetName(),
+	}
+
+	willDeletableResources := map[namespacedNameWithGVK]client.Object{}
+
+	for deletableGVK := range c.deletableResources {
+		unstructuredList := &unstructured.UnstructuredList{}
+		unstructuredGVK := deletableGVK
+		unstructuredList.SetGroupVersionKind(unstructuredGVK)
+
+		if err := c.List(ctx, unstructuredList, &client.ListOptions{
+			Namespace: owner.GetNamespace(),
+		}); err != nil {
+			return err
+		}
+
+		if err := unstructuredList.EachListItem(func(o runtime.Object) error {
+			unstructured, ok := o.DeepCopyObject().(client.Object)
+			if !ok {
+				return nil
+			}
+			unstructured.GetObjectKind().SetGroupVersionKind(unstructuredGVK)
+			if containsOwnerReferences(unstructured, reference) {
+				willDeletableResources[namespacedNameWithGVK{
+					NamespacedName: types.NamespacedName{
+						Namespace: unstructured.GetNamespace(),
+						Name:      unstructured.GetName(),
+					},
+					GroupVersionKind: unstructured.GetObjectKind().GroupVersionKind(),
+				}] = unstructured
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	for _, res := range g.GetAllResources(ctx) {
+		resource, ok := res.(*Resource)
+		if !ok {
+			continue
+		}
+
+		obj := resource.resource.DeepCopyObject().(client.Object)
+		key := namespacedNameWithGVK{
+			NamespacedName: types.NamespacedName{
+				Namespace: obj.GetNamespace(),
+				Name:      obj.GetName(),
+			},
+			GroupVersionKind: c.AddGVKToSpan(ctx, span, obj),
+		}
+		delete(willDeletableResources, key)
+	}
+
+	for _, o := range willDeletableResources {
+		if err := g.AddResource(ctx, o.DeepCopyObject(), []graph.Resource{}, c.Sweep); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type namespacedNameWithGVK struct {
+	types.NamespacedName
+	schema.GroupVersionKind
+}
+
+func containsOwnerReferences(o client.Object, reference metav1.OwnerReference) bool {
+	f := o.GetOwnerReferences()
+	for _, e := range f {
+		if e.APIVersion == reference.APIVersion &&
+			e.Kind == reference.Kind &&
+			e.Name == reference.Name {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/pkg/controller/internal/graph/graph.go b/pkg/controller/internal/graph/graph.go
@@ -8,8 +8,8 @@ import (
 
 var graphKey = "graph"
 
-func WithGraph(ctx context.Context) context.Context {
-	return context.WithValue(ctx, &graphKey, graph.NewResourceManager())
+func SetGraph(ctx *context.Context, mgr graph.Manager) {
+	*ctx = context.WithValue(*ctx, &graphKey, mgr)
 }
 
 func Get(ctx context.Context) graph.Manager {

diff --git a/pkg/factories/application/application.go b/pkg/factories/application/application.go
@@ -2,12 +2,15 @@ package application
 
 import (
 	"context"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
 var (
-	appNameContext      = "app-name"
-	appVersionContext   = "app-version"
-	appGitCommitContext = "app-git-commit"
+	appNameContext        = "app-name"
+	appVersionContext     = "app-version"
+	appGitCommitContext   = "app-git-commit"
+	appDeletableResources = "app-deletable-resources"
 )
 
 func GetName(ctx context.Context) string {
@@ -33,3 +36,16 @@ func GetGitCommit(ctx context.Context) string {
 func SetGitCommit(ctx *context.Context, gitCommit string) {
 	*ctx = context.WithValue(*ctx, &appGitCommitContext, gitCommit)
 }
+
+func GetDeletableResources(ctx context.Context) map[schema.GroupVersionKind]struct{} {
+	deletableResources := ctx.Value(&appDeletableResources)
+	if deletableResources == nil {
+		return nil
+	}
+
+	return deletableResources.(map[schema.GroupVersionKind]struct{})
+}
+
+func SetDeletableResources(ctx *context.Context, deletableResources map[schema.GroupVersionKind]struct{}) {
+	*ctx = context.WithValue(*ctx, &appDeletableResources, deletableResources)
+}
diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go
@@ -12,7 +12,7 @@ type resourceManager struct {
 	resources map[Resource][]Resource
 	functions map[Resource]RunFunc
 
-	lock sync.Mutex
+	lock sync.RWMutex
 }
 
 func NewResourceManager() Manager {
@@ -22,6 +22,19 @@ func NewResourceManager() Manager {
 	}
 }
 
+func (rm *resourceManager) GetAllResources(ctx context.Context) []Resource {
+	resources := []Resource{}
+
+	rm.lock.RLock()
+	defer rm.lock.RUnlock()
+
+	for resource := range rm.resources {
+		resources = append(resources, resource)
+	}
+
+	return resources
+}
+
 func (rm *resourceManager) AddResource(ctx context.Context, resource Resource, blockers []Resource, run RunFunc) error {
 	if resource == nil {
 		return nil

diff --git a/pkg/graph/types.go b/pkg/graph/types.go
@@ -1,12 +1,16 @@
 package graph
 
-import "context"
+import (
+	"context"
+)
 
-type Resource interface{}
-
-type RunFunc func(context.Context, Resource) error
+type (
+	Resource interface{}
+	RunFunc  func(context.Context, Resource) error
+)
 
 type Manager interface {
 	Run(context.Context) error
 	AddResource(context.Context, Resource, []Resource, RunFunc) error
+	GetAllResources(context.Context) []Resource
 }
diff --git a/pkg/setup/setup.go b/pkg/setup/setup.go
@@ -3,9 +3,15 @@ package setup
 import (
 	"context"
 
+	"github.com/goharbor/harbor-operator/pkg/factories/application"
 	"github.com/goharbor/harbor-operator/pkg/factories/logger"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
+	kauthn "k8s.io/api/authorization/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 )
 
@@ -23,7 +29,60 @@ func WithManager(ctx context.Context, mgr manager.Manager) error {
 	return g.Wait()
 }
 
+func populateContext(ctx context.Context, mgr manager.Manager) (context.Context, error) {
+	discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(mgr.GetConfig())
+
+	preferredResources, err := discoveryClient.ServerPreferredResources()
+	if err != nil {
+		return ctx, err
+	}
+
+	resources := discovery.FilteredBy(discovery.ResourcePredicateFunc(func(groupVersion string, r *metav1.APIResource) bool {
+		check := sets.NewString([]string(r.Verbs)...).HasAll("delete", "list", "create")
+		check = check && r.Namespaced
+
+		return check
+	}), preferredResources)
+	deletableResources := make(map[schema.GroupVersionKind]struct{})
+
+	for _, rl := range resources {
+		gv, err := schema.ParseGroupVersion(rl.GroupVersion)
+		if err != nil {
+			return ctx, err
+		}
+
+		for i := range rl.APIResources {
+			sar := &kauthn.SelfSubjectAccessReview{
+				Spec: kauthn.SelfSubjectAccessReviewSpec{
+					ResourceAttributes: &kauthn.ResourceAttributes{
+						Verb:     "delete",
+						Group:    gv.Group,
+						Version:  gv.Version,
+						Resource: rl.APIResources[i].Name,
+					},
+				},
+			}
+			if err := mgr.GetClient().Create(ctx, sar); err != nil {
+				return ctx, err
+			}
+
+			if sar.Status.Allowed {
+				deletableResources[gv.WithKind(rl.APIResources[i].Kind)] = struct{}{}
+			}
+		}
+	}
+
+	application.SetDeletableResources(&ctx, deletableResources)
+
+	return ctx, nil
+}
+
 func ControllersWithManager(ctx context.Context, mgr manager.Manager) error {
+	ctx, err := populateContext(ctx, mgr)
+	if err != nil {
+		return errors.Wrap(err, "populateContext")
+	}
+
 	var g errgroup.Group
 
 	for name, builder := range controllersBuilder {