feat: merge registry and registryctl pods into one pod (goharbor#796)

Signed-off-by: chlins <chenyuzh@vmware.com>
thcdrt · Nov 9, 2021 · 74eee8b · 74eee8b
1 parent 1565a88
commit 74eee8b
Show file tree

Hide file tree

Showing 22 changed files with 403 additions and 954 deletions.
diff --git a/controllers/goharbor/controller_test.go b/controllers/goharbor/controller_test.go
@@ -101,7 +101,6 @@ var _ = DescribeTable(
 	},
 	Entry("Portal", newPortalController(), 30*time.Second, 2*time.Second),
 	Entry("Registry", newRegistryController(), time.Minute, 5*time.Second),
-	Entry("RegistryCtl", newRegistryCtlController(), 2*time.Minute, 5*time.Second),
 	Entry("ChartMuseum", newChartMuseumController(), time.Minute, 5*time.Second),
 	Entry("Trivy", newTrivyController(), 3*time.Minute, 5*time.Second),
 	Entry("NotaryServer", newNotaryServerController(), time.Minute, 5*time.Second),

diff --git a/controllers/goharbor/harbor/registryctl.go b/controllers/goharbor/harbor/registryctl.go
@@ -25,7 +25,7 @@ func (r *Reconciler) AddRegistryController(ctx context.Context, harbor *goharbor
 		return nil, nil, errors.Wrap(err, "cannot get registryCtl")
 	}
 
-	registryCtlRes, err := r.AddBasicResource(ctx, registryCtl, registry, certificate)
+	registryCtlRes, err := r.AddBasicResource(ctx, registryCtl, certificate)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "cannot add registryCtl")
 	}

diff --git a/controllers/goharbor/internal/test/controllers/controllers.go b/controllers/goharbor/internal/test/controllers/controllers.go
@@ -13,7 +13,6 @@ import (
 	"github.com/goharbor/harbor-operator/controllers/goharbor/notarysigner"
 	"github.com/goharbor/harbor-operator/controllers/goharbor/portal"
 	"github.com/goharbor/harbor-operator/controllers/goharbor/registry"
-	"github.com/goharbor/harbor-operator/controllers/goharbor/registryctl"
 	"github.com/goharbor/harbor-operator/controllers/goharbor/trivy"
 	"github.com/goharbor/harbor-operator/pkg/config"
 	"github.com/goharbor/harbor-operator/pkg/controller"
@@ -56,10 +55,6 @@ func NewRegistry(ctx context.Context, className string) *registry.Reconciler {
 	return New(ctx, controllers.Registry, className, registry.New).(*registry.Reconciler)
 }
 
-func NewRegistryCtl(ctx context.Context, className string) *registryctl.Reconciler {
-	return New(ctx, controllers.RegistryController, className, registryctl.New).(*registryctl.Reconciler)
-}
-
 type CtrlBuilder func(context.Context, *configstore.Store) (controller.Reconciler, error)
 
 func New(ctx context.Context, name controllers.Controller, className string, builder CtrlBuilder) controller.Reconciler {

diff --git a/controllers/goharbor/registry/configs.go b/controllers/goharbor/registry/configs.go
@@ -8,12 +8,16 @@ import (
 	goharborv1 "github.com/goharbor/harbor-operator/apis/goharbor.io/v1beta1"
 	conftemplate "github.com/goharbor/harbor-operator/pkg/config/template"
 	"github.com/goharbor/harbor-operator/pkg/resources/checksum"
+	"github.com/goharbor/harbor-operator/pkg/utils/strings"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const ConfigName = "config.yml"
+const (
+	ConfigName      = "config.yml"
+	RegistryCtlName = "registryctl"
+)
 
 func (r *Reconciler) GetConfigMap(ctx context.Context, registry *goharborv1.Registry) (*corev1.ConfigMap, error) {
 	templateConfig, err := r.ConfigStore.GetItemValue(conftemplate.ConfigTemplateKey)
@@ -43,3 +47,31 @@ func (r *Reconciler) GetConfigMap(ctx context.Context, registry *goharborv1.Regi
 		},
 	}, nil
 }
+
+func (r *Reconciler) GetCtlConfigMap(ctx context.Context, registryCtl *goharborv1.RegistryController) (*corev1.ConfigMap, error) {
+	templateConfig, err := r.ConfigStore.GetItemValue(CtlConfigTemplateKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot get template")
+	}
+
+	content, err := r.GetTemplatedConfig(ctx, templateConfig, registryCtl)
+	if err != nil {
+		return nil, err
+	}
+
+	name := strings.NormalizeName(registryCtl.GetName(), RegistryCtlName)
+	namespace := registryCtl.GetNamespace()
+
+	return &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Annotations: map[string]string{
+				checksum.GetStaticID("template"): fmt.Sprintf("%x", sha256.Sum256([]byte(templateConfig))),
+			},
+		},
+		BinaryData: map[string][]byte{
+			ConfigName: content,
+		},
+	}, nil
+}
diff --git a/controllers/goharbor/registry/deployments.go b/controllers/goharbor/registry/deployments.go
@@ -10,6 +10,7 @@ import (
 	"github.com/goharbor/harbor-operator/controllers"
 	serrors "github.com/goharbor/harbor-operator/pkg/controller/errors"
 	"github.com/goharbor/harbor-operator/pkg/image"
+	utilStrings "github.com/goharbor/harbor-operator/pkg/utils/strings"
 	"github.com/goharbor/harbor-operator/pkg/version"
 	"github.com/pkg/errors"
 	appsv1 "k8s.io/api/apps/v1"
@@ -20,17 +21,22 @@ import (
 
 const (
 	VolumeName                            = "registry-config"
+	CtlVolumeName                         = "registryctl-config"
 	ConfigPath                            = "/etc/registry"
+	CtlConfigPath                         = "/etc/registryctl"
 	CompatibilitySchema1Path              = ConfigPath + "/compatibility-schema1"
 	CompatibilitySchema1VolumeName        = "compatibility-schema1-certificate"
 	AuthenticationHTPasswdPath            = ConfigPath + "/auth"
 	AuthenticationHTPasswdVolumeName      = "authentication-htpasswd"
 	InternalCertificatesVolumeName        = "internal-certificates"
+	CtlInternalCertificatesVolumeName     = "ctl-internal-certificates"
 	InternalCertificateAuthorityDirectory = "/harbor_cust_cert"
 	InternalCertificatesPath              = ConfigPath + "/ssl"
+	CtlInternalCertificatesPath           = CtlConfigPath + "/ssl"
 	StorageName                           = "storage"
 	StoragePath                           = "/var/lib/registry"
 	HealthPath                            = "/"
+	CtlHealthPath                         = "/api/health"
 	StorageServiceCAName                  = "storage-service-ca"
 	StorageServiceCAMountPath             = "/harbor_cust_cert/custom-ca-bundle.crt"
 )
@@ -47,6 +53,9 @@ var (
 const (
 	apiPort     = 5000 // https://github.com/docker/distribution/blob/749f6afb4572201e3c37325d0ffedb6f32be8950/contrib/compose/docker-compose.yml#L15
 	metricsPort = 5001 // https://github.com/docker/distribution/blob/b12bd4004afc203f1cbd2072317c8fda30b89710/cmd/registry/config-dev.yml#L34
+	// registry controller port.
+	httpsPort = 8443
+	httpPort  = 8080
 )
 
 func (r *Reconciler) GetDeployment(ctx context.Context, registry *goharborv1.Registry) (*appsv1.Deployment, error) { // nolint:funlen
@@ -288,11 +297,143 @@ func (r *Reconciler) GetDeployment(ctx context.Context, registry *goharborv1.Reg
 		},
 	}
 
-	err = r.ApplyStorageConfiguration(ctx, registry, deploy)
+	if err = r.ApplyStorageConfiguration(ctx, registry, deploy); err != nil {
+		return nil, errors.Wrap(err, "cannot apply storage configuration")
+	}
+
+	// attach registry controller container
+	if err = r.attachRegistryCtlContainer(ctx, registry, deploy); err != nil {
+		return nil, errors.Wrap(err, "cannot attach registryctl container")
+	}
 
 	registry.Spec.ComponentSpec.ApplyToDeployment(deploy)
 
-	return deploy, errors.Wrap(err, "cannot apply storage configuration")
+	return deploy, nil
+}
+
+func (r *Reconciler) attachRegistryCtlContainer(ctx context.Context, registry *goharborv1.Registry, deploy *appsv1.Deployment) error { // nolint:funlen
+	registryCtl, err := r.GetRegistryCtl(ctx, registry)
+	if err != nil {
+		return errors.Wrap(err, "can not get registryctl from registry")
+	}
+
+	getImageOptions := []image.Option{
+		image.WithImageFromSpec(registryCtl.Spec.Image),
+		image.WithHarborVersion(version.GetVersion(registryCtl.Annotations)),
+	}
+
+	image, err := image.GetImage(ctx, harbormetav1.RegistryControllerComponent.String(), getImageOptions...)
+	if err != nil {
+		return errors.Wrap(err, "cannot get registryctl image")
+	}
+
+	name := controllers.RegistryController.String()
+
+	volumeMounts := append(deploy.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
+		Name:      CtlVolumeName,
+		MountPath: CtlConfigPath,
+	})
+
+	if registryCtl.Spec.TLS.Enabled() {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      CtlInternalCertificatesVolumeName,
+			MountPath: path.Join(InternalCertificateAuthorityDirectory, "ctl-"+corev1.ServiceAccountRootCAKey),
+			SubPath:   strings.TrimLeft(corev1.ServiceAccountRootCAKey, "/"),
+			ReadOnly:  true,
+		}, corev1.VolumeMount{
+			Name:      CtlInternalCertificatesVolumeName,
+			MountPath: CtlInternalCertificatesPath,
+			ReadOnly:  true,
+		})
+
+		deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, corev1.Volume{
+			Name: CtlInternalCertificatesVolumeName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: registryCtl.Spec.TLS.CertificateRef,
+				},
+			},
+		})
+	}
+
+	deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, corev1.Volume{
+		Name: CtlVolumeName,
+		VolumeSource: corev1.VolumeSource{
+			ConfigMap: &corev1.ConfigMapVolumeSource{
+				LocalObjectReference: corev1.LocalObjectReference{
+					Name: utilStrings.NormalizeName(registryCtl.GetName(), RegistryCtlName),
+				},
+				Optional: &varFalse,
+			},
+		},
+	})
+
+	envs := deploy.Spec.Template.Spec.Containers[0].Env
+	if registryCtl.Spec.Authentication.JobServiceSecretRef != "" {
+		envs = append(envs, corev1.EnvVar{
+			Name: "JOBSERVICE_SECRET",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: registryCtl.Spec.Authentication.JobServiceSecretRef,
+					},
+					Key: harbormetav1.SharedSecretKey,
+				},
+			},
+		})
+	}
+
+	if registryCtl.Spec.Authentication.CoreSecretRef != "" {
+		envs = append(envs, corev1.EnvVar{
+			Name: "CORE_SECRET",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: registryCtl.Spec.Authentication.CoreSecretRef,
+					},
+					Key: harbormetav1.SharedSecretKey,
+				},
+			},
+		})
+	}
+
+	ports := []corev1.ContainerPort{{
+		Name:          harbormetav1.RegistryControllerHTTPPortName,
+		ContainerPort: httpPort,
+		Protocol:      corev1.ProtocolTCP,
+	}, {
+		Name:          harbormetav1.RegistryControllerHTTPSPortName,
+		ContainerPort: httpsPort,
+		Protocol:      corev1.ProtocolTCP,
+	}}
+
+	port := harbormetav1.RegistryControllerHTTPPortName
+	if registryCtl.Spec.TLS.Enabled() {
+		port = harbormetav1.RegistryControllerHTTPSPortName
+	}
+
+	probe := &corev1.Probe{
+		Handler: corev1.Handler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   CtlHealthPath,
+				Port:   intstr.FromString(port),
+				Scheme: registryCtl.Spec.TLS.GetScheme(),
+			},
+		},
+	}
+
+	container := &corev1.Container{
+		Name:           name,
+		Image:          image,
+		Env:            envs,
+		Ports:          ports,
+		LivenessProbe:  probe,
+		ReadinessProbe: probe,
+		VolumeMounts:   volumeMounts,
+	}
+	deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *container)
+
+	return nil
 }
 
 const registryContainerIndex = 0

diff --git a/controllers/goharbor/registry/registry.go b/controllers/goharbor/registry/registry.go
@@ -19,8 +19,10 @@ import (
 )
 
 const (
-	DefaultRequeueWait            = 2 * time.Second
-	DefaultConfigTemplateFileName = "registry-config.yaml.tmpl"
+	DefaultRequeueWait               = 2 * time.Second
+	DefaultConfigTemplateFileName    = "registry-config.yaml.tmpl"
+	DefaultCtlConfigTemplateFileName = "registryctl-config.yaml.tmpl"
+	CtlConfigTemplateKey             = "template-content-ctl"
 )
 
 // Reconciler reconciles a Registry object.
@@ -34,6 +36,9 @@ type Reconciler struct {
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=configmaps;services,verbs=get;list;watch;create;update;patch;delete
 
+// +kubebuilder:rbac:groups=goharbor.io,resources=registrycontrollers,verbs=get;list;watch
+// +kubebuilder:rbac:groups=goharbor.io,resources=registrycontrollers/status,verbs=get;update;patch
+
 func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error {
 	err := r.Controller.SetupWithManager(ctx, mgr)
 	if err != nil {
@@ -84,7 +89,15 @@ func (r *Reconciler) Template(ctx context.Context) (*template.ConfigTemplate, er
 		return nil, errors.Wrap(err, "from configstore")
 	}
 
+	ctlTemplateConfig, err := template.FromConfigStore(r.ConfigStore, DefaultCtlConfigTemplateFileName)
+	if err != nil {
+		return nil, errors.Wrap(err, "from ctl configstore")
+	}
+
 	templateConfig.Register(r.ConfigStore)
+	// mutate key
+	ctlTemplateConfig.Key = CtlConfigTemplateKey
+	ctlTemplateConfig.Register(r.ConfigStore)
 
 	return templateConfig, nil
 }

diff --git a/controllers/goharbor/registry/registry_test.go b/controllers/goharbor/registry/registry_test.go
@@ -56,6 +56,19 @@ var _ = Describe("Registry", func() {
 					},
 				},
 			}
+
+			registryCtl := &goharborv1.RegistryController{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:        registry.GetName(),
+					Namespace:   registry.GetNamespace(),
+					Annotations: test.AddVersionAnnotations(nil),
+				},
+				Spec: goharborv1.RegistryControllerSpec{
+					RegistryRef: registry.GetName(),
+				},
+			}
+			Ω(test.GetClient(ctx).Create(ctx, registryCtl)).
+				Should(test.SuccessOrExists)
 		})
 
 		It("Should works", func() {