From 425e8fd65cdaeb1d96a5cca588b2086a29dd1bd2 Mon Sep 17 00:00:00 2001 From: Jasin Aferkou Date: Tue, 30 Jun 2026 11:17:37 +0200 Subject: [PATCH 1/2] feat(openbao): add imagePullSecrets + use mirrored bank vaults operator --- cli/cmd/install_openbao.go | 40 ++- docs/oms_install_openbao.md | 44 ++- .../installer/manifests/openbao/vault-cr.yaml | 4 + internal/installer/openbao.go | 283 ++++++++++++++++-- internal/installer/openbao_test.go | 232 +++++++++++++- 5 files changed, 561 insertions(+), 42 deletions(-) diff --git a/cli/cmd/install_openbao.go b/cli/cmd/install_openbao.go index f7f49776..66c70d65 100644 --- a/cli/cmd/install_openbao.go +++ b/cli/cmd/install_openbao.go @@ -43,6 +43,10 @@ type InstallOpenBaoOpts struct { Timeout time.Duration AgeKeyFile string Yes bool + OpenBaoImage string + BankVaultsImage string + OperatorImage string + OperatorChartRepo string } func (c *InstallOpenBaoCmd) RunE(_ *cobra.Command, _ []string) error { @@ -74,6 +78,18 @@ func (c *InstallOpenBaoCmd) RunE(_ *cobra.Command, _ []string) error { Timeout: c.Opts.Timeout, AgeRecipient: recipient, AgeKeyPath: keyPath, + // Optional GHCR credentials for the private OpenBao/bank-vaults image + // mirror. When both are set the installer creates a pull secret and + // wires it onto the openbao ServiceAccount; when unset, behavior is + // unchanged. + RegistryUser: os.Getenv("OMS_REGISTRY_USER"), + RegistryPassword: os.Getenv("OMS_REGISTRY_PASSWORD"), + // Image/chart overrides for mirrored OCI registries. Defaults are set on + // the flags (the installer's Default* values). + OpenBaoImage: c.Opts.OpenBaoImage, + BankVaultsImage: c.Opts.BankVaultsImage, + OperatorImage: c.Opts.OperatorImage, + OperatorChartRepo: c.Opts.OperatorChartRepo, } inst, err := installer.NewOpenBaoInstaller(cfg) @@ -128,11 +144,29 @@ func AddInstallOpenBaoCmd(install *cobra.Command, opts *GlobalOptions) { 5. Wait for initialization to complete 6. Extract and encrypt unseal keys + password as SOPS DR backup - The command is idempotent and safe to re-run.`), + The command is idempotent and safe to re-run. + + By default the OpenBao, bank-vaults, and operator images and the operator + Helm chart are pulled from the private Codesphere registry mirror. Use the + --openbao-image, --bank-vaults-image, --operator-image and + --operator-chart-repo flags to repoint them at your own mirrored OCI + registry. + + Because the default registry is private, set both environment variables + below: the installer creates an image pull secret (with an entry for every + registry host the configured images live on), attaches it to the openbao + ServiceAccount and operator pod, and uses the credentials to authenticate + the operator chart pull. Leave them unset only on clusters with node-level + registry access or fully public images. + + Environment variables: + OMS_REGISTRY_USER Registry username (e.g. GitHub user for ghcr.io) + OMS_REGISTRY_PASSWORD Registry token/PAT (read:packages for ghcr.io)`), Example: formatExamples("install openbao", []packageio.Example{ {Cmd: "--dr-backup-path ./backups/cluster-1.enc.json", Desc: "Fresh bootstrap with DR backup saved locally"}, {Cmd: "--dr-backup-path ./backups/cluster-1.enc.json --secrets-engine my-engine --bao-user myuser", Desc: "Custom engine and user"}, {Cmd: "--dr-backup-path ./backups/cluster-1.enc.json --timeout 10m", Desc: "Extended timeout for slower clusters"}, + {Cmd: "--dr-backup-path ./backups/cluster-1.enc.json --openbao-image my-mirror.example.com/openbao/openbao:2.5.4 --operator-chart-repo oci://my-mirror.example.com/bank-vaults/helm-charts", Desc: "Use a mirrored OCI registry (set OMS_REGISTRY_USER/OMS_REGISTRY_PASSWORD)"}, }), }, Opts: &InstallOpenBaoOpts{GlobalOptions: opts}, @@ -146,6 +180,10 @@ func AddInstallOpenBaoCmd(install *cobra.Command, opts *GlobalOptions) { openbao.cmd.Flags().DurationVar(&openbao.Opts.Timeout, "timeout", 5*time.Minute, "Timeout for waiting on initialization") openbao.cmd.Flags().StringVarP(&openbao.Opts.AgeKeyFile, "age-key-file", "k", "", "Path to age private key file for SOPS encryption/decryption (auto-detected if not set)") openbao.cmd.Flags().BoolVarP(&openbao.Opts.Yes, "yes", "y", false, "Auto-approve re-initialization of an existing deployment when no DR backup is found") + openbao.cmd.Flags().StringVar(&openbao.Opts.OpenBaoImage, "openbao-image", installer.DefaultOpenBaoImage, "OpenBao server image (override for a mirrored OCI registry)") + openbao.cmd.Flags().StringVar(&openbao.Opts.BankVaultsImage, "bank-vaults-image", installer.DefaultBankVaultsImage, "Bank-Vaults configurer image (override for a mirrored OCI registry)") + openbao.cmd.Flags().StringVar(&openbao.Opts.OperatorImage, "operator-image", installer.DefaultOperatorImage, "Bank-Vaults operator pod image (override for a mirrored OCI registry)") + openbao.cmd.Flags().StringVar(&openbao.Opts.OperatorChartRepo, "operator-chart-repo", installer.DefaultBankVaultsChartRepo, "OCI repo hosting the vault-operator Helm chart (override for a mirrored OCI registry)") util.MarkFlagRequired(openbao.cmd, "dr-backup-path") diff --git a/docs/oms_install_openbao.md b/docs/oms_install_openbao.md index f020a023..af0929a4 100644 --- a/docs/oms_install_openbao.md +++ b/docs/oms_install_openbao.md @@ -16,6 +16,23 @@ This command performs the full lifecycle: The command is idempotent and safe to re-run. +By default the OpenBao, bank-vaults, and operator images and the operator +Helm chart are pulled from the private Codesphere registry mirror. Use the +--openbao-image, --bank-vaults-image, --operator-image and +--operator-chart-repo flags to repoint them at your own mirrored OCI +registry. + +Because the default registry is private, set both environment variables +below: the installer creates an image pull secret (with an entry for every +registry host the configured images live on), attaches it to the openbao +ServiceAccount and operator pod, and uses the credentials to authenticate +the operator chart pull. Leave them unset only on clusters with node-level +registry access or fully public images. + +Environment variables: + OMS_REGISTRY_USER Registry username (e.g. GitHub user for ghcr.io) + OMS_REGISTRY_PASSWORD Registry token/PAT (read:packages for ghcr.io) + ``` oms install openbao [flags] ``` @@ -32,21 +49,28 @@ $ oms install openbao --dr-backup-path ./backups/cluster-1.enc.json --secrets-en # Extended timeout for slower clusters $ oms install openbao --dr-backup-path ./backups/cluster-1.enc.json --timeout 10m +# Use a mirrored OCI registry (set OMS_REGISTRY_USER/OMS_REGISTRY_PASSWORD) +$ oms install openbao --dr-backup-path ./backups/cluster-1.enc.json --openbao-image my-mirror.example.com/openbao/openbao:2.5.4 --operator-chart-repo oci://my-mirror.example.com/bank-vaults/helm-charts + ``` ### Options ``` - -k, --age-key-file string Path to age private key file for SOPS encryption/decryption (auto-detected if not set) - --bao-user string Username for the userpass auth method (ignored on restore, uses DR backup value) (default "admin") - --dr-backup-path string Path for SOPS-encrypted DR backup file (required) - -h, --help help for openbao - -n, --namespace string Kubernetes namespace for OpenBao deployment (default "vault") - --replicas int Number of OpenBao replicas (1 for single-node, odd number >= 3 for HA) (default 3) - --secrets-engine string Name of the KV-v2 secrets engine to provision (default "cs-secrets-engine") - --storage-size string PVC storage size for each OpenBao replica (default "10Gi") - --timeout duration Timeout for waiting on initialization (default 5m0s) - -y, --yes Auto-approve re-initialization of an existing deployment when no DR backup is found + -k, --age-key-file string Path to age private key file for SOPS encryption/decryption (auto-detected if not set) + --bank-vaults-image string Bank-Vaults configurer image (override for a mirrored OCI registry) (default "ghcr.io/codesphere-cloud/docker/banzaicloud/bank-vaults:1.19.0") + --bao-user string Username for the userpass auth method (ignored on restore, uses DR backup value) (default "admin") + --dr-backup-path string Path for SOPS-encrypted DR backup file (required) + -h, --help help for openbao + -n, --namespace string Kubernetes namespace for OpenBao deployment (default "vault") + --openbao-image string OpenBao server image (override for a mirrored OCI registry) (default "ghcr.io/codesphere-cloud/docker/quay.io/openbao/openbao-cs-patched:2.5.4") + --operator-chart-repo string OCI repo hosting the vault-operator Helm chart (override for a mirrored OCI registry) (default "oci://ghcr.io/codesphere-cloud/docker/ghcr.io/bank-vaults/helm-charts") + --operator-image string Bank-Vaults operator pod image (override for a mirrored OCI registry) (default "ghcr.io/codesphere-cloud/docker/ghcr.io/bank-vaults/vault-operator:1.24.0") + --replicas int Number of OpenBao replicas (1 for single-node, odd number >= 3 for HA) (default 3) + --secrets-engine string Name of the KV-v2 secrets engine to provision (default "cs-secrets-engine") + --storage-size string PVC storage size for each OpenBao replica (default "10Gi") + --timeout duration Timeout for waiting on initialization (default 5m0s) + -y, --yes Auto-approve re-initialization of an existing deployment when no DR backup is found ``` ### SEE ALSO diff --git a/internal/installer/manifests/openbao/vault-cr.yaml b/internal/installer/manifests/openbao/vault-cr.yaml index 329383e6..a9cf6d5c 100644 --- a/internal/installer/manifests/openbao/vault-cr.yaml +++ b/internal/installer/manifests/openbao/vault-cr.yaml @@ -6,6 +6,10 @@ kind: ServiceAccount metadata: name: openbao namespace: {{ .Namespace }} +{{- if .ImagePullSecretName }} +imagePullSecrets: + - name: {{ .ImagePullSecretName }} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role diff --git a/internal/installer/openbao.go b/internal/installer/openbao.go index 88cbdaed..ef1c774c 100644 --- a/internal/installer/openbao.go +++ b/internal/installer/openbao.go @@ -11,6 +11,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/url" "os" "path/filepath" "strings" @@ -19,6 +20,7 @@ import ( "github.com/codesphere-cloud/oms/internal/bootstrap" k8s "github.com/codesphere-cloud/oms/internal/util" + "github.com/distribution/reference" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,14 +36,27 @@ var vaultCRTemplate []byte const ( openBaoUnsealSecretName = "openbao-unseal-keys" DefaultOpenBaoNamespace = "vault" - openBaoImage = "ghcr.io/codesphere-cloud/docker/quay.io/openbao/openbao-cs-patched:2.5.4" - bankVaultsImage = "ghcr.io/codesphere-cloud/docker/banzaicloud/bank-vaults:1.19.0" - bankVaultsChartRepo = "oci://ghcr.io/bank-vaults/helm-charts" - bankVaultsChartName = "vault-operator" - bankVaultsChartVersion = "1.22.5" - defaultPasswordLength = 32 - pollInterval = 5 * time.Second - maxPollInterval = 30 * time.Second + + // imagePullSecretName is the dockerconfigjson Secret created from the + // OMS_REGISTRY_USER/OMS_REGISTRY_PASSWORD env vars and attached to the + // "openbao" ServiceAccount so the operator-managed pods can pull the + // OpenBao and bank-vaults images from the configured (private) registry. + imagePullSecretName = "openbao-registry" + + // Default image and chart locations. All point at the private Codesphere + // GHCR mirror; each is overridable via a CLI flag so a customer can use + // their own mirrored OCI registry. The registry the pull secret + // authenticates to is derived from these refs at runtime, not hardcoded. + DefaultOpenBaoImage = "ghcr.io/codesphere-cloud/docker/quay.io/openbao/openbao-cs-patched:2.5.4" + DefaultBankVaultsImage = "ghcr.io/codesphere-cloud/docker/banzaicloud/bank-vaults:1.19.0" + DefaultOperatorImage = "ghcr.io/codesphere-cloud/docker/ghcr.io/bank-vaults/vault-operator:1.24.0" + DefaultBankVaultsChartRepo = "oci://ghcr.io/codesphere-cloud/docker/ghcr.io/bank-vaults/helm-charts" + + bankVaultsChartName = "vault-operator" + bankVaultsChartVersion = "1.24.0" + defaultPasswordLength = 32 + pollInterval = 5 * time.Second + maxPollInterval = 30 * time.Second // defaultReadinessTimeoutPerReplica is added to the base timeout for each // replica when waiting for all OpenBao pods to become ready. Pods come up @@ -96,6 +111,22 @@ type OpenBaoInstallerConfig struct { ReadinessTimeoutPerReplica time.Duration AgeRecipient string AgeKeyPath string + // RegistryUser/RegistryPassword are read from OMS_REGISTRY_USER and + // OMS_REGISTRY_PASSWORD. When both are set, an image pull secret for the + // configured registry is created and wired onto the openbao ServiceAccount + // (and the operator chart), and a Helm OCI login is performed before pulling + // the operator chart. When both are empty, no pull secret is configured and + // no registry login is attempted (unchanged behavior). + RegistryUser string + RegistryPassword string + + // Image and chart overrides. Empty values are backfilled in validateConfig + // from the Default* constants, so a customer can repoint any of them at a + // mirrored OCI registry without affecting the default install. + OpenBaoImage string // OpenBao server image (Vault CR spec.image) + BankVaultsImage string // bank-vaults configurer image (Vault CR spec.bankVaultsImage) + OperatorImage string // bank-vaults operator pod image (Helm values override) + OperatorChartRepo string // OCI repo hosting the vault-operator Helm chart } // OpenBaoInstaller orchestrates the Day-0 bootstrap, configuration, and DR @@ -166,6 +197,18 @@ func (o *OpenBaoInstaller) validateConfig() error { if o.Config.ReadinessTimeoutPerReplica <= 0 { o.Config.ReadinessTimeoutPerReplica = defaultReadinessTimeoutPerReplica } + if o.Config.OpenBaoImage == "" { + o.Config.OpenBaoImage = DefaultOpenBaoImage + } + if o.Config.BankVaultsImage == "" { + o.Config.BankVaultsImage = DefaultBankVaultsImage + } + if o.Config.OperatorImage == "" { + o.Config.OperatorImage = DefaultOperatorImage + } + if o.Config.OperatorChartRepo == "" { + o.Config.OperatorChartRepo = DefaultBankVaultsChartRepo + } return nil } @@ -218,6 +261,13 @@ func (o *OpenBaoInstaller) Install(ctx context.Context) error { return fmt.Errorf("failed to ensure namespace: %w", err) } + // Create the GHCR pull secret before the operator schedules any pods, so + // the openbao ServiceAccount can reference it from the moment pods appear. + err = o.Logger.Step("Ensuring image pull secret", o.EnsureImagePullSecret) + if err != nil { + return fmt.Errorf("failed to ensure image pull secret: %w", err) + } + err = o.Logger.Step("Deploying Bank-Vaults Operator", o.DeployBankVaultsOperator) if err != nil { return fmt.Errorf("failed to deploy Bank-Vaults Operator: %w", err) @@ -324,15 +374,19 @@ func (o *OpenBaoInstaller) GeneratePassword() error { // installed in a different namespace, we skip re-deployment — one instance // is sufficient for the entire cluster. func (o *OpenBaoInstaller) DeployBankVaultsOperator() error { + values, err := o.operatorChartValues() + if err != nil { + return err + } cfg := ChartConfig{ ReleaseName: operatorName, - ChartName: bankVaultsChartRepo + "/" + bankVaultsChartName, + ChartName: o.Config.OperatorChartRepo + "/" + bankVaultsChartName, Version: bankVaultsChartVersion, Namespace: o.Config.Namespace, // Namespace creation is handled exclusively by ensureNamespace, which // runs earlier in the install pipeline — keep a single creation path. CreateNamespace: false, - Values: map[string]interface{}{}, + Values: values, } // Upgrade in place when a release already exists in the target namespace. @@ -341,6 +395,9 @@ func (o *OpenBaoInstaller) DeployBankVaultsOperator() error { return err } if exists { + if err := o.loginChartRegistry(); err != nil { + return err + } return o.Helm.UpgradeChart(o.ctx, cfg, UpgradeChartOptions{}) } @@ -365,9 +422,70 @@ func (o *OpenBaoInstaller) DeployBankVaultsOperator() error { } // Operator does not exist — perform fresh install. + if err := o.loginChartRegistry(); err != nil { + return err + } return o.Helm.InstallChart(o.ctx, cfg, InstallChartOptions{}) } +// loginChartRegistry authenticates the Helm client against the OCI registry +// hosting the operator chart, so a chart mirrored to a private registry can be +// pulled. No-op when no credentials were supplied (public chart registry). The +// authenticated registry client is reused by the subsequent chart pull. Mirrors +// the pattern in pc_apps.go. +func (o *OpenBaoInstaller) loginChartRegistry() error { + if !o.imagePullSecretConfigured() { + return nil + } + parsed, err := url.Parse(o.Config.OperatorChartRepo) + if err != nil { + return fmt.Errorf("parsing operator chart repo %q: %w", o.Config.OperatorChartRepo, err) + } + if parsed.Host == "" { + return fmt.Errorf("operator chart repo %q has no host", o.Config.OperatorChartRepo) + } + if err := o.Helm.LoginRegistry(o.ctx, parsed.Host, o.Config.RegistryUser, o.Config.RegistryPassword); err != nil { + return fmt.Errorf("authenticating to chart registry %q: %w", parsed.Host, err) + } + return nil +} + +// operatorChartValues builds the Helm values overriding the vault-operator +// chart: the operator pod image (always, from OperatorImage) and, when +// credentials are configured, the image pull secret referencing +// imagePullSecretName so the operator pod can pull from a private registry. +// +// NOTE: the value keys (image.repository, image.tag, image.imagePullSecrets) +// follow the bank-vaults vault-operator chart schema. A chart version bump can +// move these — confirm with: +// +// helm show values oci://ghcr.io/bank-vaults/helm-charts/vault-operator --version 1.24.0 +// +// Wrong keys are silently ignored and the operator falls back to its chart +// defaults (public image, no pull secret). +func (o *OpenBaoInstaller) operatorChartValues() (map[string]interface{}, error) { + image := map[string]interface{}{} + if o.Config.OperatorImage != "" { + ref, err := reference.ParseNormalizedNamed(o.Config.OperatorImage) + if err != nil { + return nil, fmt.Errorf("parsing operator image %q: %w", o.Config.OperatorImage, err) + } + image["repository"] = reference.TrimNamed(ref).Name() + if tagged, ok := ref.(reference.Tagged); ok { + image["tag"] = tagged.Tag() + } + } + if o.imagePullSecretConfigured() { + image["imagePullSecrets"] = []interface{}{ + map[string]interface{}{"name": imagePullSecretName}, + } + } + if len(image) == 0 { + return map[string]interface{}{}, nil + } + return map[string]interface{}{"image": image}, nil +} + // cleanOrphanedOperatorRBAC best-effort deletes the operator's cluster-scoped // ClusterRole and ClusterRoleBinding. These are not garbage-collected when the // operator's namespace is deleted, so they can linger after a teardown and make @@ -446,6 +564,10 @@ type vaultCRTemplateData struct { Replicas int StorageSize string RetryJoinAddrs []string + // ImagePullSecretName is the name of the dockerconfigjson Secret to attach + // to the openbao ServiceAccount. Empty when no registry credentials were + // supplied, in which case the template omits imagePullSecrets entirely. + ImagePullSecretName string } // Build retry_join addresses for Raft so each node can autonomously @@ -469,16 +591,24 @@ func (o *OpenBaoInstaller) ApplyVaultCR() error { retryJoinAddrs := buildRetryJoinAddrs(o.Config.Replicas, o.Config.Namespace) + // Wire the pull secret onto the ServiceAccount only when credentials were + // supplied; EnsureImagePullSecret has already created it by this point. + var pullSecretName string + if o.imagePullSecretConfigured() { + pullSecretName = imagePullSecretName + } + data := vaultCRTemplateData{ - Namespace: o.Config.Namespace, - OpenBaoImage: openBaoImage, - BankVaultsImage: bankVaultsImage, - SecretsEngineName: o.Config.SecretsEngineName, - BaoUsername: o.Config.Username, - BaoPassword: o.password, - Replicas: o.Config.Replicas, - StorageSize: o.Config.StorageSize, - RetryJoinAddrs: retryJoinAddrs, + Namespace: o.Config.Namespace, + OpenBaoImage: o.Config.OpenBaoImage, + BankVaultsImage: o.Config.BankVaultsImage, + SecretsEngineName: o.Config.SecretsEngineName, + BaoUsername: o.Config.Username, + BaoPassword: o.password, + Replicas: o.Config.Replicas, + StorageSize: o.Config.StorageSize, + RetryJoinAddrs: retryJoinAddrs, + ImagePullSecretName: pullSecretName, } var buf bytes.Buffer @@ -590,6 +720,121 @@ func (o *OpenBaoInstaller) ensureUnsealSecret(secretsClient corev1client.SecretI return nil } +// imagePullSecretConfigured reports whether registry credentials were supplied +// (both username and password). Used to decide whether the pull secret is +// created and whether to wire it onto the openbao ServiceAccount. +func (o *OpenBaoInstaller) imagePullSecretConfigured() bool { + return o.Config.RegistryUser != "" && o.Config.RegistryPassword != "" +} + +// EnsureImagePullSecret creates or updates the dockerconfigjson Secret used to +// pull the OpenBao, bank-vaults configurer, and operator images from their +// (possibly private, possibly mirrored) registries. Credentials come from +// OMS_REGISTRY_USER/OMS_REGISTRY_PASSWORD: +// - both empty: no-op (clusters with node-level creds or public access). +// - both set: create/update the secret idempotently. +// - exactly one set: error, since a partial credential never works. +// +// The dockerconfigjson contains one auths entry per distinct registry host +// derived from the configured image refs, so a single secret authenticates to +// every registry the install pulls from. It is attached to the openbao +// ServiceAccount by ApplyVaultCR and to the operator chart by +// DeployBankVaultsOperator when credentials are present. +func (o *OpenBaoInstaller) EnsureImagePullSecret() error { + if o.Config.RegistryUser == "" && o.Config.RegistryPassword == "" { + return nil + } + if !o.imagePullSecretConfigured() { + return fmt.Errorf("incomplete registry credentials: set both OMS_REGISTRY_USER and OMS_REGISTRY_PASSWORD, or neither") + } + + hosts, err := registryHostsFor(o.Config.OpenBaoImage, o.Config.BankVaultsImage, o.Config.OperatorImage) + if err != nil { + return err + } + dockerConfig, err := buildDockerConfigJSON(hosts, o.Config.RegistryUser, o.Config.RegistryPassword) + if err != nil { + return fmt.Errorf("building docker config: %w", err) + } + + secretsClient := o.Clientset.CoreV1().Secrets(o.Config.Namespace) + existing, err := secretsClient.Get(o.ctx, imagePullSecretName, metav1.GetOptions{}) + if err != nil { + if !k8serrors.IsNotFound(err) { + return fmt.Errorf("checking image pull secret: %w", err) + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: imagePullSecretName, + Namespace: o.Config.Namespace, + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig}, + } + _, err = secretsClient.Create(o.ctx, secret, metav1.CreateOptions{}) + if err == nil { + return nil + } + if !k8serrors.IsAlreadyExists(err) { + return fmt.Errorf("creating image pull secret: %w", err) + } + // Created concurrently between our Get and Create — re-fetch and update. + existing, err = secretsClient.Get(o.ctx, imagePullSecretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("re-fetching image pull secret after create conflict: %w", err) + } + } + + // Update existing secret — preserve metadata, refresh type and data. + existing.Type = corev1.SecretTypeDockerConfigJson + existing.Data = map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig} + if _, err := secretsClient.Update(o.ctx, existing, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("updating image pull secret: %w", err) + } + return nil +} + +// buildDockerConfigJSON renders a .dockerconfigjson payload authenticating to +// each given registry host with the same username/password. +func buildDockerConfigJSON(hosts []string, username, password string) ([]byte, error) { + type dockerAuth struct { + Username string `json:"username"` + Password string `json:"password"` + Auth string `json:"auth"` + } + auth := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + auths := make(map[string]dockerAuth, len(hosts)) + for _, host := range hosts { + auths[host] = dockerAuth{Username: username, Password: password, Auth: auth} + } + return json.Marshal(map[string]map[string]dockerAuth{"auths": auths}) +} + +// registryHostsFor returns the distinct registry hosts of the given image +// references (e.g. "ghcr.io/codesphere-cloud/.../openbao:2.5.4" -> "ghcr.io"). +// Empty refs are skipped. The result is order-stable so the rendered secret is +// deterministic across runs. +func registryHostsFor(images ...string) ([]string, error) { + seen := make(map[string]struct{}) + var hosts []string + for _, image := range images { + if image == "" { + continue + } + ref, err := reference.ParseNormalizedNamed(image) + if err != nil { + return nil, fmt.Errorf("parsing image reference %q: %w", image, err) + } + host := reference.Domain(ref) + if _, ok := seen[host]; ok { + continue + } + seen[host] = struct{}{} + hosts = append(hosts, host) + } + return hosts, nil +} + // readinessTimeout returns how long to wait for all pods to become ready. // StatefulSet pods start sequentially and each Raft member must initialize and // join before the next comes up, so the wait grows with replica count: the diff --git a/internal/installer/openbao_test.go b/internal/installer/openbao_test.go index bd469338..9bfe1ee2 100644 --- a/internal/installer/openbao_test.go +++ b/internal/installer/openbao_test.go @@ -6,6 +6,7 @@ package installer_test import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -23,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -67,8 +69,8 @@ var _ = Describe("OpenBaoInstaller", func() { // No operator Deployment exists (fake clientset has nothing), so InstallChart is called helmMock.EXPECT().InstallChart(mock.Anything, mock.MatchedBy(func(cfg installer.ChartConfig) bool { return cfg.ReleaseName == "vault-operator" && - cfg.ChartName == "oci://ghcr.io/bank-vaults/helm-charts/vault-operator" && - cfg.Version == "1.22.5" && + cfg.ChartName == installer.DefaultBankVaultsChartRepo+"/vault-operator" && + cfg.Version == "1.24.0" && cfg.Namespace == "vault" && cfg.CreateNamespace == false }), mock.Anything).Return(nil) @@ -77,7 +79,10 @@ var _ = Describe("OpenBaoInstaller", func() { Helm: helmMock, Clientset: clientset, Logger: bootstrap.NewStepLogger(true), - Config: installer.OpenBaoInstallerConfig{Namespace: "vault"}, + Config: installer.OpenBaoInstallerConfig{ + Namespace: "vault", + OperatorChartRepo: installer.DefaultBankVaultsChartRepo, + }, } inst.SetCtx(ctx) @@ -85,6 +90,48 @@ var _ = Describe("OpenBaoInstaller", func() { Expect(err).ToNot(HaveOccurred()) }) + It("installs with mirror overrides: custom chart repo, operator image, and pull secret", func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "vault"}} + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + helmMock.EXPECT().FindRelease("vault", "vault-operator").Return(nil, nil) + // Credentials are set, so the chart registry login must happen first. + helmMock.EXPECT().LoginRegistry(mock.Anything, "mirror.example.com", "u", "p").Return(nil) + + helmMock.EXPECT().InstallChart(mock.Anything, mock.MatchedBy(func(cfg installer.ChartConfig) bool { + if cfg.ChartName != "oci://mirror.example.com/bank-vaults/helm-charts/vault-operator" { + return false + } + image, ok := cfg.Values["image"].(map[string]interface{}) + if !ok { + return false + } + if image["repository"] != "mirror.example.com/bank-vaults/vault-operator" || image["tag"] != "1.24.0" { + return false + } + secrets, ok := image["imagePullSecrets"].([]interface{}) + return ok && len(secrets) == 1 && + secrets[0].(map[string]interface{})["name"] == "openbao-registry" + }), mock.Anything).Return(nil) + + inst := &installer.OpenBaoInstaller{ + Helm: helmMock, + Clientset: clientset, + Logger: bootstrap.NewStepLogger(true), + Config: installer.OpenBaoInstallerConfig{ + Namespace: "vault", + RegistryUser: "u", + RegistryPassword: "p", + OperatorImage: "mirror.example.com/bank-vaults/vault-operator:1.24.0", + OperatorChartRepo: "oci://mirror.example.com/bank-vaults/helm-charts", + }, + } + inst.SetCtx(ctx) + + Expect(inst.DeployBankVaultsOperator()).To(Succeed()) + }) + It("performs fresh install when target namespace does not exist", func() { // Namespace "new-ns" is NOT created — FindRelease must be skipped. // No operator Deployment exists, so InstallChart is called directly. @@ -658,19 +705,139 @@ var _ = Describe("OpenBaoInstaller", func() { }) }) + Describe("EnsureImagePullSecret", func() { + newInstaller := func(user, password string) *installer.OpenBaoInstaller { + inst := &installer.OpenBaoInstaller{ + Clientset: clientset, + Logger: bootstrap.NewStepLogger(true), + Config: installer.OpenBaoInstallerConfig{ + Namespace: "vault", + RegistryUser: user, + RegistryPassword: password, + // Both default images live on ghcr.io. + OpenBaoImage: installer.DefaultOpenBaoImage, + BankVaultsImage: installer.DefaultBankVaultsImage, + }, + } + inst.SetCtx(ctx) + return inst + } + + It("creates a dockerconfigjson secret when both credentials are set", func() { + inst := newInstaller("gh-user", "gh-token") + Expect(inst.EnsureImagePullSecret()).To(Succeed()) + + secret, err := clientset.CoreV1().Secrets("vault").Get(ctx, "openbao-registry", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(secret.Type).To(Equal(corev1.SecretTypeDockerConfigJson)) + + var cfg struct { + Auths map[string]struct { + Username string `json:"username"` + Password string `json:"password"` + Auth string `json:"auth"` + } `json:"auths"` + } + Expect(json.Unmarshal(secret.Data[corev1.DockerConfigJsonKey], &cfg)).To(Succeed()) + entry, ok := cfg.Auths["ghcr.io"] + Expect(ok).To(BeTrue()) + Expect(entry.Username).To(Equal("gh-user")) + Expect(entry.Password).To(Equal("gh-token")) + Expect(entry.Auth).To(Equal(base64.StdEncoding.EncodeToString([]byte("gh-user:gh-token")))) + }) + + It("is a no-op when no credentials are set", func() { + inst := newInstaller("", "") + Expect(inst.EnsureImagePullSecret()).To(Succeed()) + + _, err := clientset.CoreV1().Secrets("vault").Get(ctx, "openbao-registry", metav1.GetOptions{}) + Expect(k8serrors.IsNotFound(err)).To(BeTrue()) + }) + + It("errors when only one credential is set", func() { + Expect(newInstaller("gh-user", "").EnsureImagePullSecret()).ToNot(Succeed()) + Expect(newInstaller("", "gh-token").EnsureImagePullSecret()).ToNot(Succeed()) + }) + + It("is idempotent and refreshes credentials on re-run", func() { + Expect(newInstaller("gh-user", "old-token").EnsureImagePullSecret()).To(Succeed()) + Expect(newInstaller("gh-user", "new-token").EnsureImagePullSecret()).To(Succeed()) + + secret, err := clientset.CoreV1().Secrets("vault").Get(ctx, "openbao-registry", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(string(secret.Data[corev1.DockerConfigJsonKey])).To(ContainSubstring("new-token")) + }) + + It("emits one deduplicated auths entry per distinct registry host", func() { + inst := &installer.OpenBaoInstaller{ + Clientset: clientset, + Logger: bootstrap.NewStepLogger(true), + Config: installer.OpenBaoInstallerConfig{ + Namespace: "vault", + RegistryUser: "u", + RegistryPassword: "p", + OpenBaoImage: "registry-a.example.com/openbao/openbao:2.5.4", + BankVaultsImage: "registry-b.example.com/bank-vaults/bank-vaults:1.19.0", + // Same host as OpenBaoImage — must be deduplicated. + OperatorImage: "registry-a.example.com/bank-vaults/vault-operator:1.24.0", + }, + } + inst.SetCtx(ctx) + Expect(inst.EnsureImagePullSecret()).To(Succeed()) + + secret, err := clientset.CoreV1().Secrets("vault").Get(ctx, "openbao-registry", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + Expect(json.Unmarshal(secret.Data[corev1.DockerConfigJsonKey], &cfg)).To(Succeed()) + Expect(cfg.Auths).To(HaveLen(2)) + Expect(cfg.Auths).To(HaveKey("registry-a.example.com")) + Expect(cfg.Auths).To(HaveKey("registry-b.example.com")) + }) + }) + + Describe("validateConfig image/chart defaults", func() { + It("backfills empty image and chart fields with the Default* values", func() { + inst := &installer.OpenBaoInstaller{ + Logger: bootstrap.NewStepLogger(true), + Config: installer.OpenBaoInstallerConfig{Namespace: "vault", Replicas: 1}, + } + Expect(inst.ValidateConfig()).To(Succeed()) + Expect(inst.Config.OpenBaoImage).To(Equal(installer.DefaultOpenBaoImage)) + Expect(inst.Config.BankVaultsImage).To(Equal(installer.DefaultBankVaultsImage)) + Expect(inst.Config.OperatorImage).To(Equal(installer.DefaultOperatorImage)) + Expect(inst.Config.OperatorChartRepo).To(Equal(installer.DefaultBankVaultsChartRepo)) + }) + + It("leaves explicitly-set overrides untouched", func() { + inst := &installer.OpenBaoInstaller{ + Logger: bootstrap.NewStepLogger(true), + Config: installer.OpenBaoInstallerConfig{ + Namespace: "vault", + Replicas: 1, + OpenBaoImage: "mirror.example.com/openbao:2.5.4", + }, + } + Expect(inst.ValidateConfig()).To(Succeed()) + Expect(inst.Config.OpenBaoImage).To(Equal("mirror.example.com/openbao:2.5.4")) + }) + }) + Describe("Vault CR template rendering", func() { // templateData mirrors the unexported vaultCRTemplateData struct // so the test can render the template independently. type templateData struct { - Namespace string - OpenBaoImage string - BankVaultsImage string - SecretsEngineName string - BaoUsername string - BaoPassword string - Replicas int - StorageSize string - RetryJoinAddrs []string + Namespace string + OpenBaoImage string + BankVaultsImage string + SecretsEngineName string + BaoUsername string + BaoPassword string + Replicas int + StorageSize string + RetryJoinAddrs []string + ImagePullSecretName string } renderTemplate := func(data templateData) []map[string]interface{} { @@ -707,6 +874,47 @@ var _ = Describe("OpenBaoInstaller", func() { return nil } + It("wires imagePullSecrets onto the openbao ServiceAccount when set", func() { + data := templateData{ + Namespace: "vault", + OpenBaoImage: "ghcr.io/codesphere-cloud/docker/quay.io/openbao/openbao-cs-patched:2.5.4", + BankVaultsImage: "ghcr.io/codesphere-cloud/docker/banzaicloud/bank-vaults:1.19.0", + SecretsEngineName: "cs-secrets-engine", + BaoUsername: "admin", + BaoPassword: "test-password", + Replicas: 1, + StorageSize: "10Gi", + RetryJoinAddrs: []string{"http://openbao-0.vault.svc.cluster.local:8200"}, + ImagePullSecretName: "openbao-registry", + } + + docs := renderTemplate(data) + sa := findDoc(docs, "ServiceAccount") + Expect(sa).ToNot(BeNil()) + pullSecrets := sa["imagePullSecrets"].([]interface{}) + Expect(pullSecrets).To(HaveLen(1)) + Expect(pullSecrets[0].(map[string]interface{})["name"]).To(Equal("openbao-registry")) + }) + + It("omits imagePullSecrets when no pull secret name is set", func() { + data := templateData{ + Namespace: "vault", + OpenBaoImage: "ghcr.io/codesphere-cloud/docker/quay.io/openbao/openbao-cs-patched:2.5.4", + BankVaultsImage: "ghcr.io/codesphere-cloud/docker/banzaicloud/bank-vaults:1.19.0", + SecretsEngineName: "cs-secrets-engine", + BaoUsername: "admin", + BaoPassword: "test-password", + Replicas: 1, + StorageSize: "10Gi", + RetryJoinAddrs: []string{"http://openbao-0.vault.svc.cluster.local:8200"}, + } + + docs := renderTemplate(data) + sa := findDoc(docs, "ServiceAccount") + Expect(sa).ToNot(BeNil()) + Expect(sa).ToNot(HaveKey("imagePullSecrets")) + }) + It("renders valid YAML with raft storage and PVC for replicas=1", func() { data := templateData{ Namespace: "vault", From 11cadfdf2bdd759e8e6ac2d28e4f3ceeb2ec8599 Mon Sep 17 00:00:00 2001 From: Jasin Aferkou Date: Tue, 30 Jun 2026 18:33:06 +0200 Subject: [PATCH 2/2] address comments --- internal/installer/openbao.go | 67 +++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/internal/installer/openbao.go b/internal/installer/openbao.go index ef1c774c..062d9924 100644 --- a/internal/installer/openbao.go +++ b/internal/installer/openbao.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/util/retry" ) //go:embed manifests/openbao/vault-cr.yaml @@ -758,40 +759,44 @@ func (o *OpenBaoInstaller) EnsureImagePullSecret() error { } secretsClient := o.Clientset.CoreV1().Secrets(o.Config.Namespace) - existing, err := secretsClient.Get(o.ctx, imagePullSecretName, metav1.GetOptions{}) - if err != nil { - if !k8serrors.IsNotFound(err) { - return fmt.Errorf("checking image pull secret: %w", err) - } - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: imagePullSecretName, - Namespace: o.Config.Namespace, - }, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig}, - } - _, err = secretsClient.Create(o.ctx, secret, metav1.CreateOptions{}) - if err == nil { - return nil - } - if !k8serrors.IsAlreadyExists(err) { - return fmt.Errorf("creating image pull secret: %w", err) - } - // Created concurrently between our Get and Create — re-fetch and update. - existing, err = secretsClient.Get(o.ctx, imagePullSecretName, metav1.GetOptions{}) + + // RetryOnConflict re-runs the closure on a resourceVersion conflict (409), + // covering both the concurrent-create race (Get says NotFound, Create says + // AlreadyExists) and a concurrent update between our Get and Update — each + // retry re-fetches the latest object before re-applying. + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + existing, err := secretsClient.Get(o.ctx, imagePullSecretName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("re-fetching image pull secret after create conflict: %w", err) + if !k8serrors.IsNotFound(err) { + return fmt.Errorf("checking image pull secret: %w", err) + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: imagePullSecretName, + Namespace: o.Config.Namespace, + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig}, + } + // Treat a concurrent create as a conflict so RetryOnConflict + // re-enters and falls through to the update branch. + if _, err := secretsClient.Create(o.ctx, secret, metav1.CreateOptions{}); err != nil { + if k8serrors.IsAlreadyExists(err) { + return k8serrors.NewConflict(corev1.Resource("secrets"), imagePullSecretName, err) + } + return fmt.Errorf("creating image pull secret: %w", err) + } + return nil } - } - // Update existing secret — preserve metadata, refresh type and data. - existing.Type = corev1.SecretTypeDockerConfigJson - existing.Data = map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig} - if _, err := secretsClient.Update(o.ctx, existing, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("updating image pull secret: %w", err) - } - return nil + // Update existing secret — preserve metadata, refresh type and data. + existing.Type = corev1.SecretTypeDockerConfigJson + existing.Data = map[string][]byte{corev1.DockerConfigJsonKey: dockerConfig} + if _, err := secretsClient.Update(o.ctx, existing, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("updating image pull secret: %w", err) + } + return nil + }) } // buildDockerConfigJSON renders a .dockerconfigjson payload authenticating to