diff --git a/NOTICE b/NOTICE index 7ad93ccf..29a1e3a3 100644 --- a/NOTICE +++ b/NOTICE @@ -267,6 +267,18 @@ Version: v0.6.0 License: Apache-2.0 License URL: https://github.com/distribution/reference/blob/v0.6.0/LICENSE +---------- +Module: github.com/docker/cli/cli/config +Version: v29.5.3 +License: Apache-2.0 +License URL: https://github.com/docker/cli/blob/v29.5.3/LICENSE + +---------- +Module: github.com/docker/docker-credential-helpers +Version: v0.9.8 +License: MIT +License URL: https://github.com/docker/docker-credential-helpers/blob/v0.9.8/LICENSE + ---------- Module: github.com/dylibso/observe-sdk/go Version: v0.0.0-20240828172851-9145d8ad07e1 @@ -507,6 +519,12 @@ Version: v0.7.0 License: BSD-3-Clause License URL: https://github.com/google/go-cmp/blob/v0.7.0/LICENSE +---------- +Module: github.com/google/go-containerregistry +Version: v0.21.6 +License: Apache-2.0 +License URL: https://github.com/google/go-containerregistry/blob/v0.21.6/LICENSE + ---------- Module: github.com/google/go-github/v74/github Version: v74.0.0 @@ -681,6 +699,36 @@ Version: v1.7.7 License: Apache-2.0 License URL: https://github.com/k8snetworkplumbingwg/network-attachment-definition-client/blob/v1.7.7/LICENSE +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: MIT +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: Apache-2.0 +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: BSD-3-Clause +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress/internal/snapref +Version: v1.18.6 +License: BSD-3-Clause +License URL: https://github.com/klauspost/compress/blob/v1.18.6/internal/snapref/LICENSE + +---------- +Module: github.com/klauspost/compress/zstd/internal/xxhash +Version: v1.18.6 +License: MIT +License URL: https://github.com/klauspost/compress/blob/v1.18.6/zstd/internal/xxhash/LICENSE.txt + ---------- Module: github.com/kr/fs Version: v0.1.0 diff --git a/cli/cmd/install_k0s_test.go b/cli/cmd/install_k0s_test.go index aa21eec0..21251a96 100644 --- a/cli/cmd/install_k0s_test.go +++ b/cli/cmd/install_k0s_test.go @@ -528,6 +528,7 @@ var _ = Describe("InstallK0sCmd", func() { c.FileWriter = mockFileWriter mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) + mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false).Return(nil) mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) diff --git a/cli/cmd/load.go b/cli/cmd/load.go new file mode 100644 index 00000000..eb651e9d --- /dev/null +++ b/cli/cmd/load.go @@ -0,0 +1,28 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/spf13/cobra" +) + +// LoadCmd represents the load command. +type LoadCmd struct { + cmd *cobra.Command +} + +func AddLoadCmd(rootCmd *cobra.Command, opts *GlobalOptions) { + load := LoadCmd{ + cmd: &cobra.Command{ + Use: "load", + Short: "Load resources into a local or custom registry", + Long: io.Long(`Load resources from external sources into a local or custom registry, + e.g. mirror images from GHCR.`), + }, + } + + AddCmd(rootCmd, load.cmd) + AddLoadImagesCmd(load.cmd, opts) +} diff --git a/cli/cmd/load_images.go b/cli/cmd/load_images.go new file mode 100644 index 00000000..824aaa62 --- /dev/null +++ b/cli/cmd/load_images.go @@ -0,0 +1,112 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "fmt" + + packageio "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/bom" + "github.com/codesphere-cloud/oms/internal/registry" + "github.com/spf13/cobra" +) + +const packageBomJSON = "bom.json" + +type LoadImagesCmd struct { + cmd *cobra.Command + Opts *LoadImagesOpts + Copier registry.ImageCopier + Env env.Env +} + +type LoadImagesOpts struct { + *GlobalOptions + DryRun bool + Force bool +} + +func AddLoadImagesCmd(load *cobra.Command, opts *GlobalOptions) { + c := &LoadImagesCmd{ + cmd: &cobra.Command{ + Use: "images ", + Short: "Mirror all Codesphere OCI images required to install package from Codesphere's registry", + Long: packageio.Long(`Mirror all Codesphere OCI images required to install package from Codesphere's registry into a target registry. + This is required for installations that require a custom registry, such as air-gapped environments. + + Ensure that the target registry is reachable and that you have permission to push images to it. + Registry authentication is read from local container registry credentials, such as Docker config, + Docker credential helpers, or Podman auth files. + + Logging in to the source and target registry before running this command is required. + + To use the custom registry, it must be configured using Codesphere's configuration file before installing Codesphere.`), + Example: formatExamples("load images", []packageio.Example{ + { + Cmd: "codesphere-v1.68.0.tar.gz registry.internal.example.com", + Desc: "Mirror every Codesphere OCI image required for Codesphere 1.68.0 into the target registry", + }, + }), + Args: cobra.ExactArgs(2), + }, + Opts: &LoadImagesOpts{ + GlobalOptions: opts, + }, + Env: env.NewEnv(), + } + + c.cmd.Flags().BoolVar(&c.Opts.DryRun, "dry-run", false, "Print planned copy operations without copying images") + c.cmd.Flags().BoolVarP(&c.Opts.Force, "force", "f", false, "Force new package extraction even if already extracted") + + AddCmd(load, c.cmd) + c.cmd.RunE = c.RunE +} + +func (c *LoadImagesCmd) RunE(cmd *cobra.Command, args []string) error { + pm := installer.NewPackage(c.Env.GetOmsWorkdir(), args[0]) + return c.LoadImagesFromPackage(cmd.Context(), pm, args[1]) +} + +func (c *LoadImagesCmd) LoadImagesFromPackage(ctx context.Context, pm installer.PackageManager, targetRegistry string) error { + bomPath, err := c.extractPackageBom(pm) + if err != nil { + return err + } + + return c.LoadImagesFromBomPath(ctx, bomPath, targetRegistry) +} + +func (c *LoadImagesCmd) LoadImagesFromBomPath(ctx context.Context, bomPath string, targetRegistry string) error { + config, err := bom.Parse(bomPath) + if err != nil { + return err + } + + copier := c.Copier + if !c.Opts.DryRun && copier == nil { + copier = registry.NewRegistryImageCopier(ctx) + } + + mirror := registry.Mirror{ + Copier: copier, + DryRun: c.Opts.DryRun, + } + _, err = mirror.MirrorImages(config.ImageReferencesForCodesphereRegistry(), targetRegistry) + if err != nil { + return fmt.Errorf("failed to load images from %s: %w", bomPath, err) + } + + return nil +} + +func (c *LoadImagesCmd) extractPackageBom(pm installer.PackageManager) (string, error) { + if err := pm.ExtractDependency(packageBomJSON, c.Opts.Force); err != nil { + return "", fmt.Errorf("failed to extract %s from package: %w", packageBomJSON, err) + } + + return pm.GetDependencyPath(packageBomJSON), nil +} diff --git a/cli/cmd/load_images_test.go b/cli/cmd/load_images_test.go new file mode 100644 index 00000000..3549ae55 --- /dev/null +++ b/cli/cmd/load_images_test.go @@ -0,0 +1,165 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd_test + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "os" + "path/filepath" + + "github.com/codesphere-cloud/oms/cli/cmd" + "github.com/codesphere-cloud/oms/internal/installer" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type fakeImageCopier struct { + calls [][2]string + err error +} + +func (r *fakeImageCopier) Copy(sourceRef string, destinationRef string) error { + r.calls = append(r.calls, [2]string{sourceRef, destinationRef}) + return r.err +} + +var _ = Describe("LoadImages", func() { + var ( + tempDir string + newPackage func() installer.PackageManager + ) + + BeforeEach(func() { + tempDir = GinkgoT().TempDir() + packagePath := filepath.Join(tempDir, "codesphere-test.tar.gz") + Expect(createLoadImagesTestPackage(packagePath, "bom.json", `{ + "components": { + "clusterPki": { + "files": { + "chart": { + "ociRef": "ghcr.io/codesphere-cloud/charts/cluster-pki:0.1.6" + } + }, + "containerImages": { + "cronjob": "ghcr.io/codesphere-cloud/docker/alpine/kubectl:1.34.2" + } + } + } + }`)).To(Succeed()) + + newPackage = func() installer.PackageManager { + return installer.NewPackage(filepath.Join(tempDir, "oms-workdir"), packagePath) + } + }) + + It("copies refs with fake copier", func() { + copier := &fakeImageCopier{} + c := &cmd.LoadImagesCmd{ + Opts: &cmd.LoadImagesOpts{}, + Copier: copier, + } + + err := c.LoadImagesFromPackage(context.TODO(), newPackage(), "registry.internal.example.com") + Expect(err).NotTo(HaveOccurred()) + Expect(copier.calls).To(Equal([][2]string{ + { + "ghcr.io/codesphere-cloud/charts/cluster-pki:0.1.6", + "registry.internal.example.com/codesphere-cloud/charts/cluster-pki:0.1.6", + }, + { + "ghcr.io/codesphere-cloud/docker/alpine/kubectl:1.34.2", + "registry.internal.example.com/codesphere-cloud/docker/alpine/kubectl:1.34.2", + }, + })) + }) + + It("does not call the copier during dry-run", func() { + copier := &fakeImageCopier{} + c := &cmd.LoadImagesCmd{ + Opts: &cmd.LoadImagesOpts{DryRun: true}, + Copier: copier, + } + + err := c.LoadImagesFromPackage(context.TODO(), newPackage(), "registry.internal.example.com") + Expect(err).NotTo(HaveOccurred()) + Expect(copier.calls).To(BeEmpty()) + }) + + It("propagates copy errors", func() { + copier := &fakeImageCopier{err: errors.New("copy failed")} + c := &cmd.LoadImagesCmd{ + Opts: &cmd.LoadImagesOpts{}, + Copier: copier, + } + + err := c.LoadImagesFromPackage(context.TODO(), newPackage(), "registry.internal.example.com") + Expect(err).To(MatchError(ContainSubstring("copy failed"))) + }) +}) + +func createLoadImagesTestPackage(filename string, bomName string, bomContent string) error { + depsContent, err := createLoadImagesDepsArchive(map[string]string{ + bomName: bomContent, + }) + if err != nil { + return err + } + + file, err := os.Create(filename) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + gzw := gzip.NewWriter(file) + defer func() { _ = gzw.Close() }() + + tw := tar.NewWriter(gzw) + defer func() { _ = tw.Close() }() + + header := &tar.Header{ + Name: "deps.tar.gz", + Mode: 0o600, + Size: int64(len(depsContent)), + } + if err := tw.WriteHeader(header); err != nil { + return err + } + _, err = tw.Write(depsContent) + return err +} + +func createLoadImagesDepsArchive(files map[string]string) ([]byte, error) { + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + + for name, content := range files { + header := &tar.Header{ + Name: name, + Mode: 0o600, + Size: int64(len(content)), + } + if err := tw.WriteHeader(header); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } + } + + if err := tw.Close(); err != nil { + return nil, err + } + if err := gzw.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go index f54f7644..545a37b0 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -72,6 +72,7 @@ func GetRootCmd() *cobra.Command { // Package commands AddListCmd(rootCmd, opts) AddDownloadCmd(rootCmd, opts) + AddLoadCmd(rootCmd, opts) AddInstallCmd(rootCmd, opts) AddInitCmd(rootCmd, opts) AddTemplateCmd(rootCmd, opts) diff --git a/docs/README.md b/docs/README.md index b17210fa..5cf11b6b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ like downloading new versions. * [oms install](oms_install.md) - Install Codesphere and other components * [oms licenses](oms_licenses.md) - Print license information * [oms list](oms_list.md) - List resources available through OMS +* [oms load](oms_load.md) - Load resources into a local or custom registry * [oms register](oms_register.md) - Register a new API key * [oms revoke](oms_revoke.md) - Revoke resources available through OMS * [oms smoketest](oms_smoketest.md) - Run smoke tests for Codesphere components diff --git a/docs/oms.md b/docs/oms.md index b17210fa..5cf11b6b 100644 --- a/docs/oms.md +++ b/docs/oms.md @@ -25,6 +25,7 @@ like downloading new versions. * [oms install](oms_install.md) - Install Codesphere and other components * [oms licenses](oms_licenses.md) - Print license information * [oms list](oms_list.md) - List resources available through OMS +* [oms load](oms_load.md) - Load resources into a local or custom registry * [oms register](oms_register.md) - Register a new API key * [oms revoke](oms_revoke.md) - Revoke resources available through OMS * [oms smoketest](oms_smoketest.md) - Run smoke tests for Codesphere components diff --git a/docs/oms_load.md b/docs/oms_load.md new file mode 100644 index 00000000..b534c494 --- /dev/null +++ b/docs/oms_load.md @@ -0,0 +1,20 @@ +## oms load + +Load resources into a local or custom registry + +### Synopsis + +Load resources from external sources into a local or custom registry, +e.g. mirror images from GHCR. + +### Options + +``` + -h, --help help for load +``` + +### SEE ALSO + +* [oms](oms.md) - Codesphere Operations Management System (OMS) +* [oms load images](oms_load_images.md) - Mirror all Codesphere OCI images required to install package from Codesphere's registry + diff --git a/docs/oms_load_images.md b/docs/oms_load_images.md new file mode 100644 index 00000000..fb5eb3d6 --- /dev/null +++ b/docs/oms_load_images.md @@ -0,0 +1,41 @@ +## oms load images + +Mirror all Codesphere OCI images required to install package from Codesphere's registry + +### Synopsis + +Mirror all Codesphere OCI images required to install package from Codesphere's registry into a target registry. +This is required for installations that require a custom registry, such as air-gapped environments. + +Ensure that the target registry is reachable and that you have permission to push images to it. +Registry authentication is read from local container registry credentials, such as Docker config, +Docker credential helpers, or Podman auth files. + +Logging in to the source and target registry before running this command is required. + +To use the custom registry, it must be configured using Codesphere's configuration file before installing Codesphere. + +``` +oms load images [flags] +``` + +### Examples + +``` +# Mirror every Codesphere OCI image required for Codesphere 1.68.0 into the target registry +$ oms load images codesphere-v1.68.0.tar.gz registry.internal.example.com + +``` + +### Options + +``` + --dry-run Print planned copy operations without copying images + -f, --force Force new package extraction even if already extracted + -h, --help help for images +``` + +### SEE ALSO + +* [oms load](oms_load.md) - Load resources into a local or custom registry + diff --git a/go.mod b/go.mod index 116bbfd4..a27275b4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,11 @@ module github.com/codesphere-cloud/oms go 1.26.4 replace ( + // GoReleaser pulls github.com/chrismellard/docker-credential-acr-env, + // which imports github.com/Azure/azure-sdk-for-go/version. Azure SDK + // v68 removed that package, so keep the legacy monorepo on a verified + // version that still provides it. + github.com/Azure/azure-sdk-for-go => github.com/Azure/azure-sdk-for-go v67.2.0+incompatible github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.4.1 github.com/kubernetes-incubator/external-storage => github.com/libopenstorage/external-storage v5.2.0+incompatible @@ -283,7 +288,7 @@ require ( github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect github.com/google/certificate-transparency-go v1.3.3 // indirect - github.com/google/go-containerregistry v0.21.6 // indirect + github.com/google/go-containerregistry v0.21.6 github.com/google/ko v0.18.2-0.20260407063826-ae9c7272d7de // indirect github.com/google/rpmpack v0.7.1 // indirect github.com/google/s2a-go v0.1.9 // indirect @@ -580,6 +585,7 @@ require ( github.com/libopenstorage/secrets v0.0.0-20240416031220-a17cf7f72c6c // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mattn/go-sqlite3 v1.14.28 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect github.com/moby/moby/api v1.54.2 // indirect @@ -617,6 +623,7 @@ require ( github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.step.sm/crypto v0.81.0 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.5 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/sync v0.21.0 // indirect diff --git a/go.sum b/go.sum index dfdca8de..44749830 100644 --- a/go.sum +++ b/go.sum @@ -2709,8 +2709,8 @@ github.com/Antonboom/nilnil v1.1.2 h1:aNlFuJhaEseXe4fHO3xbjXlSeEiQVYa2lEkWD2s2hA github.com/Antonboom/nilnil v1.1.2/go.mod h1:0ynwvphOLmAuMwTNDyBnDZmSwZoDpcFXmUHmzoHH2WA= github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= -github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v67.2.0+incompatible h1:Uu/Ww6ernvPTrpq31kITVTIm/I5jlJ1wjtEH/bmSB2k= +github.com/Azure/azure-sdk-for-go v67.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= @@ -4297,8 +4297,8 @@ github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4 github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= @@ -5188,8 +5188,8 @@ go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzK go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -go.step.sm/crypto v0.77.7 h1:6azC+pD678Vjju8yXnMDHCZJ+HzFaEmL3sCryiezTIA= -go.step.sm/crypto v0.77.7/go.mod h1:OW/2sEHwTtDKq70PvSQ5B0JGy/CrLyDKOiVy3YvZMTQ= +go.step.sm/crypto v0.81.0 h1:e+ouzpNt3Xm4dp7HGXhgYB5y4iFik3vh3phHKWmvugU= +go.step.sm/crypto v0.81.0/go.mod h1:fsTizqQeASjTXnbv9O00XtRlIuXRkCdoRiJNyXGQujc= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= diff --git a/internal/installer/bom/bom.go b/internal/installer/bom/bom.go index a532acb7..cfaa4fab 100644 --- a/internal/installer/bom/bom.go +++ b/internal/installer/bom/bom.go @@ -7,7 +7,10 @@ import ( "encoding/json" "fmt" "os" + "slices" + "sort" + "github.com/codesphere-cloud/oms/internal/registry" "github.com/distribution/reference" ) @@ -100,3 +103,31 @@ func (b *Config) GetCodesphereContainerImages() (map[string]string, error) { } return comp.ContainerImages, nil } + +// ImageReferencesForCodesphereRegistry returns unique image and OCI chart references for the Codesphere registry. +func (b *Config) ImageReferencesForCodesphereRegistry() []string { + if b == nil { + return nil + } + + refs := []string{} + for _, component := range b.Components { + for i := range component.ContainerImages { + imageRef := component.ContainerImages[i] + if registry.IsCodesphereImageReference(imageRef) && !slices.Contains(refs, imageRef) { + refs = append(refs, imageRef) + } + } + + for i := range component.Files { + fileRef := component.Files[i] + if registry.IsCodesphereImageReference(fileRef.OciRef) && !slices.Contains(refs, fileRef.OciRef) { + refs = append(refs, fileRef.OciRef) + } + } + } + + sort.Strings(refs) + + return refs +} diff --git a/internal/installer/bom/bom_test.go b/internal/installer/bom/bom_test.go index 0576388b..cf7d287f 100644 --- a/internal/installer/bom/bom_test.go +++ b/internal/installer/bom/bom_test.go @@ -8,10 +8,9 @@ import ( "os" "path/filepath" + "github.com/codesphere-cloud/oms/internal/installer/bom" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "github.com/codesphere-cloud/oms/internal/installer/bom" ) var _ = Describe("Bom", func() { @@ -44,7 +43,7 @@ var _ = Describe("Bom", func() { "glob": map[string]interface{}{ "cwd": "helm/codesphere", "include": "**/*", - "exclude": []string{"*.json5", "values-*.yaml"}, + "exclude": []string{"*.json", "values-*.yaml"}, }, }, "schemaDump": map[string]interface{}{ @@ -213,4 +212,39 @@ var _ = Describe("Bom", func() { Expect(images).To(BeEmpty()) }) }) + + Describe("CodesphereImageReferences", func() { + It("returns sorted unique Codesphere refs from typed BOM fields", func() { + cfg := &bom.Config{ + Components: map[string]bom.ComponentConfig{ + "gateway": { + Files: map[string]bom.FileRef{ + "chart": { + OciRef: "ghcr.io/codesphere-cloud/charts/gateway:0.13.3", + }, + }, + ContainerImages: map[string]string{ + "envoyProxy": "ghcr.io/codesphere-cloud/docker/envoyproxy/envoy:distroless-v1.37.0", + "ingressNginx": "registry.k8s.io/ingress-nginx/controller:v1.13.2", + }, + }, + "duplicate": { + Files: map[string]bom.FileRef{ + "chart": { + OciRef: "ghcr.io/codesphere-cloud/charts/gateway:0.13.3", + }, + }, + ContainerImages: map[string]string{ + "missingTag": "ghcr.io/codesphere-cloud/docker/alpine/kubectl", + }, + }, + }, + } + + Expect(cfg.ImageReferencesForCodesphereRegistry()).To(Equal([]string{ + "ghcr.io/codesphere-cloud/charts/gateway:0.13.3", + "ghcr.io/codesphere-cloud/docker/envoyproxy/envoy:distroless-v1.37.0", + })) + }) + }) }) diff --git a/internal/registry/image_copy.go b/internal/registry/image_copy.go new file mode 100644 index 00000000..fb97b107 --- /dev/null +++ b/internal/registry/image_copy.go @@ -0,0 +1,31 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package registry + +import ( + "context" + "fmt" + + "github.com/google/go-containerregistry/pkg/crane" +) + +type ImageCopier interface { + Copy(sourceRef string, destinationRef string) error +} + +type RegistryImageCopier struct { + ctx context.Context +} + +func NewRegistryImageCopier(ctx context.Context) *RegistryImageCopier { + return &RegistryImageCopier{ctx: ctx} +} + +func (c *RegistryImageCopier) Copy(sourceRef string, destinationRef string) error { + if err := crane.Copy(sourceRef, destinationRef, crane.WithContext(c.ctx)); err != nil { + return fmt.Errorf("failed to copy %s to %s: %w", sourceRef, destinationRef, err) + } + + return nil +} diff --git a/internal/registry/mirror.go b/internal/registry/mirror.go new file mode 100644 index 00000000..66d24255 --- /dev/null +++ b/internal/registry/mirror.go @@ -0,0 +1,92 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package registry + +import ( + "fmt" + "log" + "strings" + + "github.com/distribution/reference" +) + +const CodesphereRegistry = "ghcr.io" + +type Mirror struct { + Copier ImageCopier + DryRun bool +} + +func (m *Mirror) MirrorImages(sourceRefs []string, targetRegistry string) (int, error) { + if len(sourceRefs) == 0 { + return 0, fmt.Errorf("no %s image references found in BOM", CodesphereRegistry) + } + if !m.DryRun && m.Copier == nil { + return 0, fmt.Errorf("image copier is required") + } + + for _, sourceRef := range sourceRefs { + destinationRef, err := TargetImageRef(sourceRef, targetRegistry) + if err != nil { + return 0, err + } + + log.Printf("Mirroring %s -> %s", sourceRef, destinationRef) + if m.DryRun { + continue + } + + if err := m.Copier.Copy(sourceRef, destinationRef); err != nil { + return 0, err + } + } + + log.Printf("Processed %d %s image references", len(sourceRefs), CodesphereRegistry) + + return len(sourceRefs), nil +} + +func TargetImageRef(sourceRef string, targetRegistry string) (string, error) { + if targetRegistry == "" { + return "", fmt.Errorf("target registry must not be empty") + } + ref, err := reference.ParseNormalizedNamed(sourceRef) + if err != nil { + return "", fmt.Errorf("invalid source reference %s: %w", sourceRef, err) + } + if reference.Domain(ref) != CodesphereRegistry { + return "", fmt.Errorf("source reference %s is not a Codesphere image reference", sourceRef) + } + return targetRegistry + "/" + strings.TrimPrefix(sourceRef, CodesphereRegistry+"/"), nil +} + +// IsCodesphereImageReference returns true if the given value is a valid Codesphere image reference. +func IsCodesphereImageReference(value string) bool { + if value == "" { + return false + } + + named, err := reference.ParseNormalizedNamed(value) + if err != nil { + return false + } + if reference.Domain(named) != CodesphereRegistry { + return false + } + if !hasTagOrDigest(named) { + return false + } + + return true +} + +func hasTagOrDigest(named reference.Named) bool { + if _, ok := named.(reference.Tagged); ok { + return true + } + if _, ok := named.(reference.Digested); ok { + return true + } + return false +} diff --git a/internal/registry/mirror_test.go b/internal/registry/mirror_test.go new file mode 100644 index 00000000..b67ab757 --- /dev/null +++ b/internal/registry/mirror_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package registry_test + +import ( + "errors" + + "github.com/codesphere-cloud/oms/internal/installer/bom" + "github.com/codesphere-cloud/oms/internal/registry" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type fakeCopier struct { + calls [][2]string + err error +} + +func (c *fakeCopier) Copy(sourceRef string, destinationRef string) error { + c.calls = append(c.calls, [2]string{sourceRef, destinationRef}) + return c.err +} + +var _ = Describe("TargetImageRef", func() { + It("maps a Codesphere image reference into the target registry", func() { + target, err := registry.TargetImageRef( + "ghcr.io/codesphere-cloud/charts/gateway:0.13.3", + "registry.internal.example.com/mirror", + ) + + Expect(err).NotTo(HaveOccurred()) + Expect(target).To(Equal("registry.internal.example.com/mirror/codesphere-cloud/charts/gateway:0.13.3")) + }) + + It("rejects references outside the Codesphere registry", func() { + _, err := registry.TargetImageRef("quay.io/prometheus/prometheus:v2.51.0", "registry.internal.example.com") + + Expect(err).To(HaveOccurred()) + }) + + It("rejects an empty target registry", func() { + _, err := registry.TargetImageRef("ghcr.io/codesphere-cloud/charts/gateway:0.13.3", "") + + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("MirrorGHCRImages", func() { + var config *bom.Config + + BeforeEach(func() { + config = &bom.Config{ + Components: map[string]bom.ComponentConfig{ + "cluster-pki": { + ContainerImages: map[string]string{ + "cronjob": "ghcr.io/codesphere-cloud/docker/alpine/kubectl:1.34.2", + }, + }, + }, + } + }) + + It("copies all refs", func() { + copier := &fakeCopier{} + mirror := ®istry.Mirror{Copier: copier} + + count, err := mirror.MirrorImages(config.ImageReferencesForCodesphereRegistry(), "registry.internal.example.com") + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(1)) + Expect(copier.calls).To(Equal([][2]string{{ + "ghcr.io/codesphere-cloud/docker/alpine/kubectl:1.34.2", + "registry.internal.example.com/codesphere-cloud/docker/alpine/kubectl:1.34.2", + }})) + }) + + It("does not copy on dry run", func() { + copier := &fakeCopier{} + mirror := ®istry.Mirror{Copier: copier, DryRun: true} + + count, err := mirror.MirrorImages(config.ImageReferencesForCodesphereRegistry(), "registry.internal.example.com") + Expect(err).NotTo(HaveOccurred()) + Expect(count).To(Equal(1)) + Expect(copier.calls).To(BeEmpty()) + }) + + It("propagates copy errors", func() { + mirror := ®istry.Mirror{Copier: &fakeCopier{err: errors.New("copy failed")}} + + _, err := mirror.MirrorImages(config.ImageReferencesForCodesphereRegistry(), "registry.internal.example.com") + Expect(err).To(MatchError(ContainSubstring("copy failed"))) + }) +}) diff --git a/internal/registry/registry_suite_test.go b/internal/registry/registry_suite_test.go new file mode 100644 index 00000000..8d212fa1 --- /dev/null +++ b/internal/registry/registry_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package registry_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestRegistry(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Registry Suite") +} diff --git a/internal/tmpl/NOTICE b/internal/tmpl/NOTICE index 7ad93ccf..29a1e3a3 100644 --- a/internal/tmpl/NOTICE +++ b/internal/tmpl/NOTICE @@ -267,6 +267,18 @@ Version: v0.6.0 License: Apache-2.0 License URL: https://github.com/distribution/reference/blob/v0.6.0/LICENSE +---------- +Module: github.com/docker/cli/cli/config +Version: v29.5.3 +License: Apache-2.0 +License URL: https://github.com/docker/cli/blob/v29.5.3/LICENSE + +---------- +Module: github.com/docker/docker-credential-helpers +Version: v0.9.8 +License: MIT +License URL: https://github.com/docker/docker-credential-helpers/blob/v0.9.8/LICENSE + ---------- Module: github.com/dylibso/observe-sdk/go Version: v0.0.0-20240828172851-9145d8ad07e1 @@ -507,6 +519,12 @@ Version: v0.7.0 License: BSD-3-Clause License URL: https://github.com/google/go-cmp/blob/v0.7.0/LICENSE +---------- +Module: github.com/google/go-containerregistry +Version: v0.21.6 +License: Apache-2.0 +License URL: https://github.com/google/go-containerregistry/blob/v0.21.6/LICENSE + ---------- Module: github.com/google/go-github/v74/github Version: v74.0.0 @@ -681,6 +699,36 @@ Version: v1.7.7 License: Apache-2.0 License URL: https://github.com/k8snetworkplumbingwg/network-attachment-definition-client/blob/v1.7.7/LICENSE +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: MIT +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: Apache-2.0 +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress +Version: v1.18.6 +License: BSD-3-Clause +License URL: https://github.com/klauspost/compress/blob/v1.18.6/LICENSE + +---------- +Module: github.com/klauspost/compress/internal/snapref +Version: v1.18.6 +License: BSD-3-Clause +License URL: https://github.com/klauspost/compress/blob/v1.18.6/internal/snapref/LICENSE + +---------- +Module: github.com/klauspost/compress/zstd/internal/xxhash +Version: v1.18.6 +License: MIT +License URL: https://github.com/klauspost/compress/blob/v1.18.6/zstd/internal/xxhash/LICENSE.txt + ---------- Module: github.com/kr/fs Version: v0.1.0