From 4da18b72249eac98bade1124cc2cfe66d07b3777 Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Fri, 9 May 2025 12:33:23 -0500 Subject: [PATCH 1/6] Add external k8s cluster usecase --- gcp/external-k8s/README.md | 147 ++++++++++++++++++ gcp/{vpc => external-k8s}/firewall.k.yaml | 71 ++++----- .../fixtures}/gcp-environment-crd.yaml | 0 .../fixtures/k8s-external-test.yaml | 22 +++ gcp/external-k8s/iam_policy_member.k.yaml | 29 ++++ .../iam_policy_member_ident_ksa.k.yaml | 29 ++++ .../iam_policy_member_token_ksa.k.yaml | 29 ++++ gcp/external-k8s/iam_service_account.k.yaml | 26 ++++ gcp/external-k8s/k8s_cluster.k.yaml | 68 ++++++++ gcp/external-k8s/k8s_cluster_config.k.yaml | 21 +++ gcp/external-k8s/k8s_service_account.k.yaml | 21 +++ gcp/external-k8s/metadata.k.yaml | 26 ++++ gcp/{vpc => external-k8s}/network.k.yaml | 31 ++-- gcp/{vpc => external-k8s}/subnet.k.yaml | 51 +++--- gcp/external-k8s/workflow.k.yaml | 108 +++++++++++++ gcp/vpc/README.md | 5 - gcp/vpc/config.k.yaml | 34 ---- gcp/vpc/gcp-environment.yaml | 7 - gcp/vpc/workflow.k.yaml | 48 ------ 19 files changed, 608 insertions(+), 165 deletions(-) create mode 100644 gcp/external-k8s/README.md rename gcp/{vpc => external-k8s}/firewall.k.yaml (55%) rename gcp/{vpc => external-k8s/fixtures}/gcp-environment-crd.yaml (100%) create mode 100644 gcp/external-k8s/fixtures/k8s-external-test.yaml create mode 100644 gcp/external-k8s/iam_policy_member.k.yaml create mode 100644 gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml create mode 100644 gcp/external-k8s/iam_policy_member_token_ksa.k.yaml create mode 100644 gcp/external-k8s/iam_service_account.k.yaml create mode 100644 gcp/external-k8s/k8s_cluster.k.yaml create mode 100644 gcp/external-k8s/k8s_cluster_config.k.yaml create mode 100644 gcp/external-k8s/k8s_service_account.k.yaml create mode 100644 gcp/external-k8s/metadata.k.yaml rename gcp/{vpc => external-k8s}/network.k.yaml (72%) rename gcp/{vpc => external-k8s}/subnet.k.yaml (53%) create mode 100644 gcp/external-k8s/workflow.k.yaml delete mode 100644 gcp/vpc/README.md delete mode 100644 gcp/vpc/config.k.yaml delete mode 100644 gcp/vpc/gcp-environment.yaml delete mode 100644 gcp/vpc/workflow.k.yaml diff --git a/gcp/external-k8s/README.md b/gcp/external-k8s/README.md new file mode 100644 index 0000000..6366f35 --- /dev/null +++ b/gcp/external-k8s/README.md @@ -0,0 +1,147 @@ +# GCP Environment Workflow + +This workflow provisions and configures a complete GCP-based Kubernetes environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. + +It creates the necessary network infrastructure, a GKE cluster, service accounts, and sets up Workload Identity to allow Kubernetes ServiceAccounts to impersonate GCP Service Accounts securely. + +## Overview + +- **CRD Reference**: Targets the `GcpEnvironment` CRD defined under `acme.example.com/v1beta1`. +- **Inputs**: Expects a `parent` object. +- **Output**: A fully provisioned Kubernetes cluster with Workload Identity configured. + +## Workflow Steps + +| Step | Description | +|------|-------------| +| `metadata` | Generates standard metadata used across all resources. | +| `network` | Creates a VPC network for the environment. | +| `subnet` | Provisions a subnet within the created VPC with CIDR `10.10.0.0/16`. | +| `firewall` | Configures firewall rules for the subnet. | +| `k8sCluster` | Provisions a GKE cluster using the created VPC and subnet. | +| `k8sClusterSecret` | Extracts the Kubernetes API endpoint and CA cert to create a Kubernetes config secret. | +| `workloadIdentityServiceAccount` | Creates a GCP IAM Service Account used for Workload Identity. | +| `workloadIdentityPolicyMember` | Grants the IAM service account the necessary project-level permissions. | +| `ksaServiceAccount` | Creates a Kubernetes Service Account (KSA) annotated to impersonate the IAM Service Account. | +| `workloadIdentityPolicyMemberIdentKSA` | Grants the IAM service account permission to be impersonated by the KSA (`iam.workloadIdentityUser`). | +| `workloadIdentityPolicyMemberTokenKSA` | Grants the KSA permission to create identity tokens on behalf of the IAM service account (`iam.serviceAccountTokenCreator`). | + +## Requirements + +- GCP project with appropriate APIs enabled (IAM, GKE, Compute Engine). +- Koreo engine with support for `ResourceFunction` and `ValueFunction`. +- IAM permissions to create networks, clusters, service accounts, and IAM policies. + +## Notes + +- The workflow uses [Google Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) to enable secure communication between GCP and Kubernetes workloads. +- Each resource function should be implemented to create or reconcile its respective GCP or K8s object. +- The `metadata` step centralizes naming and labeling to ensure consistent tagging and traceability. + + +### Notes +- Ensure Koreo service account has permissions to edit IAM policy + - I made it owner of the account, but this could be restricted I am sure. +- Ensure Identity and Access Management API is enabled +- Ensure Kubernetes API is enabled +- Ensure Cloud Resource Manager API is enabled +- To get the `koreo` service account you will need edit the above workflow to use the koreo service account or run the following commands. + +``` sh +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:@.iam.gserviceaccount.com" \ + --role="roles/container.developer" +``` + +``` sh +gcloud iam service-accounts add-iam-policy-binding \ + @.iam.gserviceaccount.com \ + --role "roles/iam.workloadIdentityUser" \ + --member "serviceAccount:.svc.id.goog[/koreo]" +``` + +``` sh +gcloud iam service-accounts add-iam-policy-binding \ + @.iam.gserviceaccount.com \ + --role "roles/iam.serviceAccountTokenCreator" \ + --member "serviceAccount:.svc.id.goog[/koreo]" +``` + +``` sh +# Edit serviceAccountName to koreo or whatever the workflow generates ("-workload-ksa") +# Edit the secret name references to the GcpEnvironmentName +kubectl apply -f ./fixtures/k8s-external-test.yaml + +kubectl exec -it -n external-k8s -- /bin/sh + +$ pip install pykube-ng google-auth +$ python +``` + +Run the following script to ensure your connection is working. + +``` python +import os +import base64 +import tempfile +import yaml +from google.auth.transport.requests import Request +from google.auth import default +import pykube +from pykube import HTTPClient, KubeConfig + +# Decode credentials passed in as env vars +cert = os.environ["GKE_CLUSTER_CA"].encode() +if cert.startswith(b"LS0"): + cert = base64.b64decode(cert) # only decode once if still encoded + +token = None +creds, _ = default() +creds.refresh(Request()) +token = creds.token + +# Clean up endpoint +endpoint = os.environ["GKE_CLUSTER_ENDPOINT"] +if endpoint.startswith("https://"): + endpoint = endpoint[len("https://"):] + +# Build minimal kubeconfig +kubeconfig = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [{ + "name": "inline", + "cluster": { + "server": f"https://{endpoint}", + "certificate-authority-data": base64.b64encode(cert).decode(), + }, + }], + "users": [{ + "name": "inline", + "user": { + "token": token, + }, + }], + "contexts": [{ + "name": "inline", + "context": { + "cluster": "inline", + "user": "inline", + }, + }], + "current-context": "inline", +} + +# Write to a named temporary file +kubeconfig_path = "/tmp/inline-kubeconfig.yaml" +with open(kubeconfig_path, "w") as f: + yaml.safe_dump(kubeconfig, f) + +print(f"Wrote kubeconfig to {kubeconfig_path}") + +# Load and use it +api = HTTPClient(KubeConfig.from_file(kubeconfig_path)) + +for pod in pykube.Pod.objects(api).filter(namespace="kube-system"): + print(pod.name) +``` diff --git a/gcp/vpc/firewall.k.yaml b/gcp/external-k8s/firewall.k.yaml similarity index 55% rename from gcp/vpc/firewall.k.yaml rename to gcp/external-k8s/firewall.k.yaml index 29bf5c2..a3a1283 100644 --- a/gcp/vpc/firewall.k.yaml +++ b/gcp/external-k8s/firewall.k.yaml @@ -1,7 +1,8 @@ +--- apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-firewall + name: gcp-environment-firewall spec: apiConfig: apiVersion: compute.cnrm.cloud.google.com/v1beta1 @@ -11,65 +12,61 @@ spec: resource: spec: - allow: - - ports: - - 0-65535 - protocol: tcp - - ports: - - 0-65535 - protocol: udp - - protocol: icmp - direction: INGRESS networkRef: name: =inputs.networkName - sourceRanges: + allow: + - protocol: tcp + ports: ["0-65535"] + - protocol: udp + ports: ["0-65535"] + - protocol: icmp + sourceRanges: - =inputs.subnet.range - + direction: INGRESS + postconditions: - assert: =resource.config_connect_ready() retry: - delay: 10 message: Waiting for firewall to be created - + delay: 10 + return: name: =inputs.metadata.name --- apiVersion: koreo.dev/v1beta1 kind: FunctionTest metadata: - name: gcp-environment-firewall-test + name: gcp-environment-firewall-test spec: functionRef: kind: ResourceFunction name: gcp-environment-firewall inputs: + networkName: test-network-name metadata: name: test-network namespace: test-namespace - networkName: test-network-name subnet: name: subnet1 range: 10.0.0.0/20 - + testCases: - - expectResource: - apiVersion: compute.cnrm.cloud.google.com/v1beta1 - kind: ComputeFirewall - metadata: - name: test-network - namespace: test-namespace - spec: - allow: - - ports: - - 0-65535 - protocol: tcp - - ports: - - 0-65535 - protocol: udp - - protocol: icmp - direction: INGRESS - networkRef: - name: test-network-name - sourceRanges: - - 10.0.0.0/20 + - expectResource: + apiVersion: compute.cnrm.cloud.google.com/v1beta1 + kind: ComputeFirewall + metadata: + name: test-network + namespace: test-namespace + spec: + allow: + - protocol: tcp + ports: ["0-65535"] + - protocol: udp + ports: ["0-65535"] + - protocol: icmp + sourceRanges: + - 10.0.0.0/20 + direction: INGRESS + networkRef: + name: test-network-name \ No newline at end of file diff --git a/gcp/vpc/gcp-environment-crd.yaml b/gcp/external-k8s/fixtures/gcp-environment-crd.yaml similarity index 100% rename from gcp/vpc/gcp-environment-crd.yaml rename to gcp/external-k8s/fixtures/gcp-environment-crd.yaml diff --git a/gcp/external-k8s/fixtures/k8s-external-test.yaml b/gcp/external-k8s/fixtures/k8s-external-test.yaml new file mode 100644 index 0000000..78cc681 --- /dev/null +++ b/gcp/external-k8s/fixtures/k8s-external-test.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: external-k8s + namespace: skybit-systems-dev +spec: + serviceAccountName: dev-workload-ksa + containers: + - name: app + image: python:3.11 + command: ["sleep", "infinity"] + env: + - name: GKE_CLUSTER_ENDPOINT + valueFrom: + secretKeyRef: + name: dev + key: endpoint + - name: GKE_CLUSTER_CA + valueFrom: + secretKeyRef: + name: dev + key: ca.crt diff --git a/gcp/external-k8s/iam_policy_member.k.yaml b/gcp/external-k8s/iam_policy_member.k.yaml new file mode 100644 index 0000000..cfc7466 --- /dev/null +++ b/gcp/external-k8s/iam_policy_member.k.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ResourceFunction +metadata: + name: gcp-environment-iam-policy-member +spec: + locals: + name: =inputs.metadata.name + "-workload-identity" + apiConfig: + apiVersion: iam.cnrm.cloud.google.com/v1beta1 + kind: IAMPolicyMember + name: =locals.name + namespace: =inputs.metadata.namespace + + resource: + spec: + member: =inputs.saMember + role: roles/container.developer + resourceRef: + kind: Project + external: ="projects/" + inputs.projectId + postconditions: + - assert: =resource.config_connect_ready() + retry: + message: Waiting for connector to be created + delay: 10 + + return: + name: =locals.name diff --git a/gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml b/gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml new file mode 100644 index 0000000..be21819 --- /dev/null +++ b/gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ResourceFunction +metadata: + name: gcp-environment-iam-policy-member-ident-ksa +spec: + locals: + name: =inputs.metadata.name + "-ksa-ident-user" + apiConfig: + apiVersion: iam.cnrm.cloud.google.com/v1beta1 + kind: IAMPolicyMember + name: =locals.name + namespace: =inputs.metadata.namespace + + resource: + spec: + member: ="serviceAccount:konfig-platform.svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" + role: roles/iam.workloadIdentityUser + resourceRef: + kind: IAMServiceAccount + name: =inputs.saName + postconditions: + - assert: =resource.config_connect_ready() + retry: + message: Waiting for identity policy member to be created + delay: 10 + + return: + name: =locals.name diff --git a/gcp/external-k8s/iam_policy_member_token_ksa.k.yaml b/gcp/external-k8s/iam_policy_member_token_ksa.k.yaml new file mode 100644 index 0000000..3f60bc1 --- /dev/null +++ b/gcp/external-k8s/iam_policy_member_token_ksa.k.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ResourceFunction +metadata: + name: gcp-environment-iam-policy-member-token-ksa +spec: + locals: + name: =inputs.metadata.name + "-ksa-token-user" + apiConfig: + apiVersion: iam.cnrm.cloud.google.com/v1beta1 + kind: IAMPolicyMember + name: =locals.name + namespace: =inputs.metadata.namespace + + resource: + spec: + member: ="serviceAccount:konfig-platform.svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" + role: roles/iam.serviceAccountTokenCreator + resourceRef: + kind: IAMServiceAccount + name: =inputs.saName + postconditions: + - assert: =resource.config_connect_ready() + retry: + message: Waiting for token member to be created + delay: 10 + + return: + name: =locals.name diff --git a/gcp/external-k8s/iam_service_account.k.yaml b/gcp/external-k8s/iam_service_account.k.yaml new file mode 100644 index 0000000..66b6932 --- /dev/null +++ b/gcp/external-k8s/iam_service_account.k.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ResourceFunction +metadata: + name: gcp-environment-iam-service-account +spec: + locals: + name: =inputs.metadata.name + "-workload-identity" + apiConfig: + apiVersion: iam.cnrm.cloud.google.com/v1beta1 + kind: IAMServiceAccount + name: =locals.name + namespace: =inputs.metadata.namespace + + resource: + spec: + displayName: "GKE remote access via WI" + postconditions: + - assert: =resource.config_connect_ready() + retry: + message: Waiting for connector to be created + delay: 10 + + return: + name: =locals.name + member: =resource.status.member diff --git a/gcp/external-k8s/k8s_cluster.k.yaml b/gcp/external-k8s/k8s_cluster.k.yaml new file mode 100644 index 0000000..081f92b --- /dev/null +++ b/gcp/external-k8s/k8s_cluster.k.yaml @@ -0,0 +1,68 @@ +--- + apiVersion: koreo.dev/v1beta1 + kind: ResourceFunction + metadata: + name: gcp-environment-k8s-cluster + spec: + apiConfig: + apiVersion: container.cnrm.cloud.google.com/v1beta1 + kind: ContainerCluster + name: =inputs.metadata.name + namespace: =inputs.metadata.namespace + + resource: + spec: + description: A large regional VPC-native cluster set up with special networking considerations. + location: us-central1 + initialNodeCount: 1 + defaultMaxPodsPerNode: 16 + nodeLocations: + - us-central1-a + - us-central1-b + - us-central1-c + - us-central1-f + workloadIdentityConfig: + workloadPool: skybit-systems.svc.id.goog + networkingMode: VPC_NATIVE + networkRef: + name: =inputs.networkName + subnetworkRef: + name: =inputs.subnetworkName + ipAllocationPolicy: + servicesSecondaryRangeName: servicesrange + clusterSecondaryRangeName: clusterrange + clusterAutoscaling: + enabled: true + autoscalingProfile: BALANCED + resourceLimits: + - resourceType: cpu + maximum: 100 + minimum: 10 + - resourceType: memory + maximum: 1000 + minimum: 100 + maintenancePolicy: + dailyMaintenanceWindow: + startTime: 00:00 + releaseChannel: + channel: STABLE + enableBinaryAuthorization: true + enableIntranodeVisibility: true + enableShieldedNodes: true + addonsConfig: + networkPolicyConfig: + disabled: false + networkPolicy: + enabled: true + podSecurityPolicyConfig: + enabled: true + + postconditions: + - assert: =resource.config_connect_ready() + retry: + message: Waiting for GKE Cluster to be created + delay: 30 + + return: + endpoint: ="https://" + resource.status.observedState.privateClusterConfig.publicEndpoint + caCert: =resource.status.observedState.masterAuth.clusterCaCertificate \ No newline at end of file diff --git a/gcp/external-k8s/k8s_cluster_config.k.yaml b/gcp/external-k8s/k8s_cluster_config.k.yaml new file mode 100644 index 0000000..cbebdfd --- /dev/null +++ b/gcp/external-k8s/k8s_cluster_config.k.yaml @@ -0,0 +1,21 @@ +--- + apiVersion: koreo.dev/v1beta1 + kind: ResourceFunction + metadata: + name: gcp-environment-k8s-cluster-config + spec: + apiConfig: + apiVersion: v1 + kind: Secret + name: =inputs.metadata.name + namespace: =inputs.metadata.namespace + + resource: + type: Opaque + data: + endpoint: =b64encode(inputs.endpoint) + ca.crt: =b64encode(inputs.caCert) + + return: + name: =inputs.metadata.name + \ No newline at end of file diff --git a/gcp/external-k8s/k8s_service_account.k.yaml b/gcp/external-k8s/k8s_service_account.k.yaml new file mode 100644 index 0000000..497cb43 --- /dev/null +++ b/gcp/external-k8s/k8s_service_account.k.yaml @@ -0,0 +1,21 @@ +--- + apiVersion: koreo.dev/v1beta1 + kind: ResourceFunction + metadata: + name: gcp-environment-k8s-service-account + spec: + locals: + name: =inputs.metadata.name + "-workload-ksa" + apiConfig: + apiVersion: v1 + kind: ServiceAccount + name: =locals.name + namespace: =inputs.metadata.namespace + + resource: + metadata: + annotations: + iam.gke.io/gcp-service-account: =inputs.saName + "@" + inputs.projectId + ".iam.gserviceaccount.com" + return: + name: =locals.name + \ No newline at end of file diff --git a/gcp/external-k8s/metadata.k.yaml b/gcp/external-k8s/metadata.k.yaml new file mode 100644 index 0000000..6c9ef91 --- /dev/null +++ b/gcp/external-k8s/metadata.k.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ValueFunction +metadata: + name: gcp-environment-metadata +spec: + return: + name: =inputs.environmentResourceName + namespace: =inputs.environmentNamespace +--- +apiVersion: koreo.dev/v1beta1 +kind: FunctionTest +metadata: + name: gcp-environment-metadata-test +spec: + functionRef: + name: gcp-environment-metadata + kind: ValueFunction + inputs: + environmentResourceName: example-test + environmentNamespace: example-ns + testCases: + - expectReturn: + name: example-test + namespace: example-ns + \ No newline at end of file diff --git a/gcp/vpc/network.k.yaml b/gcp/external-k8s/network.k.yaml similarity index 72% rename from gcp/vpc/network.k.yaml rename to gcp/external-k8s/network.k.yaml index 75f808b..45611aa 100644 --- a/gcp/vpc/network.k.yaml +++ b/gcp/external-k8s/network.k.yaml @@ -1,3 +1,4 @@ +--- apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: @@ -11,18 +12,20 @@ spec: resource: spec: + routingMode: REGIONAL autoCreateSubnetworks: false enableUlaInternalIpv6: false - routingMode: REGIONAL + postconditions: - assert: =resource.config_connect_ready() retry: - delay: 10 message: Waiting for compute network to be created - + delay: 10 + return: name: =inputs.metadata.name + --- apiVersion: koreo.dev/v1beta1 kind: FunctionTest @@ -37,15 +40,15 @@ spec: metadata: name: test-network namespace: test-namespace - + testCases: - - expectResource: - apiVersion: compute.cnrm.cloud.google.com/v1beta1 - kind: ComputeNetwork - metadata: - name: test-network - namespace: test-namespace - spec: - autoCreateSubnetworks: false - enableUlaInternalIpv6: false - routingMode: REGIONAL + - expectResource: + apiVersion: compute.cnrm.cloud.google.com/v1beta1 + kind: ComputeNetwork + metadata: + name: test-network + namespace: test-namespace + spec: + routingMode: REGIONAL + autoCreateSubnetworks: false + enableUlaInternalIpv6: false \ No newline at end of file diff --git a/gcp/vpc/subnet.k.yaml b/gcp/external-k8s/subnet.k.yaml similarity index 53% rename from gcp/vpc/subnet.k.yaml rename to gcp/external-k8s/subnet.k.yaml index e4f315e..50de6d0 100644 --- a/gcp/vpc/subnet.k.yaml +++ b/gcp/external-k8s/subnet.k.yaml @@ -1,7 +1,8 @@ +--- apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-subnet + name: gcp-environment-subnet spec: apiConfig: apiVersion: compute.cnrm.cloud.google.com/v1beta1 @@ -11,17 +12,22 @@ spec: resource: spec: - ipCidrRange: =inputs.range networkRef: name: =inputs.networkName region: us-central1 - + ipCidrRange: =inputs.range + secondaryIpRange: + - rangeName: servicesrange + ipCidrRange: 10.11.0.0/16 + - rangeName: clusterrange + ipCidrRange: 10.12.0.0/16 + postconditions: - assert: =resource.config_connect_ready() retry: - delay: 10 message: Waiting for subnetwork to be created - + delay: 10 + return: name: =inputs.metadata.name range: =inputs.range @@ -29,28 +35,33 @@ spec: apiVersion: koreo.dev/v1beta1 kind: FunctionTest metadata: - name: gcp-environment-subnet-test + name: gcp-environment-subnet-test spec: functionRef: kind: ResourceFunction name: gcp-environment-subnet inputs: + range: 10.0.0.0/16 + networkName: test-network-name metadata: name: test-network namespace: test-namespace - networkName: test-network-name - range: 10.0.0.0/16 - + testCases: - - expectResource: - apiVersion: compute.cnrm.cloud.google.com/v1beta1 - kind: ComputeSubnetwork - metadata: - name: test-network - namespace: test-namespace - spec: - ipCidrRange: 10.0.0.0/16 - networkRef: - name: test-network-name - region: us-central1 + - expectResource: + apiVersion: compute.cnrm.cloud.google.com/v1beta1 + kind: ComputeSubnetwork + metadata: + name: test-network + namespace: test-namespace + spec: + region: us-central1 + ipCidrRange: 10.0.0.0/16 + networkRef: + name: test-network-name + secondaryIpRange: + - rangeName: servicesrange + ipCidrRange: 10.11.0.0/16 + - rangeName: clusterrange + ipCidrRange: 10.12.0.0/16 \ No newline at end of file diff --git a/gcp/external-k8s/workflow.k.yaml b/gcp/external-k8s/workflow.k.yaml new file mode 100644 index 0000000..466c609 --- /dev/null +++ b/gcp/external-k8s/workflow.k.yaml @@ -0,0 +1,108 @@ +apiVersion: koreo.dev/v1beta1 +kind: Workflow +metadata: + name: gcp-environment +spec: + crdRef: + apiGroup: acme.example.com + version: v1beta1 + kind: GcpEnvironment + steps: + - label: metadata + ref: + name: gcp-environment-metadata + kind: ValueFunction + inputs: + environmentResourceName: =parent.metadata.name + environmentNamespace: =parent.metadata.namespace + + - label: network + ref: + kind: ResourceFunction + name: gcp-environment-network + inputs: + metadata: =steps.metadata + state: + networkName: =value.name + + - label: subnet + ref: + kind: ResourceFunction + name: gcp-environment-subnet + inputs: + metadata: =steps.metadata + networkName: =steps.network.name + range: 10.10.0.0/16 + state: + subnetName: =value.name + + - label: firewall + ref: + kind: ResourceFunction + name: gcp-environment-firewall + inputs: + metadata: =steps.metadata + networkName: =steps.network.name + subnet: =steps.subnet + + - label: k8sCluster + ref: + kind: ResourceFunction + name: gcp-environment-k8s-cluster + inputs: + metadata: =steps.metadata + networkName: =steps.network.name + subnetworkName: =steps.subnet.name + state: + k8sCluster: =value + + - label: k8sClusterSecret + ref: + kind: ResourceFunction + name: gcp-environment-k8s-cluster-config + inputs: + metadata: =steps.metadata + endpoint: =steps.k8sCluster.endpoint + caCert: =steps.k8sCluster.caCert + + - label: workloadIdentityServiceAccount + ref: + kind: ResourceFunction + name: gcp-environment-iam-service-account + inputs: + metadata: =steps.metadata + - label: workloadIdentityPolicyMember + ref: + kind: ResourceFunction + name: gcp-environment-iam-policy-member + inputs: + metadata: =steps.metadata + saMember: =steps.workloadIdentityServiceAccount.member + projectId: =parent.spec.projectId + + - label: ksaServiceAccount + ref: + kind: ResourceFunction + name: gcp-environment-k8s-service-account + inputs: + metadata: =steps.metadata + saName: =steps.workloadIdentityServiceAccount.name + projectId: =parent.spec.projectId + + - label: workloadIdentityPolicyMemberIdentKSA + ref: + kind: ResourceFunction + name: gcp-environment-iam-policy-member-ident-ksa + inputs: + metadata: =steps.metadata + saName: =steps.workloadIdentityServiceAccount.name + ksaName: =steps.ksaServiceAccount.name + + - label: workloadIdentityPolicyMemberTokenKSA + ref: + kind: ResourceFunction + name: gcp-environment-iam-policy-member-token-ksa + inputs: + metadata: =steps.metadata + saName: =steps.workloadIdentityServiceAccount.name + ksaName: =steps.ksaServiceAccount.name \ No newline at end of file diff --git a/gcp/vpc/README.md b/gcp/vpc/README.md deleted file mode 100644 index c5b326b..0000000 --- a/gcp/vpc/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# VPC - -This example demonstrates building a VPC with a subnet and firewall that is -exposed through a GcpEnvironment abstraction. This uses [Config Connector](https://cloud.google.com/config-connector/docs/overview) -for provisioning GCP resources. diff --git a/gcp/vpc/config.k.yaml b/gcp/vpc/config.k.yaml deleted file mode 100644 index 16f9f16..0000000 --- a/gcp/vpc/config.k.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: koreo.dev/v1beta1 -kind: ValueFunction -metadata: - name: gcp-environment-config -spec: - return: - environmentNamespace: =inputs.parent.metadata.namespace - environmentResourceName: =inputs.parent.metadata.name - projectId: =inputs.parent.spec.projectId ---- -apiVersion: koreo.dev/v1beta1 -kind: FunctionTest -metadata: - name: gcp-environment-config-test -spec: - functionRef: - kind: ValueFunction - name: gcp-environment-config - - inputs: - parent: - apiVersion: acme.example.com/v1beta1 - kind: GcpEnvironment - metadata: - name: test-gcp-environment - namespace: test-namespace - spec: - projectId: test-project - - testCases: - - expectReturn: - environmentNamespace: test-namespace - environmentResourceName: test-gcp-environment - projectId: test-project diff --git a/gcp/vpc/gcp-environment.yaml b/gcp/vpc/gcp-environment.yaml deleted file mode 100644 index d7714e0..0000000 --- a/gcp/vpc/gcp-environment.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: acme.example.com/v1beta1 -kind: GcpEnvironment -metadata: - name: test-gcp-environment -spec: - description: A test GCP environment - projectId: test-project diff --git a/gcp/vpc/workflow.k.yaml b/gcp/vpc/workflow.k.yaml deleted file mode 100644 index 8c881be..0000000 --- a/gcp/vpc/workflow.k.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: koreo.dev/v1beta1 -kind: Workflow -metadata: - name: gcp-environment -spec: - crdRef: - apiGroup: acme.example.com - kind: GcpEnvironment - version: v1beta1 - - steps: - - label: config - ref: - kind: ValueFunction - name: gcp-environment-config - inputs: - parent: =parent - state: - projectId: =value.projectId - - - label: network - ref: - kind: ResourceFunction - name: gcp-environment-network - inputs: - metadata: =steps.config - state: - networkName: =value.name - - - label: subnet - ref: - kind: ResourceFunction - name: gcp-environment-subnet - inputs: - metadata: =steps.config - networkName: =steps.network.name - range: 10.10.0.0/16 - state: - subnetName: =value.name - - - label: firewall - ref: - kind: ResourceFunction - name: gcp-environment-firewall - inputs: - metadata: =steps.config - networkName: =steps.network.name - subnet: =steps.subnet From dd2af571f0e6bd75c2a634468d753182d7c7087d Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Fri, 9 May 2025 13:12:44 -0500 Subject: [PATCH 2/6] Remove namesapce --- gcp/external-k8s/fixtures/k8s-external-test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/gcp/external-k8s/fixtures/k8s-external-test.yaml b/gcp/external-k8s/fixtures/k8s-external-test.yaml index 78cc681..f1c7d99 100644 --- a/gcp/external-k8s/fixtures/k8s-external-test.yaml +++ b/gcp/external-k8s/fixtures/k8s-external-test.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: Pod metadata: name: external-k8s - namespace: skybit-systems-dev spec: serviceAccountName: dev-workload-ksa containers: From 465dcd719acf422f36d924c221ba90a4173ead88 Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Thu, 15 May 2025 09:11:28 -0500 Subject: [PATCH 3/6] Broke out the two examples into their own workflows --- .gitignore | 2 +- gcp/environment/README.md | 26 ++++++ .../crd}/gcp-environment-crd.yaml | 4 +- .../firewall.k.yaml | 0 gcp/environment/fixtures/environment.yaml | 7 ++ .../metadata.k.yaml | 0 .../network.k.yaml | 0 .../subnet.k.yaml | 0 gcp/environment/workflow.k.yaml | 47 +++++++++++ gcp/{external-k8s => federated-gke}/README.md | 16 ++-- gcp/federated-gke/crd/federated-gke-crd.yaml | 80 +++++++++++++++++++ gcp/federated-gke/fixtures/cluster.yaml | 9 +++ .../fixtures/validation-pod.yaml} | 4 +- .../iam_policy_member.k.yaml | 3 +- .../iam_policy_member_ident_ksa.k.yaml | 4 +- .../iam_policy_member_token_ksa.k.yaml | 4 +- .../iam_service_account.k.yaml | 5 +- .../k8s_cluster.k.yaml | 2 +- .../k8s_cluster_secret.k.yaml} | 2 +- .../k8s_service_account.k.yaml | 2 +- gcp/federated-gke/metadata.k.yaml | 25 ++++++ .../workflow.k.yaml | 65 +++++---------- .../hello-workload/hello-workload.k.yaml | 6 +- 23 files changed, 240 insertions(+), 73 deletions(-) create mode 100644 gcp/environment/README.md rename gcp/{external-k8s/fixtures => environment/crd}/gcp-environment-crd.yaml (97%) rename gcp/{external-k8s => environment}/firewall.k.yaml (100%) create mode 100644 gcp/environment/fixtures/environment.yaml rename gcp/{external-k8s => environment}/metadata.k.yaml (100%) rename gcp/{external-k8s => environment}/network.k.yaml (100%) rename gcp/{external-k8s => environment}/subnet.k.yaml (100%) create mode 100644 gcp/environment/workflow.k.yaml rename gcp/{external-k8s => federated-gke}/README.md (90%) create mode 100644 gcp/federated-gke/crd/federated-gke-crd.yaml create mode 100644 gcp/federated-gke/fixtures/cluster.yaml rename gcp/{external-k8s/fixtures/k8s-external-test.yaml => federated-gke/fixtures/validation-pod.yaml} (73%) rename gcp/{external-k8s => federated-gke}/iam_policy_member.k.yaml (93%) rename gcp/{external-k8s => federated-gke}/iam_policy_member_ident_ksa.k.yaml (78%) rename gcp/{external-k8s => federated-gke}/iam_policy_member_token_ksa.k.yaml (78%) rename gcp/{external-k8s => federated-gke}/iam_service_account.k.yaml (83%) rename gcp/{external-k8s => federated-gke}/k8s_cluster.k.yaml (98%) rename gcp/{external-k8s/k8s_cluster_config.k.yaml => federated-gke/k8s_cluster_secret.k.yaml} (89%) rename gcp/{external-k8s => federated-gke}/k8s_service_account.k.yaml (91%) create mode 100644 gcp/federated-gke/metadata.k.yaml rename gcp/{external-k8s => federated-gke}/workflow.k.yaml (53%) diff --git a/.gitignore b/.gitignore index 9869ee2..77a952c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -.last-modified +.last_modified diff --git a/gcp/environment/README.md b/gcp/environment/README.md new file mode 100644 index 0000000..d5cd055 --- /dev/null +++ b/gcp/environment/README.md @@ -0,0 +1,26 @@ +# GCP Environment Workflow + +This workflow provisions and configures a complete GCP-based Kubernetes environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. + +It creates an example vpc and subnet with a firewall. + +## Overview + +- **CRD Reference**: Targets the `GcpEnvironment` CRD defined under `example.koreo.dev/v1beta1`. +- **Inputs**: Expects a `parent` object. +- **Output**: An example vpc and subnet with a firewall + +## Workflow Steps + +| Step | Description | +|------|-------------| +| `metadata` | Generates standard metadata used across all resources. | +| `network` | Creates a VPC network for the environment. | +| `subnet` | Provisions a subnet within the created VPC with CIDR `10.10.0.0/16`. | +| `firewall` | Configures firewall rules for the subnet. | + +## Requirements + +- GCP project with appropriate APIs enabled (IAM, Compute Engine). +- Koreo engine with support for `ResourceFunction` and `ValueFunction`. +- IAM permissions to create the various vpc resources diff --git a/gcp/external-k8s/fixtures/gcp-environment-crd.yaml b/gcp/environment/crd/gcp-environment-crd.yaml similarity index 97% rename from gcp/external-k8s/fixtures/gcp-environment-crd.yaml rename to gcp/environment/crd/gcp-environment-crd.yaml index 4693b24..e02eb45 100644 --- a/gcp/external-k8s/fixtures/gcp-environment-crd.yaml +++ b/gcp/environment/crd/gcp-environment-crd.yaml @@ -1,10 +1,10 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: gcpenvironments.acme.example.com + name: gcpenvironments.example.koreo.dev spec: scope: Namespaced - group: acme.example.com + group: example.koreo.dev names: kind: GcpEnvironment plural: gcpenvironments diff --git a/gcp/external-k8s/firewall.k.yaml b/gcp/environment/firewall.k.yaml similarity index 100% rename from gcp/external-k8s/firewall.k.yaml rename to gcp/environment/firewall.k.yaml diff --git a/gcp/environment/fixtures/environment.yaml b/gcp/environment/fixtures/environment.yaml new file mode 100644 index 0000000..245c5f2 --- /dev/null +++ b/gcp/environment/fixtures/environment.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: example.koreo.dev/v1 +kind: GcpEnvironment +metadata: + name: my-environment +spec: + description: My GCP environment diff --git a/gcp/external-k8s/metadata.k.yaml b/gcp/environment/metadata.k.yaml similarity index 100% rename from gcp/external-k8s/metadata.k.yaml rename to gcp/environment/metadata.k.yaml diff --git a/gcp/external-k8s/network.k.yaml b/gcp/environment/network.k.yaml similarity index 100% rename from gcp/external-k8s/network.k.yaml rename to gcp/environment/network.k.yaml diff --git a/gcp/external-k8s/subnet.k.yaml b/gcp/environment/subnet.k.yaml similarity index 100% rename from gcp/external-k8s/subnet.k.yaml rename to gcp/environment/subnet.k.yaml diff --git a/gcp/environment/workflow.k.yaml b/gcp/environment/workflow.k.yaml new file mode 100644 index 0000000..c467f62 --- /dev/null +++ b/gcp/environment/workflow.k.yaml @@ -0,0 +1,47 @@ +apiVersion: koreo.dev/v1beta1 +kind: Workflow +metadata: + name: gcp-environment +spec: + crdRef: + apiGroup: example.koreo.dev + version: v1beta1 + kind: GcpEnvironment + + steps: + - label: metadata + ref: + name: gcp-environment-metadata + kind: ValueFunction + inputs: + environmentResourceName: =parent.metadata.name + environmentNamespace: =parent.metadata.namespace + + - label: network + ref: + kind: ResourceFunction + name: gcp-environment-network + inputs: + metadata: =steps.metadata + state: + networkName: =value.name + + - label: subnet + ref: + kind: ResourceFunction + name: gcp-environment-subnet + inputs: + metadata: =steps.metadata + networkName: =steps.network.name + range: 10.10.0.0/16 + state: + subnetName: =value.name + + - label: firewall + ref: + kind: ResourceFunction + name: gcp-environment-firewall + inputs: + metadata: =steps.metadata + networkName: =steps.network.name + subnet: =steps.subnet diff --git a/gcp/external-k8s/README.md b/gcp/federated-gke/README.md similarity index 90% rename from gcp/external-k8s/README.md rename to gcp/federated-gke/README.md index 6366f35..9f1c2ff 100644 --- a/gcp/external-k8s/README.md +++ b/gcp/federated-gke/README.md @@ -1,4 +1,4 @@ -# GCP Environment Workflow +# Federated GKE Workflow This workflow provisions and configures a complete GCP-based Kubernetes environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. @@ -6,8 +6,8 @@ It creates the necessary network infrastructure, a GKE cluster, service accounts ## Overview -- **CRD Reference**: Targets the `GcpEnvironment` CRD defined under `acme.example.com/v1beta1`. -- **Inputs**: Expects a `parent` object. +- **CRD Reference**: Targets the `FederatedGKE` CRD defined under `example.koreo.dev/v1beta1`. +- **Inputs**: Expects a `parent` object, that has a networkName, subnetworkName, and projectId input. - **Output**: A fully provisioned Kubernetes cluster with Workload Identity configured. ## Workflow Steps @@ -15,9 +15,6 @@ It creates the necessary network infrastructure, a GKE cluster, service accounts | Step | Description | |------|-------------| | `metadata` | Generates standard metadata used across all resources. | -| `network` | Creates a VPC network for the environment. | -| `subnet` | Provisions a subnet within the created VPC with CIDR `10.10.0.0/16`. | -| `firewall` | Configures firewall rules for the subnet. | | `k8sCluster` | Provisions a GKE cluster using the created VPC and subnet. | | `k8sClusterSecret` | Extracts the Kubernetes API endpoint and CA cert to create a Kubernetes config secret. | | `workloadIdentityServiceAccount` | Creates a GCP IAM Service Account used for Workload Identity. | @@ -38,7 +35,6 @@ It creates the necessary network infrastructure, a GKE cluster, service accounts - Each resource function should be implemented to create or reconcile its respective GCP or K8s object. - The `metadata` step centralizes naming and labeling to ensure consistent tagging and traceability. - ### Notes - Ensure Koreo service account has permissions to edit IAM policy - I made it owner of the account, but this could be restricted I am sure. @@ -69,10 +65,10 @@ gcloud iam service-accounts add-iam-policy-binding \ ``` sh # Edit serviceAccountName to koreo or whatever the workflow generates ("-workload-ksa") -# Edit the secret name references to the GcpEnvironmentName -kubectl apply -f ./fixtures/k8s-external-test.yaml +# Edit the secret name references to the FederatedGKE instance name +kubectl apply -f ./fixtures/validation-pod.yaml -kubectl exec -it -n external-k8s -- /bin/sh +kubectl exec -it -n federated-k8s -- /bin/sh $ pip install pykube-ng google-auth $ python diff --git a/gcp/federated-gke/crd/federated-gke-crd.yaml b/gcp/federated-gke/crd/federated-gke-crd.yaml new file mode 100644 index 0000000..7315329 --- /dev/null +++ b/gcp/federated-gke/crd/federated-gke-crd.yaml @@ -0,0 +1,80 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: federatedgke.example.koreo.dev +spec: + scope: Namespaced + group: acme.example.com + names: + kind: FederatedGKE + plural: federatedgkes + singular: federatedgke + versions: + - name: v1beta1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + type: object + properties: + description: + description: The description of the environment + nullable: true + type: string + subnetworkName: + description: The gcp subnetwork name to use for the cluster + type: string + networkName: + description: The gcp network name to use for the cluster + type: string + projectId: + description: The gcp project id + type: string + status: + x-kubernetes-preserve-unknown-fields: true + properties: + conditions: + description: + Conditions represent the latest available observation + of the resource's current state. + items: + properties: + lastTransitionTime: + description: + Last time the condition transitioned from one status + to another. + type: string + message: + description: + Human-readable message indicating details about + last transition. + type: string + reason: + description: + Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: + Status is the status of the condition. Can be True, + False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + type: object + type: array + type: object + required: + - spec diff --git a/gcp/federated-gke/fixtures/cluster.yaml b/gcp/federated-gke/fixtures/cluster.yaml new file mode 100644 index 0000000..99a4972 --- /dev/null +++ b/gcp/federated-gke/fixtures/cluster.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: example.koreo.dev/v1 +kind: FederatedGke +metadata: + name: my-cluster +spec: + projectId: my-project-id + networkName: my-network + subnetworkName: my-subnetwork diff --git a/gcp/external-k8s/fixtures/k8s-external-test.yaml b/gcp/federated-gke/fixtures/validation-pod.yaml similarity index 73% rename from gcp/external-k8s/fixtures/k8s-external-test.yaml rename to gcp/federated-gke/fixtures/validation-pod.yaml index f1c7d99..3889ea4 100644 --- a/gcp/external-k8s/fixtures/k8s-external-test.yaml +++ b/gcp/federated-gke/fixtures/validation-pod.yaml @@ -1,9 +1,9 @@ apiVersion: v1 kind: Pod metadata: - name: external-k8s + name: federated-k8s spec: - serviceAccountName: dev-workload-ksa + serviceAccountName: dev-workload-ksa # This is the name of the KSA created by the federated-k8s-service-account function containers: - name: app image: python:3.11 diff --git a/gcp/external-k8s/iam_policy_member.k.yaml b/gcp/federated-gke/iam_policy_member.k.yaml similarity index 93% rename from gcp/external-k8s/iam_policy_member.k.yaml rename to gcp/federated-gke/iam_policy_member.k.yaml index cfc7466..ffa58f3 100644 --- a/gcp/external-k8s/iam_policy_member.k.yaml +++ b/gcp/federated-gke/iam_policy_member.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-iam-policy-member + name: federated-iam-policy-member spec: locals: name: =inputs.metadata.name + "-workload-identity" @@ -19,6 +19,7 @@ spec: resourceRef: kind: Project external: ="projects/" + inputs.projectId + postconditions: - assert: =resource.config_connect_ready() retry: diff --git a/gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml b/gcp/federated-gke/iam_policy_member_ident_ksa.k.yaml similarity index 78% rename from gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml rename to gcp/federated-gke/iam_policy_member_ident_ksa.k.yaml index be21819..e08d41e 100644 --- a/gcp/external-k8s/iam_policy_member_ident_ksa.k.yaml +++ b/gcp/federated-gke/iam_policy_member_ident_ksa.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-iam-policy-member-ident-ksa + name: federated-iam-policy-member-ident-ksa spec: locals: name: =inputs.metadata.name + "-ksa-ident-user" @@ -14,7 +14,7 @@ spec: resource: spec: - member: ="serviceAccount:konfig-platform.svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" + member: ="serviceAccount:" + inputs.projectId + ".svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" role: roles/iam.workloadIdentityUser resourceRef: kind: IAMServiceAccount diff --git a/gcp/external-k8s/iam_policy_member_token_ksa.k.yaml b/gcp/federated-gke/iam_policy_member_token_ksa.k.yaml similarity index 78% rename from gcp/external-k8s/iam_policy_member_token_ksa.k.yaml rename to gcp/federated-gke/iam_policy_member_token_ksa.k.yaml index 3f60bc1..86a3081 100644 --- a/gcp/external-k8s/iam_policy_member_token_ksa.k.yaml +++ b/gcp/federated-gke/iam_policy_member_token_ksa.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-iam-policy-member-token-ksa + name: federated-iam-policy-member-token-ksa spec: locals: name: =inputs.metadata.name + "-ksa-token-user" @@ -14,7 +14,7 @@ spec: resource: spec: - member: ="serviceAccount:konfig-platform.svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" + member: ="serviceAccount:" + inputs.projectId + ".svc.id.goog[" + inputs.metadata.namespace + "/" + inputs.ksaName + "]" role: roles/iam.serviceAccountTokenCreator resourceRef: kind: IAMServiceAccount diff --git a/gcp/external-k8s/iam_service_account.k.yaml b/gcp/federated-gke/iam_service_account.k.yaml similarity index 83% rename from gcp/external-k8s/iam_service_account.k.yaml rename to gcp/federated-gke/iam_service_account.k.yaml index 66b6932..1d78072 100644 --- a/gcp/external-k8s/iam_service_account.k.yaml +++ b/gcp/federated-gke/iam_service_account.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-iam-service-account + name: federated-iam-service-account spec: locals: name: =inputs.metadata.name + "-workload-identity" @@ -14,7 +14,8 @@ spec: resource: spec: - displayName: "GKE remote access via WI" + displayName: "GKE remote access via Workload Identity" + postconditions: - assert: =resource.config_connect_ready() retry: diff --git a/gcp/external-k8s/k8s_cluster.k.yaml b/gcp/federated-gke/k8s_cluster.k.yaml similarity index 98% rename from gcp/external-k8s/k8s_cluster.k.yaml rename to gcp/federated-gke/k8s_cluster.k.yaml index 081f92b..c8d6564 100644 --- a/gcp/external-k8s/k8s_cluster.k.yaml +++ b/gcp/federated-gke/k8s_cluster.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-k8s-cluster + name: federated-k8s-cluster spec: apiConfig: apiVersion: container.cnrm.cloud.google.com/v1beta1 diff --git a/gcp/external-k8s/k8s_cluster_config.k.yaml b/gcp/federated-gke/k8s_cluster_secret.k.yaml similarity index 89% rename from gcp/external-k8s/k8s_cluster_config.k.yaml rename to gcp/federated-gke/k8s_cluster_secret.k.yaml index cbebdfd..552da45 100644 --- a/gcp/external-k8s/k8s_cluster_config.k.yaml +++ b/gcp/federated-gke/k8s_cluster_secret.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-k8s-cluster-config + name: federated-k8s-cluster-secret spec: apiConfig: apiVersion: v1 diff --git a/gcp/external-k8s/k8s_service_account.k.yaml b/gcp/federated-gke/k8s_service_account.k.yaml similarity index 91% rename from gcp/external-k8s/k8s_service_account.k.yaml rename to gcp/federated-gke/k8s_service_account.k.yaml index 497cb43..9fe83c0 100644 --- a/gcp/external-k8s/k8s_service_account.k.yaml +++ b/gcp/federated-gke/k8s_service_account.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-k8s-service-account + name: federated-k8s-service-account spec: locals: name: =inputs.metadata.name + "-workload-ksa" diff --git a/gcp/federated-gke/metadata.k.yaml b/gcp/federated-gke/metadata.k.yaml new file mode 100644 index 0000000..825369f --- /dev/null +++ b/gcp/federated-gke/metadata.k.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: koreo.dev/v1beta1 +kind: ValueFunction +metadata: + name: federated-metadata +spec: + return: + name: =inputs.clusterResourceName + namespace: =inputs.clusterNamespace +--- +apiVersion: koreo.dev/v1beta1 +kind: FunctionTest +metadata: + name: federated-metadata-test +spec: + functionRef: + name: federated-metadata + kind: ValueFunction + inputs: + clusterResourceName: example-test + clusterNamespace: example-ns + testCases: + - expectReturn: + name: example-test + namespace: example-ns diff --git a/gcp/external-k8s/workflow.k.yaml b/gcp/federated-gke/workflow.k.yaml similarity index 53% rename from gcp/external-k8s/workflow.k.yaml rename to gcp/federated-gke/workflow.k.yaml index 466c609..0936cbe 100644 --- a/gcp/external-k8s/workflow.k.yaml +++ b/gcp/federated-gke/workflow.k.yaml @@ -1,65 +1,37 @@ apiVersion: koreo.dev/v1beta1 kind: Workflow metadata: - name: gcp-environment + name: federated-gke spec: crdRef: - apiGroup: acme.example.com + apiGroup: example.koreo.dev version: v1beta1 - kind: GcpEnvironment + kind: FederatedGKE + steps: - label: metadata ref: - name: gcp-environment-metadata + name: federated-metadata kind: ValueFunction inputs: - environmentResourceName: =parent.metadata.name - environmentNamespace: =parent.metadata.namespace - - - label: network - ref: - kind: ResourceFunction - name: gcp-environment-network - inputs: - metadata: =steps.metadata - state: - networkName: =value.name - - - label: subnet - ref: - kind: ResourceFunction - name: gcp-environment-subnet - inputs: - metadata: =steps.metadata - networkName: =steps.network.name - range: 10.10.0.0/16 - state: - subnetName: =value.name - - - label: firewall - ref: - kind: ResourceFunction - name: gcp-environment-firewall - inputs: - metadata: =steps.metadata - networkName: =steps.network.name - subnet: =steps.subnet + clusterResourceName: =parent.metadata.name + clusterNamespace: =parent.metadata.namespace - label: k8sCluster ref: kind: ResourceFunction - name: gcp-environment-k8s-cluster + name: federated-k8s-cluster inputs: metadata: =steps.metadata - networkName: =steps.network.name - subnetworkName: =steps.subnet.name + networkName: =parent.networkName + subnetworkName: =parent.subnetworkName state: k8sCluster: =value - label: k8sClusterSecret ref: kind: ResourceFunction - name: gcp-environment-k8s-cluster-config + name: federated-k8s-cluster-secret inputs: metadata: =steps.metadata endpoint: =steps.k8sCluster.endpoint @@ -68,13 +40,14 @@ spec: - label: workloadIdentityServiceAccount ref: kind: ResourceFunction - name: gcp-environment-iam-service-account + name: federated-iam-service-account inputs: metadata: =steps.metadata + - label: workloadIdentityPolicyMember ref: kind: ResourceFunction - name: gcp-environment-iam-policy-member + name: federated-iam-policy-member inputs: metadata: =steps.metadata saMember: =steps.workloadIdentityServiceAccount.member @@ -83,7 +56,7 @@ spec: - label: ksaServiceAccount ref: kind: ResourceFunction - name: gcp-environment-k8s-service-account + name: federated-k8s-service-account inputs: metadata: =steps.metadata saName: =steps.workloadIdentityServiceAccount.name @@ -92,17 +65,19 @@ spec: - label: workloadIdentityPolicyMemberIdentKSA ref: kind: ResourceFunction - name: gcp-environment-iam-policy-member-ident-ksa + name: federated-iam-policy-member-ident-ksa inputs: metadata: =steps.metadata saName: =steps.workloadIdentityServiceAccount.name ksaName: =steps.ksaServiceAccount.name + projectId: =parent.spec.projectId - label: workloadIdentityPolicyMemberTokenKSA ref: kind: ResourceFunction - name: gcp-environment-iam-policy-member-token-ksa + name: federated-iam-policy-member-token-ksa inputs: metadata: =steps.metadata saName: =steps.workloadIdentityServiceAccount.name - ksaName: =steps.ksaServiceAccount.name \ No newline at end of file + ksaName: =steps.ksaServiceAccount.name + projectId: =parent.spec.projectId \ No newline at end of file diff --git a/kubernetes/hello-workload/hello-workload.k.yaml b/kubernetes/hello-workload/hello-workload.k.yaml index 7662110..aa65731 100644 --- a/kubernetes/hello-workload/hello-workload.k.yaml +++ b/kubernetes/hello-workload/hello-workload.k.yaml @@ -15,21 +15,21 @@ spec: name: deployment-factory inputs: name: =parent.metadata.name - namespace: =parent.metadata.namespace + namespace: "default" env: =parent.spec.environment image: =parent.spec.container.image port: =parent.spec.container.port condition: type: Deployment name: Workload Deployment - + - label: create_service ref: kind: ResourceFunction name: service-factory.v2 inputs: name: =parent.metadata.name - namespace: =parent.metadata.namespace + namespace: "default" env: =parent.spec.environment targetPort: =parent.spec.container.port condition: From def529bb031c00b30027bc7fbef321d3bf1de250 Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Thu, 15 May 2025 09:14:54 -0500 Subject: [PATCH 4/6] Fix unnecessary changes --- gcp/environment/firewall.k.yaml | 4 ++-- gcp/environment/network.k.yaml | 1 - gcp/environment/subnet.k.yaml | 4 ++-- kubernetes/hello-workload/hello-workload.k.yaml | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/gcp/environment/firewall.k.yaml b/gcp/environment/firewall.k.yaml index a3a1283..1d066e5 100644 --- a/gcp/environment/firewall.k.yaml +++ b/gcp/environment/firewall.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-firewall + name: gcp-environment-firewall spec: apiConfig: apiVersion: compute.cnrm.cloud.google.com/v1beta1 @@ -36,7 +36,7 @@ spec: apiVersion: koreo.dev/v1beta1 kind: FunctionTest metadata: - name: gcp-environment-firewall-test + name: gcp-environment-firewall-test spec: functionRef: kind: ResourceFunction diff --git a/gcp/environment/network.k.yaml b/gcp/environment/network.k.yaml index 45611aa..941ae9d 100644 --- a/gcp/environment/network.k.yaml +++ b/gcp/environment/network.k.yaml @@ -15,7 +15,6 @@ spec: routingMode: REGIONAL autoCreateSubnetworks: false enableUlaInternalIpv6: false - postconditions: - assert: =resource.config_connect_ready() diff --git a/gcp/environment/subnet.k.yaml b/gcp/environment/subnet.k.yaml index 50de6d0..75bf866 100644 --- a/gcp/environment/subnet.k.yaml +++ b/gcp/environment/subnet.k.yaml @@ -2,7 +2,7 @@ apiVersion: koreo.dev/v1beta1 kind: ResourceFunction metadata: - name: gcp-environment-subnet + name: gcp-environment-subnet spec: apiConfig: apiVersion: compute.cnrm.cloud.google.com/v1beta1 @@ -35,7 +35,7 @@ spec: apiVersion: koreo.dev/v1beta1 kind: FunctionTest metadata: - name: gcp-environment-subnet-test + name: gcp-environment-subnet-test spec: functionRef: kind: ResourceFunction diff --git a/kubernetes/hello-workload/hello-workload.k.yaml b/kubernetes/hello-workload/hello-workload.k.yaml index aa65731..38b4695 100644 --- a/kubernetes/hello-workload/hello-workload.k.yaml +++ b/kubernetes/hello-workload/hello-workload.k.yaml @@ -15,7 +15,7 @@ spec: name: deployment-factory inputs: name: =parent.metadata.name - namespace: "default" + namespace: =parent.metadata.namespace env: =parent.spec.environment image: =parent.spec.container.image port: =parent.spec.container.port @@ -29,7 +29,7 @@ spec: name: service-factory.v2 inputs: name: =parent.metadata.name - namespace: "default" + namespace: =parent.metadata.namespace env: =parent.spec.environment targetPort: =parent.spec.container.port condition: From e9d42ac907a3e04773e8eb6b6fdb310e606aef24 Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Thu, 15 May 2025 10:02:09 -0500 Subject: [PATCH 5/6] Update documentation --- gcp/environment/README.md | 4 ++-- gcp/environment/subnet.k.yaml | 6 ++++-- gcp/federated-gke/README.md | 19 +++++++++++-------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/gcp/environment/README.md b/gcp/environment/README.md index d5cd055..afc6bec 100644 --- a/gcp/environment/README.md +++ b/gcp/environment/README.md @@ -1,8 +1,8 @@ # GCP Environment Workflow -This workflow provisions and configures a complete GCP-based Kubernetes environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. +This workflow provisions and configures a GCP-based environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. -It creates an example vpc and subnet with a firewall. +Included in the environment are a VPC, Subnet, and Firewall. ## Overview diff --git a/gcp/environment/subnet.k.yaml b/gcp/environment/subnet.k.yaml index 75bf866..0827782 100644 --- a/gcp/environment/subnet.k.yaml +++ b/gcp/environment/subnet.k.yaml @@ -18,9 +18,9 @@ spec: ipCidrRange: =inputs.range secondaryIpRange: - rangeName: servicesrange - ipCidrRange: 10.11.0.0/16 + ipCidrRange: =inputs.servicesRange - rangeName: clusterrange - ipCidrRange: 10.12.0.0/16 + ipCidrRange: =inputs.clusterRange postconditions: - assert: =resource.config_connect_ready() @@ -43,6 +43,8 @@ spec: inputs: range: 10.0.0.0/16 + servicesRange: 10.11.0.0/16 + clusterRange: 10.12.0.0/16 networkName: test-network-name metadata: name: test-network diff --git a/gcp/federated-gke/README.md b/gcp/federated-gke/README.md index 9f1c2ff..19508d8 100644 --- a/gcp/federated-gke/README.md +++ b/gcp/federated-gke/README.md @@ -1,8 +1,10 @@ # Federated GKE Workflow -This workflow provisions and configures a complete GCP-based Kubernetes environment using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. +This workflow provisions and configures a complete Kubernetes cluster in another project and creates +a k8s service account in the Koreo Cluster with `roles/container.developer` access to it +using the `koreo.dev` workflow engine and custom `ResourceFunction` and `ValueFunction` modules. -It creates the necessary network infrastructure, a GKE cluster, service accounts, and sets up Workload Identity to allow Kubernetes ServiceAccounts to impersonate GCP Service Accounts securely. +The workflow creates a GKE cluster, necessary IAM and K8s service accounts, sets up Workload Identity to allow Kubernetes ServiceAccounts to impersonate GCP Service Accounts securely. ## Overview @@ -29,19 +31,18 @@ It creates the necessary network infrastructure, a GKE cluster, service accounts - Koreo engine with support for `ResourceFunction` and `ValueFunction`. - IAM permissions to create networks, clusters, service accounts, and IAM policies. -## Notes +## Federated Koreo -- The workflow uses [Google Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) to enable secure communication between GCP and Kubernetes workloads. -- Each resource function should be implemented to create or reconcile its respective GCP or K8s object. -- The `metadata` step centralizes naming and labeling to ensure consistent tagging and traceability. +- To have a subsequent Koreo install use the created k8s service account and manage resources in it, you will need edit the above workflow to use the service account created in the `federated-k8s-service-account` step or run the following commands or edit the koreo service account name. + +## Debug Sanity Check -### Notes - Ensure Koreo service account has permissions to edit IAM policy - I made it owner of the account, but this could be restricted I am sure. - Ensure Identity and Access Management API is enabled - Ensure Kubernetes API is enabled - Ensure Cloud Resource Manager API is enabled -- To get the `koreo` service account you will need edit the above workflow to use the koreo service account or run the following commands. + ``` sh gcloud projects add-iam-policy-binding \ @@ -63,6 +64,8 @@ gcloud iam service-accounts add-iam-policy-binding \ --member "serviceAccount:.svc.id.goog[/koreo]" ``` +### Validate Service Account + ``` sh # Edit serviceAccountName to koreo or whatever the workflow generates ("-workload-ksa") # Edit the secret name references to the FederatedGKE instance name From c7fd8100630b7038c24f56b1377f4e139cce19f9 Mon Sep 17 00:00:00 2001 From: Eric Larssen Date: Thu, 15 May 2025 10:03:18 -0500 Subject: [PATCH 6/6] Fix group mismatch --- gcp/federated-gke/crd/federated-gke-crd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcp/federated-gke/crd/federated-gke-crd.yaml b/gcp/federated-gke/crd/federated-gke-crd.yaml index 7315329..6db0c8c 100644 --- a/gcp/federated-gke/crd/federated-gke-crd.yaml +++ b/gcp/federated-gke/crd/federated-gke-crd.yaml @@ -4,7 +4,7 @@ metadata: name: federatedgke.example.koreo.dev spec: scope: Namespaced - group: acme.example.com + group: example.koreo.dev names: kind: FederatedGKE plural: federatedgkes