From decdc174d60ec7022a8db7ed0c7ea267a64758bf Mon Sep 17 00:00:00 2001 From: Rauf Guliyev Date: Mon, 20 Jul 2026 18:34:22 +0000 Subject: [PATCH 1/5] Update GCE Terraform integration fixtures Refresh expected manager update policies after rendering max_unavailable_fixed. This restores the GCE integration tests for the fork's existing Terraform change. --- .../update_cluster/ha_gce/kubernetes.tf | 30 +++++++++++-------- .../many-addons-gce/kubernetes.tf | 10 ++++--- .../update_cluster/minimal_gce/kubernetes.tf | 15 ++++++---- .../minimal_gce_dns-none/kubernetes.tf | 15 ++++++---- .../minimal_gce_ilb/kubernetes.tf | 15 ++++++---- .../kubernetes.tf | 15 ++++++---- .../minimal_gce_longclustername/kubernetes.tf | 15 ++++++---- .../minimal_gce_plb/kubernetes.tf | 15 ++++++---- .../minimal_gce_private/kubernetes.tf | 15 ++++++---- 9 files changed, 87 insertions(+), 58 deletions(-) diff --git a/tests/integration/update_cluster/ha_gce/kubernetes.tf b/tests/integration/update_cluster/ha_gce/kubernetes.tf index 3703551febde9..a60851a05602d 100644 --- a/tests/integration/update_cluster/ha_gce/kubernetes.tf +++ b/tests/integration/update_cluster/ha_gce/kubernetes.tf @@ -501,8 +501,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-ha-gce-exa name = "a-master-us-test1-a-ha-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-ha-gce-example-com.self_link @@ -519,8 +520,9 @@ resource "google_compute_instance_group_manager" "a-nodes-ha-gce-example-com" { name = "a-nodes-ha-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-ha-gce-example-com.self_link @@ -537,8 +539,9 @@ resource "google_compute_instance_group_manager" "b-master-us-test1-b-ha-gce-exa name = "b-master-us-test1-b-ha-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-b-ha-gce-example-com.self_link @@ -555,8 +558,9 @@ resource "google_compute_instance_group_manager" "b-nodes-ha-gce-example-com" { name = "b-nodes-ha-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-ha-gce-example-com.self_link @@ -573,8 +577,9 @@ resource "google_compute_instance_group_manager" "c-master-us-test1-c-ha-gce-exa name = "c-master-us-test1-c-ha-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-c-ha-gce-example-com.self_link @@ -591,8 +596,9 @@ resource "google_compute_instance_group_manager" "c-nodes-ha-gce-example-com" { name = "c-nodes-ha-gce-example-com" target_size = 0 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-ha-gce-example-com.self_link diff --git a/tests/integration/update_cluster/many-addons-gce/kubernetes.tf b/tests/integration/update_cluster/many-addons-gce/kubernetes.tf index 88489af93399a..c2dc0a7905bec 100644 --- a/tests/integration/update_cluster/many-addons-gce/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-gce/kubernetes.tf @@ -429,8 +429,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-ex name = "a-master-us-test1-a-minimal-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-example-com.self_link @@ -447,8 +448,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-example-com" { name = "a-nodes-minimal-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce/kubernetes.tf b/tests/integration/update_cluster/minimal_gce/kubernetes.tf index 5e7f4132ff5c8..683e88075db7f 100644 --- a/tests/integration/update_cluster/minimal_gce/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce/kubernetes.tf @@ -397,8 +397,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-example-com.self_link @@ -415,8 +416,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-example-co name = "a-nodes-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-example-com.self_link @@ -433,8 +435,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-example-co name = "b-nodes-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf index c9f3c7ba2c1ab..cb15a40223953 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf @@ -469,8 +469,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-example-com.self_link @@ -487,8 +488,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-example-co name = "a-nodes-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-example-com.self_link @@ -505,8 +507,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-example-co name = "b-nodes-minimal-gce-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf index b5964667b2012..6ef9c2927a310 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf @@ -438,8 +438,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-ilb-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-ilb-example-com.self_link @@ -456,8 +457,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-ilb-exampl name = "a-nodes-minimal-gce-ilb-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-ilb-example-com.self_link @@ -474,8 +476,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-ilb-exampl name = "b-nodes-minimal-gce-ilb-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-ilb-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf index 3df6d301d550e..637c4fc9ccb06 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf @@ -438,8 +438,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-with-a-very-very-very-ve-j0fh8f" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link @@ -456,8 +457,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-with-a-ver name = "a-nodes-minimal-gce-with-a-very-very-very-very-very-long-qk78uj" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link @@ -474,8 +476,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-with-a-ver name = "b-nodes-minimal-gce-with-a-very-very-very-very-very-long-h6o2lg" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_longclustername/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_longclustername/kubernetes.tf index a6e573eba4761..bac977d976d9d 100644 --- a/tests/integration/update_cluster/minimal_gce_longclustername/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_longclustername/kubernetes.tf @@ -405,8 +405,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-with-a-very-very-very-ve-j0fh8f" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link @@ -423,8 +424,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-with-a-ver name = "a-nodes-minimal-gce-with-a-very-very-very-very-very-long-qk78uj" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link @@ -441,8 +443,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-with-a-ver name = "b-nodes-minimal-gce-with-a-very-very-very-very-very-long-h6o2lg" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf index a97a5c952e9a0..f8e12dc9be91f 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf @@ -462,8 +462,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc target_pools = [google_compute_target_pool.api-minimal-gce-plb-example-com.self_link] target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-plb-example-com.self_link @@ -480,8 +481,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-plb-exampl name = "a-nodes-minimal-gce-plb-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-plb-example-com.self_link @@ -498,8 +500,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-plb-exampl name = "b-nodes-minimal-gce-plb-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-plb-example-com.self_link diff --git a/tests/integration/update_cluster/minimal_gce_private/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_private/kubernetes.tf index f1587836715a2..d51a0d8273662 100644 --- a/tests/integration/update_cluster/minimal_gce_private/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_private/kubernetes.tf @@ -405,8 +405,9 @@ resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gc name = "a-master-us-test1-a-minimal-gce-private-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.master-us-test1-a-minimal-gce-private-example-com.self_link @@ -423,8 +424,9 @@ resource "google_compute_instance_group_manager" "a-nodes-minimal-gce-private-ex name = "a-nodes-minimal-gce-private-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-private-example-com.self_link @@ -441,8 +443,9 @@ resource "google_compute_instance_group_manager" "b-nodes-minimal-gce-private-ex name = "b-nodes-minimal-gce-private-example-com" target_size = 1 update_policy { - minimal_action = "REPLACE" - type = "OPPORTUNISTIC" + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" } version { instance_template = google_compute_instance_template.nodes-minimal-gce-private-example-com.self_link From 3d06cd0a24b33a63bd1a3e80b2439c164d0bdc0a Mon Sep 17 00:00:00 2001 From: Rauf Guliyev Date: Mon, 20 Jul 2026 18:38:29 +0000 Subject: [PATCH 2/5] Render GCE project IAM as additive members The API target adds a principal without owning the full role binding, but the Terraform target emitted an authoritative binding that could remove unrelated principals. Use google_project_iam_member so both targets have the same additive semantics. --- .../many-addons-gce/kubernetes.tf | 8 ++-- .../minimal_gce_dns-none/kubernetes.tf | 8 ++-- .../fi/cloudup/gcetasks/projectiambinding.go | 18 +++++---- .../gcetasks/projectiambinding_test.go | 40 +++++++++++++++++++ 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/tests/integration/update_cluster/many-addons-gce/kubernetes.tf b/tests/integration/update_cluster/many-addons-gce/kubernetes.tf index c2dc0a7905bec..cc92c0ad67c1d 100644 --- a/tests/integration/update_cluster/many-addons-gce/kubernetes.tf +++ b/tests/integration/update_cluster/many-addons-gce/kubernetes.tf @@ -579,14 +579,14 @@ resource "google_compute_subnetwork" "us-test1-minimal-example-com" { stack_type = "IPV4_ONLY" } -resource "google_project_iam_binding" "serviceaccount-control-plane" { - members = [format("serviceAccount:%s", google_service_account.control-plane.email)] +resource "google_project_iam_member" "serviceaccount-control-plane" { + member = format("serviceAccount:%s", google_service_account.control-plane.email) project = "testproject" role = "roles/container.serviceAgent" } -resource "google_project_iam_binding" "serviceaccount-nodes" { - members = [format("serviceAccount:%s", google_service_account.node.email)] +resource "google_project_iam_member" "serviceaccount-nodes" { + member = format("serviceAccount:%s", google_service_account.node.email) project = "testproject" role = "roles/compute.viewer" } diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf index cb15a40223953..1871d0fef16e8 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf @@ -669,14 +669,14 @@ resource "google_compute_subnetwork" "us-test1-minimal-gce-example-com" { stack_type = "IPV4_ONLY" } -resource "google_project_iam_binding" "serviceaccount-control-plane" { - members = [format("serviceAccount:%s", google_service_account.control-plane.email)] +resource "google_project_iam_member" "serviceaccount-control-plane" { + member = format("serviceAccount:%s", google_service_account.control-plane.email) project = "testproject" role = "roles/container.serviceAgent" } -resource "google_project_iam_binding" "serviceaccount-nodes" { - members = [format("serviceAccount:%s", google_service_account.node.email)] +resource "google_project_iam_member" "serviceaccount-nodes" { + member = format("serviceAccount:%s", google_service_account.node.email) project = "testproject" role = "roles/compute.viewer" } diff --git a/upup/pkg/fi/cloudup/gcetasks/projectiambinding.go b/upup/pkg/fi/cloudup/gcetasks/projectiambinding.go index 5b0077a2f96e3..eb1c52613e233 100644 --- a/upup/pkg/fi/cloudup/gcetasks/projectiambinding.go +++ b/upup/pkg/fi/cloudup/gcetasks/projectiambinding.go @@ -134,21 +134,23 @@ func (_ *ProjectIAMBinding) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Projec return nil } -// terraformProjectIAMBinding is the model for a terraform google_project_iam_binding rule -type terraformProjectIAMBinding struct { - Project string `cty:"project"` - Role string `cty:"role"` - Members []*terraformWriter.Literal `cty:"members"` +// terraformProjectIAMMember is the model for a terraform google_project_iam_member rule. +type terraformProjectIAMMember struct { + Project string `cty:"project"` + Role string `cty:"role"` + Member *terraformWriter.Literal `cty:"member"` } func (_ *ProjectIAMBinding) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ProjectIAMBinding) error { - tf := &terraformProjectIAMBinding{ + // Render an additive member to match the API target and avoid owning all + // principals granted the role on the project. + tf := &terraformProjectIAMMember{ Project: fi.ValueOf(e.Project), Role: fi.ValueOf(e.Role), - Members: []*terraformWriter.Literal{e.MemberServiceAccount.TerraformLink_Member()}, + Member: e.MemberServiceAccount.TerraformLink_Member(), } - return t.RenderResource("google_project_iam_binding", *e.Name, tf) + return t.RenderResource("google_project_iam_member", *e.Name, tf) } func patchCRMPolicy(policy *cloudresourcemanager.Policy, wantMember string, wantRole string) bool { diff --git a/upup/pkg/fi/cloudup/gcetasks/projectiambinding_test.go b/upup/pkg/fi/cloudup/gcetasks/projectiambinding_test.go index 26ed07b95e3b0..a7b07485aa8db 100644 --- a/upup/pkg/fi/cloudup/gcetasks/projectiambinding_test.go +++ b/upup/pkg/fi/cloudup/gcetasks/projectiambinding_test.go @@ -18,10 +18,14 @@ package gcetasks import ( "context" + "os" + "path/filepath" + "strings" "testing" gcemock "k8s.io/kops/cloudmock/gce" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/terraform" ) func TestProjectIAMBinding(t *testing.T) { @@ -69,3 +73,39 @@ func TestProjectIAMBinding(t *testing.T) { checkNoChanges(t, ctx, cloud, allTasks) } } + +func TestProjectIAMBindingRenderTerraform(t *testing.T) { + outDir := t.TempDir() + cloud := gcemock.InstallMockGCECloud("us-test1", "testproject") + target := terraform.NewTerraformTarget(cloud, "testproject", outDir, nil) + task := &ProjectIAMBinding{ + Name: fi.PtrTo("serviceaccount-nodes"), + Project: fi.PtrTo("testproject"), + MemberServiceAccount: &ServiceAccount{ + Name: fi.PtrTo("node"), + }, + Role: fi.PtrTo("roles/compute.viewer"), + } + + if err := task.RenderTerraform(target, nil, task, task); err != nil { + t.Fatalf("RenderTerraform() error: %v", err) + } + if err := target.Finish(nil); err != nil { + t.Fatalf("Finish() error: %v", err) + } + + b, err := os.ReadFile(filepath.Join(outDir, "kubernetes.tf")) + if err != nil { + t.Fatalf("reading rendered Terraform: %v", err) + } + rendered := string(b) + if !strings.Contains(rendered, `resource "google_project_iam_member" "serviceaccount-nodes"`) { + t.Errorf("rendered Terraform does not contain google_project_iam_member:\n%s", rendered) + } + if !strings.Contains(rendered, `member = format("serviceAccount:%s", google_service_account.node.email)`) { + t.Errorf("rendered Terraform does not contain the service account member expression:\n%s", rendered) + } + if strings.Contains(rendered, "google_project_iam_binding") || strings.Contains(rendered, "members =") { + t.Errorf("rendered Terraform contains authoritative IAM binding syntax:\n%s", rendered) + } +} From 42ea71cdecc8144cb1437835c85c8aacfe2363a6 Mon Sep 17 00:00:00 2001 From: Rauf Guliyev Date: Mon, 20 Jul 2026 18:41:09 +0000 Subject: [PATCH 3/5] Format GCE update policy rendering --- upup/pkg/fi/cloudup/gcetasks/instancegroupmanager.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/upup/pkg/fi/cloudup/gcetasks/instancegroupmanager.go b/upup/pkg/fi/cloudup/gcetasks/instancegroupmanager.go index 4fcef2a49a0e3..58e65669ad374 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instancegroupmanager.go +++ b/upup/pkg/fi/cloudup/gcetasks/instancegroupmanager.go @@ -199,9 +199,9 @@ type terraformInstanceGroupManager struct { } type terraformUpdatePolicy struct { - MinimalAction string `cty:"minimal_action"` - MaxUnavailableFixed int `cty:"max_unavailable_fixed"` - Type string `cty:"type"` + MinimalAction string `cty:"minimal_action"` + MaxUnavailableFixed int `cty:"max_unavailable_fixed"` + Type string `cty:"type"` } type terraformVersion struct { @@ -221,9 +221,9 @@ func (_ *InstanceGroupManager) RenderTerraform(t *terraform.TerraformTarget, a, } if policy := e.UpdatePolicy; policy != nil { tf.UpdatePolicy = &terraformUpdatePolicy{ - MinimalAction: policy.MinimalAction, + MinimalAction: policy.MinimalAction, MaxUnavailableFixed: 1, - Type: policy.Type, + Type: policy.Type, } } tf.Version = &terraformVersion{ From 1375ad16e1cd5aed65ccc63c73cc73412a69a166 Mon Sep 17 00:00:00 2001 From: Rauf Guliyev Date: Wed, 22 Jul 2026 19:07:12 +0000 Subject: [PATCH 4/5] Add generic offline GCE equivalence fixture Exercise the kops-module topology without cloud credentials using generic fixture identities and deterministic PKI. --- cmd/kops/integration_test.go | 179 ++- pkg/nodemodel/wellknownassets/kopsassets.go | 9 + pkg/testutils/integrationtestharness.go | 9 +- ...s_s3_object_cluster-completed.spec_content | 248 ++++ ...s3_object_etcd-cluster-spec-events_content | 4 + ...s_s3_object_etcd-cluster-spec-main_content | 4 + ....example.internal-addons-bootstrap_content | 115 ++ ...ons-coredns.addons.k8s.io-k8s-1.12_content | 384 ++++++ ...-controller.addons.k8s.io-k8s-1.23_content | 549 ++++++++ ...-csi-driver.addons.k8s.io-k8s-1.23_content | 868 ++++++++++++ ...-controller.addons.k8s.io-k8s-1.16_content | 219 +++ ...let-api.rbac.addons.k8s.io-k8s-1.9_content | 16 + ...l-addons-limit-range.addons.k8s.io_content | 14 + ...rics-server.addons.k8s.io-k8s-1.11_content | 266 ++++ ...dons-networking.cilium.io-k8s-1.16_content | 1194 +++++++++++++++++ ...s-storage-gce.addons.k8s.io-v1.7.0_content | 13 + .../aws_s3_object_kops-version.txt_content | 1 + ...er-events-control-plane-us-west1-a_content | 118 ++ ...er-events-control-plane-us-west1-b_content | 118 ++ ...er-events-control-plane-us-west1-c_content | 118 ++ ...ager-main-control-plane-us-west1-a_content | 118 ++ ...ager-main-control-plane-us-west1-b_content | 118 ++ ...ager-main-control-plane-us-west1-c_content | 118 ++ ...-static-kube-apiserver-healthcheck_content | 32 + ...eupconfig-control-plane-us-west1-a_content | 310 +++++ ...eupconfig-control-plane-us-west1-b_content | 310 +++++ ...eupconfig-control-plane-us-west1-c_content | 310 +++++ ...ject_nodeupconfig-nodes-us-west1-a_content | 57 + ...ject_nodeupconfig-nodes-us-west1-b_content | 57 + ...ject_nodeupconfig-nodes-us-west1-c_content | 57 + ...alence-example-internal_metadata_user-data | 133 ++ ...alence-example-internal_metadata_user-data | 133 ++ ...alence-example-internal_metadata_user-data | 133 ++ ...alence-example-internal_metadata_user-data | 155 +++ ...alence-example-internal_metadata_user-data | 155 +++ ...alence-example-internal_metadata_user-data | 155 +++ .../golden-pki.json | 33 + .../in-v1alpha2.yaml | 203 +++ .../gce-kops-module-equivalence/kubernetes.tf | 1136 ++++++++++++++++ 39 files changed, 8151 insertions(+), 18 deletions(-) create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_cluster-completed.spec_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-events_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-main_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-bootstrap_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-coredns.addons.k8s.io-k8s-1.12_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kops-controller.addons.k8s.io-k8s-1.16_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-limit-range.addons.k8s.io_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-metrics-server.addons.k8s.io-k8s-1.11_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-networking.cilium.io-k8s-1.16_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-storage-gce.addons.k8s.io-v1.7.0_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_kops-version.txt_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-a_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-b_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-c_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-a_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-b_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-c_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-a_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-b_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-c_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-a_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-b_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-c_content create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/golden-pki.json create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/in-v1alpha2.yaml create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/kubernetes.tf diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index da2e9159c31a0..73abe231ba8b6 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -58,6 +58,8 @@ type integrationTest struct { clusterName string srcDir string version string + kopsVersion string + keysetsFixture string private bool zones int expectPolicies bool @@ -75,6 +77,11 @@ type integrationTest struct { nthRebalance bool // enable GCE startup script startupScript bool + // GCE instance group names override the legacy names derived from zones. + gceControlPlaneInstanceGroups []string + gceNodeInstanceGroups []string + gceProject string + gceRegion string // verify "kops get assets" functionality testGetAssets bool } @@ -106,11 +113,33 @@ func (i *integrationTest) withVersion(version string) *integrationTest { return i } +func (i *integrationTest) withKopsVersion(version string) *integrationTest { + i.kopsVersion = version + return i +} + +func (i *integrationTest) withKeysetsFixture(filename string) *integrationTest { + i.keysetsFixture = filename + return i +} + func (i *integrationTest) withZones(zones int) *integrationTest { i.zones = zones return i } +func (i *integrationTest) withGCEInstanceGroups(controlPlane, nodes []string) *integrationTest { + i.gceControlPlaneInstanceGroups = controlPlane + i.gceNodeInstanceGroups = nodes + return i +} + +func (i *integrationTest) withGCEProject(project, region string) *integrationTest { + i.gceProject = project + i.gceRegion = region + return i +} + func (i *integrationTest) withoutSSHKey() *integrationTest { i.sshKey = false return i @@ -441,6 +470,25 @@ func TestMinimalGCEDNSNone(t *testing.T) { runTestTerraformGCE(t) } +func TestGCEKopsModuleEquivalence(t *testing.T) { + newIntegrationTest("k8s1.equivalence.example.internal", "gce-kops-module-equivalence"). + withoutSSHKey(). + withKopsVersion("1.35.1"). + withKeysetsFixture("golden-pki.json"). + withGCEProject("example-project", "us-west1"). + withGCEInstanceGroups( + []string{"control-plane-us-west1-a", "control-plane-us-west1-b", "control-plane-us-west1-c"}, + []string{"nodes-us-west1-a", "nodes-us-west1-b", "nodes-us-west1-c"}, + ). + withAddons( + ciliumAddon, + gcpCCMAddon, + gcpPDCSIAddon, + metricsServerAddon, + ). + runTestTerraformGCE(t) +} + // TestMinimalScaleway runs tests on a minimal Scaleway cluster with gossip DNS func TestMinimalScaleway(t *testing.T) { t.Setenv("SCW_PROFILE", "REDACTED") @@ -1377,6 +1425,11 @@ func (i *integrationTest) setupCluster(t *testing.T, ctx context.Context, inputY t.Fatalf("error getting keystore: %v", err) } + if i.keysetsFixture != "" { + i.storeKeysetsFixture(t, ctx, keyStore) + return factory + } + storeKeyset(t, ctx, keyStore, fi.CertificateIDCA, &testingKeyset{ primaryKey: "-----BEGIN RSA PRIVATE KEY-----\nMIIBOgIBAAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0\nyIg1jt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAQJATmTyoZ3D+6dtBErocEVT\nKyHBhS3P6YrRLIBU0kmdiQHN8BuzvENqm5PASTq1m6yAAJs7qu9S0kO8u4G+SILv\n7QIhAPNCeJoFHmNUwQ1kxuta1RqICGcNoA4Yx5LiHXd9dPM7AiEA3D7gq8WB8csD\nghBNu/zLy3RdFCkfJqWkX5FhdX29alcCIHw4A1HTL1NV4kcuoQ1qEsw7jt7g7EyG\nhtMQuC9eVywlAiA1Z12s6Og4S+Se3fsrUQHNZHrJT6tJALMZpTO/fGy4YwIhANlJ\nR6hkVKtJp9zhipu6WpvpiAtoIlsNnPMPyuDRwV/u\n-----END RSA PRIVATE KEY-----", primaryCertificate: "-----BEGIN CERTIFICATE-----\nMIIBbjCCARigAwIBAgIMFpANqBD8NSD82AUSMA0GCSqGSIb3DQEBCwUAMBgxFjAU\nBgNVBAMTDWt1YmVybmV0ZXMtY2EwHhcNMjEwNzA3MDcwODAwWhcNMzEwNzA3MDcw\nODAwWjAYMRYwFAYDVQQDEw1rdWJlcm5ldGVzLWNhMFwwDQYJKoZIhvcNAQEBBQAD\nSwAwSAJBANFI3zr0Tk8krsW8vwjfMpzJOlWQ8616vG3YPa2qAgI7V4oKwfV0yIg1\njt+H6f4P/wkPAPTPTfRp9Iy8oHEEFw0CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEG\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNG3zVjTcLlJwDsJ4/K9DV7KohUA\nMA0GCSqGSIb3DQEBCwUAA0EAB8d03fY2w7WKpfO29qI295pu2C4ca9AiVGOpgSc8\ntmQsq6rcxt3T+rb589PVtz0mw/cKTxOk6gH2CCC+yHfy2w==\n-----END CERTIFICATE-----", @@ -1449,9 +1502,78 @@ func (i *integrationTest) setupCluster(t *testing.T, ctx context.Context, inputY return factory } +type keysetsFixture struct { + CACertificatesPEM map[string]string `json:"ca_certificates_pem"` + PrivateKeysPEM map[string]string `json:"private_keys_pem"` + KeypairIDs map[string]string `json:"keypair_ids"` + ServiceAccountPublicKey string `json:"service_account_public_keys"` +} + +func (i *integrationTest) storeKeysetsFixture(t *testing.T, ctx context.Context, keyStore fi.Keystore) { + b, err := os.ReadFile(path.Join(i.srcDir, i.keysetsFixture)) + if err != nil { + t.Fatalf("reading keysets fixture: %v", err) + } + var fixture keysetsFixture + if err := json.Unmarshal(b, &fixture); err != nil { + t.Fatalf("decoding keysets fixture: %v", err) + } + keys := func(m map[string]string) []string { + out := make([]string, 0, len(m)) + for key := range m { + out = append(out, key) + } + sort.Strings(out) + return out + } + names := keys(fixture.CACertificatesPEM) + if len(names) == 0 { + t.Fatalf("keysets fixture contains no keysets") + } + if !reflect.DeepEqual(names, keys(fixture.PrivateKeysPEM)) || !reflect.DeepEqual(names, keys(fixture.KeypairIDs)) { + t.Fatalf("keysets fixture map keys do not match") + } + if fixture.ServiceAccountPublicKey == "" { + t.Fatalf("keysets fixture service_account_public_keys is empty") + } + serviceAccountKeyPEM, ok := fixture.PrivateKeysPEM["service-account"] + if !ok { + t.Fatalf("keysets fixture does not contain service-account") + } + serviceAccountKey, err := pki.ParsePEMPrivateKey([]byte(serviceAccountKeyPEM)) + if err != nil { + t.Fatalf("loading service-account private key: %v", err) + } + if serviceAccountKey == nil { + t.Fatalf("loading service-account private key: no key found") + } + publicKeyDER, err := x509.MarshalPKIXPublicKey(serviceAccountKey.Key.Public()) + if err != nil { + t.Fatalf("marshalling service-account public key: %v", err) + } + var publicKeyPEM bytes.Buffer + if err := pem.Encode(&publicKeyPEM, &pem.Block{Type: "RSA PUBLIC KEY", Bytes: publicKeyDER}); err != nil { + t.Fatalf("encoding service-account public key: %v", err) + } + if publicKeyPEM.String() != fixture.ServiceAccountPublicKey { + t.Fatalf("keysets fixture service_account_public_keys does not match service-account private key") + } + for _, name := range names { + if fixture.CACertificatesPEM[name] == "" || fixture.PrivateKeysPEM[name] == "" || fixture.KeypairIDs[name] == "" { + t.Fatalf("keysets fixture keyset %q has an empty required value", name) + } + storeKeyset(t, ctx, keyStore, name, &testingKeyset{ + primaryKey: fixture.PrivateKeysPEM[name], + primaryCertificate: fixture.CACertificatesPEM[name], + primaryID: fixture.KeypairIDs[name], + }) + } +} + type testingKeyset struct { primaryKey string primaryCertificate string + primaryID string secondaryKey string secondaryCertificate string } @@ -1472,6 +1594,11 @@ func storeKeyset(t *testing.T, ctx context.Context, keyStore fi.Keystore, name s if err != nil { t.Fatalf("error creating keyset: %v", err) } + if testingKeyset.primaryID != "" { + delete(keyset.Items, keyset.Primary.Id) + keyset.Primary.Id = testingKeyset.primaryID + keyset.Items[keyset.Primary.Id] = keyset.Primary + } if testingKeyset.secondaryKey != "" { privateKey, err = pki.ParsePEMPrivateKey([]byte(testingKeyset.secondaryKey)) @@ -1640,25 +1767,25 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) { h := testutils.NewIntegrationTestHarness(t) defer h.Close() - h.MockKopsVersion("1.34.0-beta.1") - h.SetupMockGCE() - - expectedFilenames := i.expectTerraformFilenames - - prefix := "google_compute_instance_template_nodes-" + gce.SafeClusterName(i.clusterName) + "_metadata_" - if !i.startupScript { - expectedFilenames = append(expectedFilenames, prefix+"user-data") + kopsVersion := i.kopsVersion + if kopsVersion == "" { + kopsVersion = "1.34.0-beta.1" + } + h.MockKopsVersion(kopsVersion) + if i.gceProject == "" { + h.SetupMockGCE() } else { - expectedFilenames = append(expectedFilenames, prefix+"startup-script") + h.SetupMockGCEForProject(i.gceProject, i.gceRegion) } + expectedFilenames := i.expectTerraformFilenames + expectedFilenames = append(expectedFilenames, "aws_s3_object_cluster-completed.spec_content", "aws_s3_object_etcd-cluster-spec-events_content", "aws_s3_object_etcd-cluster-spec-main_content", "aws_s3_object_kops-version.txt_content", "aws_s3_object_manifests-static-kube-apiserver-healthcheck_content", - "aws_s3_object_nodeupconfig-nodes_content", "aws_s3_object_"+i.clusterName+"-addons-bootstrap_content", "aws_s3_object_"+i.clusterName+"-addons-coredns.addons.k8s.io-k8s-1.12_content", "aws_s3_object_"+i.clusterName+"-addons-kops-controller.addons.k8s.io-k8s-1.16_content", @@ -1667,14 +1794,34 @@ func (i *integrationTest) runTestTerraformGCE(t *testing.T) { "aws_s3_object_"+i.clusterName+"-addons-storage-gce.addons.k8s.io-v1.7.0_content", ) - for j := 0; j < i.zones; j++ { - zone := "us-test1-" + string([]byte{byte('a') + byte(j)}) + nodeInstanceGroups := i.gceNodeInstanceGroups + if nodeInstanceGroups == nil { + nodeInstanceGroups = []string{"nodes"} + } + for _, instanceGroup := range nodeInstanceGroups { + expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-"+instanceGroup+"_content") - expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-master-"+zone+"_content") - expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-master-"+zone+"_content") - expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-master-"+zone+"_content") + prefix := "google_compute_instance_template_" + instanceGroup + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_" + if !i.startupScript { + expectedFilenames = append(expectedFilenames, prefix+"user-data") + } else { + expectedFilenames = append(expectedFilenames, prefix+"startup-script") + } + } + + controlPlaneInstanceGroups := i.gceControlPlaneInstanceGroups + if controlPlaneInstanceGroups == nil { + for j := 0; j < i.zones; j++ { + zone := "us-test1-" + string([]byte{byte('a') + byte(j)}) + controlPlaneInstanceGroups = append(controlPlaneInstanceGroups, "master-"+zone) + } + } + for _, instanceGroup := range controlPlaneInstanceGroups { + expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-events-"+instanceGroup+"_content") + expectedFilenames = append(expectedFilenames, "aws_s3_object_manifests-etcdmanager-main-"+instanceGroup+"_content") + expectedFilenames = append(expectedFilenames, "aws_s3_object_nodeupconfig-"+instanceGroup+"_content") - prefix := "google_compute_instance_template_master-" + zone + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_" + prefix := "google_compute_instance_template_" + instanceGroup + "-" + gce.SafeClusterName(i.clusterName) + "_metadata_" if !i.startupScript { expectedFilenames = append(expectedFilenames, prefix+"user-data") } else { diff --git a/pkg/nodemodel/wellknownassets/kopsassets.go b/pkg/nodemodel/wellknownassets/kopsassets.go index 5b979df2b5770..0d803fd371cc0 100644 --- a/pkg/nodemodel/wellknownassets/kopsassets.go +++ b/pkg/nodemodel/wellknownassets/kopsassets.go @@ -43,6 +43,15 @@ var protokubeAsset map[architectures.Architecture]*assets.MirroredAsset // channelsAsset caches the channels binary download url/hash var channelsAsset map[architectures.Architecture]*assets.MirroredAsset +// ResetCachesForTesting clears cached kOps asset locations after tests change global configuration. +// It must not be called concurrently with asset lookup. +func ResetCachesForTesting() { + kopsBaseURL = nil + nodeUpAsset = nil + protokubeAsset = nil + channelsAsset = nil +} + // BaseURL returns the base url for the distribution of kops - in particular for nodeup & docker images func BaseURL() (*url.URL, error) { // returning cached value diff --git a/pkg/testutils/integrationtestharness.go b/pkg/testutils/integrationtestharness.go index 3d8c80cf58bff..d84512721666a 100644 --- a/pkg/testutils/integrationtestharness.go +++ b/pkg/testutils/integrationtestharness.go @@ -56,6 +56,7 @@ import ( "k8s.io/kops/cloudmock/openstack/mocknetworking" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/nodemodel/wellknownassets" "k8s.io/kops/pkg/pki" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" @@ -122,6 +123,7 @@ func (h *IntegrationTestHarness) Close() { if h.originalKopsVersion != "" { kopsroot.Version = h.originalKopsVersion + wellknownassets.ResetCachesForTesting() } if h.originalDefaultChannelBase != "" { @@ -284,9 +286,11 @@ func (h *IntegrationTestHarness) SetupMockAWS() *awsup.MockAWSCloud { // SetupMockGCE configures a mock GCE cloud provider func (h *IntegrationTestHarness) SetupMockGCE() *gcemock.MockGCECloud { - project := "testproject" - region := "us-test1" + return h.SetupMockGCEForProject("testproject", "us-test1") +} +// SetupMockGCEForProject configures a mock GCE cloud provider for the specified project and region. +func (h *IntegrationTestHarness) SetupMockGCEForProject(project, region string) *gcemock.MockGCECloud { cloud := gcemock.InstallMockGCECloud(region, project) cloud.Compute().Networks().Insert(project, &compute.Network{ @@ -369,4 +373,5 @@ func (h *IntegrationTestHarness) MockKopsVersion(version string) { h.originalKopsVersion = kopsroot.Version kopsroot.Version = version + wellknownassets.ResetCachesForTesting() } diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_cluster-completed.spec_content new file mode 100644 index 0000000000000..1c6a50d1514b5 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_cluster-completed.spec_content @@ -0,0 +1,248 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: k8s1.equivalence.example.internal +spec: + api: + loadBalancer: + type: Public + authorization: + rbac: {} + channel: stable + cloudConfig: + gcpPDCSIDriver: + defaultStorageClassName: balanced-csi + enabled: true + version: v1.22.1 + manageStorageClasses: true + multizone: true + nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node + cloudControllerManager: + allocateNodeCIDRs: true + cidrAllocatorType: CloudAllocator + clusterCIDR: 100.96.0.0/11 + clusterName: k8s1-equivalence-example-internal + controllers: + - '*' + image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0 + leaderElection: + leaderElect: true + cloudProvider: gce + clusterDNSDomain: cluster.local + configBase: memfs://tests/k8s1.equivalence.example.internal + containerd: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 + etcdClusters: + - backups: + backupStore: memfs://tests/k8s1.equivalence.example.internal/backups/etcd/main + cpuRequest: 200m + etcdMembers: + - instanceGroup: control-plane-us-west1-a + name: a + - instanceGroup: control-plane-us-west1-b + name: b + - instanceGroup: control-plane-us-west1-c + name: c + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: main + version: 3.6.11 + - backups: + backupStore: memfs://tests/k8s1.equivalence.example.internal/backups/etcd/events + cpuRequest: 100m + etcdMembers: + - instanceGroup: control-plane-us-west1-a + name: a + - instanceGroup: control-plane-us-west1-b + name: b + - instanceGroup: control-plane-us-west1-c + name: c + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: events + version: 3.6.11 + iam: + allowContainerRegistry: true + legacy: false + keyStore: memfs://tests/k8s1.equivalence.example.internal/pki + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + enableAdmissionPlugins: + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - NodeRestriction + - ResourceQuota + - RuntimeClass + - ServiceAccount + - ValidatingAdmissionPolicy + - ValidatingAdmissionWebhook + enableAggregatorRouting: true + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: registry.k8s.io/kube-apiserver:v1.35.5 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.k8s1.equivalence.example.internal + serviceAccountJWKSURI: https://api.internal.k8s1.equivalence.example.internal/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: k8s1.equivalence.example.internal + configureCloudRoutes: false + image: registry.k8s.io/kube-controller-manager:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeDNS: + cacheMaxConcurrent: 150 + cacheMaxSize: 1000 + cpuRequest: 100m + domain: cluster.local + memoryLimit: 170Mi + memoryRequest: 70Mi + nodeLocalDNS: + cpuRequest: 25m + enabled: false + image: registry.k8s.io/dns/k8s-dns-node-cache:1.26.0 + memoryRequest: 5Mi + provider: CoreDNS + serverIP: 100.64.0.10 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + enabled: false + image: registry.k8s.io/kube-proxy:v1.35.5 + logLevel: 2 + kubeScheduler: + image: registry.k8s.io/kube-scheduler:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: 1.35.5 + masterKubelet: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ImageVolume: "true" + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + metricsServer: + enabled: true + insecure: true + networkID: default + networking: + cilium: + agentPrometheusPort: 9090 + bpfCTGlobalAnyMax: 262144 + bpfCTGlobalTCPMax: 524288 + bpfLBAlgorithm: random + bpfLBMaglevTableSize: "16381" + bpfLBMapMax: 65536 + bpfNATGlobalMax: 524288 + bpfNeighGlobalMax: 524288 + bpfPolicyMapMax: 16384 + clusterName: default + cniExclusive: true + cpuRequest: 25m + disableCNPStatusUpdates: true + disableMasquerade: false + enableBPFMasquerade: false + enableEndpointHealthChecking: true + enableL7Proxy: true + enableLocalRedirectPolicy: false + enableNodePort: true + enableRemoteNodeIdentity: true + enableUnreachableRoutes: false + gatewayAPI: + enabled: false + hubble: + enabled: false + identityAllocationMode: crd + identityChangeGracePeriod: 5s + ingress: + enabled: false + ipam: kubernetes + memoryRequest: 128Mi + monitorAggregation: medium + sidecarIstioProxyImage: cilium/istio_proxy + toFqdnsDnsRejectResponseCode: refused + tunnel: vxlan + version: v1.18.6 + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + project: example-project + secretStore: memfs://tests/k8s1.equivalence.example.internal/secrets + serviceClusterIPRange: 100.64.0.0/13 + subnets: + - cidr: 10.0.16.0/20 + name: us-west1 + region: us-west1 + type: Private + topology: + dns: + type: None diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-events_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-events_content new file mode 100644 index 0000000000000..e9f98071c1d51 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-events_content @@ -0,0 +1,4 @@ +{ + "memberCount": 3, + "etcdVersion": "3.6.11" +} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-main_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-main_content new file mode 100644 index 0000000000000..e9f98071c1d51 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_etcd-cluster-spec-main_content @@ -0,0 +1,4 @@ +{ + "memberCount": 3, + "etcdVersion": "3.6.11" +} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-bootstrap_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-bootstrap_content new file mode 100644 index 0000000000000..be48d68b5681c --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-bootstrap_content @@ -0,0 +1,115 @@ +kind: Addons +metadata: + name: bootstrap +spec: + addons: + - id: k8s-1.16 + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 3259476993e6e6d7232ea17cdf9a2c6989fc71c43aa69b76084e17dd85a6c039 + name: kops-controller.addons.k8s.io + needsRollingUpdate: control-plane + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.12 + manifest: coredns.addons.k8s.io/k8s-1.12.yaml + manifestHash: 235b4f3b6100162878ef436873e2167872f1233eca78dc4e2dcb0822709ca9be + name: coredns.addons.k8s.io + selector: + k8s-addon: coredns.addons.k8s.io + version: 9.99.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: da91eb5cf9a29f1b03510007d6d54603aef2fc23a305abc9ba496c510dfd3bc7 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: 9.99.0 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 686cc69e559a1c6f5e8b94e38de54a575a25c432ed5ceec565244b965fb5f07f + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 9.99.0 + - id: k8s-1.11 + manifest: metrics-server.addons.k8s.io/k8s-1.11.yaml + manifestHash: af124c30376f03ec60e86b5852a27b34ffde2844946844ace6f09335380d8c63 + name: metrics-server.addons.k8s.io + selector: + k8s-app: metrics-server + version: 9.99.0 + - id: v1.7.0 + manifest: storage-gce.addons.k8s.io/v1.7.0.yaml + manifestHash: 4f73c7d683f04e61d60e90053a6db1fd82272dc0c13790390740f5e207181c77 + name: storage-gce.addons.k8s.io + selector: + k8s-addon: storage-gce.addons.k8s.io + version: 9.99.0 + - id: k8s-1.23 + manifest: gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml + manifestHash: cbe54943f0be055be0dd47d80fc97744fa8835acd1e75dff244b401a85db61f2 + name: gcp-pd-csi-driver.addons.k8s.io + selector: + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + version: 9.99.0 + - id: k8s-1.23 + manifest: gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml + manifestHash: 486044a21e7c02a1b345bbf283aab27d50858667a82322232d1b2715671f4a0a + name: gcp-cloud-controller.addons.k8s.io + prune: + kinds: + - kind: ConfigMap + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - kind: Service + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - kind: ServiceAccount + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: apps + kind: DaemonSet + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: apps + kind: Deployment + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: apps + kind: StatefulSet + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: policy + kind: PodDisruptionBudget + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRole + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + - group: rbac.authorization.k8s.io + kind: Role + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + - group: rbac.authorization.k8s.io + kind: RoleBinding + labelSelector: addon.kops.k8s.io/name=gcp-cloud-controller.addons.k8s.io,app.kubernetes.io/managed-by=kops + namespaces: + - kube-system + selector: + k8s-addon: gcp-cloud-controller.addons.k8s.io + version: 9.99.0 + - id: k8s-1.16 + manifest: networking.cilium.io/k8s-1.16-v1.15.yaml + manifestHash: 9cc95e89e819148d719c87abe4471231ee7f94f4766ad1d010c6e0ddcee92a66 + name: networking.cilium.io + needsRollingUpdate: all + selector: + role.kubernetes.io/networking: "1" + version: 9.99.0 diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-coredns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-coredns.addons.k8s.io-k8s-1.12_content new file mode 100644 index 0000000000000..b861ee1e3d8ca --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-coredns.addons.k8s.io-k8s-1.12_content @@ -0,0 +1,384 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: coredns + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system + +--- + +apiVersion: v1 +data: + Corefile: |- + .:53 { + errors + health { + lameduck 10s + } + ready + kubernetes cluster.local. in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + hosts /rootfs/etc/hosts k8s1.equivalence.example.internal { + ttl 30 + fallthrough + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: coredns + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-dns + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.13.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /ready + port: 8181 + scheme: HTTP + periodSeconds: 5 + timeoutSeconds: 5 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/coredns + name: config-volume + readOnly: true + - mountPath: /rootfs/etc/hosts + name: etc-hosts + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: CriticalAddonsOnly + operator: Exists + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + k8s-app: kube-dns + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - configMap: + name: coredns + name: config-volume + - hostPath: + path: /etc/hosts + type: File + name: etc-hosts + +--- + +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: CoreDNS + name: kube-dns + namespace: kube-system + resourceVersion: "0" +spec: + clusterIP: 100.64.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP + selector: + k8s-app: kube-dns + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: kube-dns + namespace: kube-system +spec: + maxUnavailable: 50% + selector: + matchLabels: + k8s-app: kube-dns + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - replicationcontrollers/scale + verbs: + - get + - update +- apiGroups: + - extensions + - apps + resources: + - deployments/scale + - replicasets/scale + verbs: + - get + - update +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + name: coredns-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: coredns-autoscaler +subjects: +- kind: ServiceAccount + name: coredns-autoscaler + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + addon.kops.k8s.io/name: coredns.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: coredns.addons.k8s.io + k8s-app: coredns-autoscaler + kubernetes.io/cluster-service: "true" + name: coredns-autoscaler + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: coredns-autoscaler + template: + metadata: + labels: + k8s-app: coredns-autoscaler + kops.k8s.io/managed-by: kops + spec: + containers: + - command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=coredns-autoscaler + - --target=Deployment/coredns + - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} + - --logtostderr=true + - --v=2 + image: registry.k8s.io/cpa/cluster-proportional-autoscaler:v1.9.0 + name: autoscaler + resources: + requests: + cpu: 20m + memory: 10Mi + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: coredns-autoscaler + tolerations: + - key: CriticalAddonsOnly + operator: Exists diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000000..2ff4c4df98759 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,549 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + component: cloud-controller-manager + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + component: cloud-controller-manager + template: + metadata: + labels: + component: cloud-controller-manager + kops.k8s.io/managed-by: kops + tier: control-plane + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - args: + - --allocate-node-cidrs=true + - --cidr-allocator-type=CloudAllocator + - --cluster-cidr=100.96.0.0/11 + - --cluster-name=k8s1-equivalence-example-internal + - --controllers=* + - --leader-elect=true + - --v=2 + - --cloud-provider=gce + - --use-service-account-credentials=true + - --cloud-config=/etc/kubernetes/cloud.config + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: registry.k8s.io/cloud-provider-gcp/cloud-controller-manager:v35.0.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + name: cloud-controller-manager + resources: + requests: + cpu: 200m + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node.kubernetes.io/not-ready + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + type: "" + name: cloudconfig + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resourceNames: + - cloud-controller-manager + resources: + - leases + verbs: + - get + - update +- apiGroups: + - "" + resources: + - endpoints + - serviceaccounts + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - update + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - update +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - '*' + resources: + - '*' + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system::leader-locking-cloud-controller-manager + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - watch +- apiGroups: + - "" + resourceNames: + - cloud-controller-manager + resources: + - configmaps + verbs: + - create + - patch + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system:controller:cloud-node-controller +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - update + - delete + - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - list + - update + - delete + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - pods/status + verbs: + - list + - delete + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system::leader-locking-cloud-controller-manager + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system::leader-locking-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system:controller:cloud-node-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:controller:cloud-node-controller +subjects: +- kind: ServiceAccount + name: cloud-node-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: system:controller:pvl-controller +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - persistentvolumes + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: gce:cloud-provider + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - patch + - update + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: gce:cloud-provider +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + kubernetes.io/deprecation: cloud-provider role is DEPRECATED in the concern of + potential collisions and will be removed in 1.16. Do not use this role. + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: cloud-provider + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - patch + - update + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + kubernetes.io/deprecation: cloud-provider clusterrole is DEPRECATED in the concern + of potential collisions and will be removed in 1.16. Do not use this role. + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: cloud-provider +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: gce:cloud-provider + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gce:cloud-provider +subjects: +- kind: ServiceAccount + name: cloud-provider + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-cloud-controller.addons.k8s.io + addonmanager.kubernetes.io/mode: Reconcile + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-cloud-controller.addons.k8s.io + name: gce:cloud-provider +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gce:cloud-provider +subjects: +- kind: ServiceAccount + name: cloud-provider + namespace: kube-system diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content new file mode 100644 index 0000000000000..6ee47c8451181 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content @@ -0,0 +1,868 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "false" + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: standard-csi +parameters: + type: pd-standard +provisioner: pd.csi.storage.gke.io +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-csi +parameters: + type: pd-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: balanced-storage +parameters: + provisioned-iops-on-create: "5000" + provisioned-throughput-on-create: 250Mi + type: hyperdisk-balanced +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: ssd-csi +parameters: + type: pd-ssd +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + +--- + +apiVersion: v1 +kind: Namespace +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: gce-pd-csi-driver + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-node-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + k8s-app: gcp-compute-persistent-disk-csi-driver + name: csi-gce-pd-leaderelection-role + namespace: gce-pd-csi-driver +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-attacher-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-provisioner-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list +- apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-resizer-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-snapshotter-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - patch +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + k8s-app: gcp-compute-persistent-disk-csi-driver + name: csi-gce-pd-controller-leaderelection-binding + namespace: gce-pd-csi-driver +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: csi-gce-pd-leaderelection-role +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-node-deploy +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller-attacher-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-attacher-role +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller-deploy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-controller-deploy +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller-provisioner-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-provisioner-role +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller-snapshotter-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-snapshotter-role +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-node-deploy +subjects: +- kind: ServiceAccount + name: csi-gce-pd-node-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-resizer-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: csi-gce-pd-resizer-role +subjects: +- kind: ServiceAccount + name: csi-gce-pd-controller-sa + namespace: gce-pd-csi-driver + +--- + +apiVersion: scheduling.k8s.io/v1 +description: This priority class should be used for the GCE PD CSI driver controller + deployment only. +globalDefault: false +kind: PriorityClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller +value: 900000000 + +--- + +apiVersion: scheduling.k8s.io/v1 +description: This priority class should be used for the GCE PD CSI driver node deployment + only. +globalDefault: false +kind: PriorityClass +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-node +value: 900001000 + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-controller + namespace: gce-pd-csi-driver +spec: + replicas: 1 + selector: + matchLabels: + app: gcp-compute-persistent-disk-csi-driver + template: + metadata: + labels: + app: gcp-compute-persistent-disk-csi-driver + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --feature-gates=Topology=true + - --http-endpoint=:22011 + - --leader-election-namespace=$(PDCSI_NAMESPACE) + - --timeout=250s + - --extra-create-metadata + - --leader-election + - --default-fstype=ext4 + - --controller-publish-readonly=true + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/sig-storage/csi-provisioner:v6.0.0 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 10 + name: csi-provisioner + ports: + - containerPort: 22011 + name: http-endpoint + protocol: TCP + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --http-endpoint=:22012 + - --leader-election + - --leader-election-namespace=$(PDCSI_NAMESPACE) + - --timeout=250s + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/sig-storage/csi-attacher:v4.8.1 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 10 + name: csi-attacher + ports: + - containerPort: 22012 + name: http-endpoint + protocol: TCP + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --http-endpoint=:22013 + - --leader-election + - --leader-election-namespace=$(PDCSI_NAMESPACE) + - --handle-volume-inuse-error=false + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/sig-storage/csi-resizer:v2.0.0 + livenessProbe: + failureThreshold: 1 + httpGet: + path: /healthz/leader-election + port: http-endpoint + initialDelaySeconds: 10 + periodSeconds: 20 + timeoutSeconds: 10 + name: csi-resizer + ports: + - containerPort: 22013 + name: http-endpoint + protocol: TCP + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --metrics-address=:22014 + - --leader-election + - --leader-election-namespace=$(PDCSI_NAMESPACE) + - --timeout=300s + - --retry-interval-max=60s + env: + - name: PDCSI_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.1 + name: csi-snapshotter + volumeMounts: + - mountPath: /csi + name: socket-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --extra-labels=k8s-io-cluster-name=k8s1-equivalence-example-internal + - --supports-dynamic-iops-provisioning=hyperdisk-balanced,hyperdisk-extreme + - --supports-dynamic-throughput-provisioning=hyperdisk-balanced,hyperdisk-throughput,hyperdisk-ml + - --enable-data-cache + - --run-node-service=false + env: [] + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + volumeMounts: + - mountPath: /csi + name: socket-dir + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: csi-gce-pd-controller + serviceAccountName: csi-gce-pd-controller-sa + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + volumes: + - emptyDir: {} + name: socket-dir + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: csi-gce-pd-node + namespace: gce-pd-csi-driver +spec: + selector: + matchLabels: + app: gcp-compute-persistent-disk-csi-driver + template: + metadata: + labels: + app: gcp-compute-persistent-disk-csi-driver + kops.k8s.io/managed-by: kops + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --v=5 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=/var/lib/kubelet/plugins/pd.csi.storage.gke.io/csi.sock + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.13.0 + name: csi-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --v=5 + - --endpoint=unix:/csi/csi.sock + - --run-controller-service=false + - --enable-data-cache + image: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver:v1.22.1 + name: gce-pd-driver + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - mountPath: /etc/udev + name: udev-rules-etc + - mountPath: /lib/udev + name: udev-rules-lib + - mountPath: /run/udev + name: udev-socket + - mountPath: /sys + name: sys + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: csi-gce-pd-node + serviceAccountName: csi-gce-pd-node-sa + tolerations: + - operator: Exists + volumes: + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/pd.csi.storage.gke.io/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /etc/udev + type: Directory + name: udev-rules-etc + - hostPath: + path: /lib/udev + type: Directory + name: udev-rules-lib + - hostPath: + path: /run/udev + type: Directory + name: udev-socket + - hostPath: + path: /sys + type: Directory + name: sys + +--- + +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + labels: + addon.kops.k8s.io/name: gcp-pd-csi-driver.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: gcp-pd-csi-driver.addons.k8s.io + name: pd.csi.storage.gke.io +spec: + attachRequired: true + podInfoOnMount: false diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kops-controller.addons.k8s.io-k8s-1.16_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kops-controller.addons.k8s.io-k8s-1.16_content new file mode 100644 index 0000000000000..b0e039f8851ec --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kops-controller.addons.k8s.io-k8s-1.16_content @@ -0,0 +1,219 @@ +apiVersion: v1 +data: + config.yaml: | + {"clusterName":"k8s1.equivalence.example.internal","cloud":"gce","configBase":"memfs://tests/k8s1.equivalence.example.internal","secretStore":"memfs://tests/k8s1.equivalence.example.internal/secrets","server":{"Listen":":3988","provider":{"gce":{"projectID":"example-project","region":"us-west1","clusterName":"k8s1.equivalence.example.internal","MaxTimeSkew":300}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server"]}} +kind: ConfigMap +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.35.1 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + annotations: + dns.alpha.kubernetes.io/internal: kops-controller.internal.k8s1.equivalence.example.internal + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + kops.k8s.io/managed-by: kops + version: v1.35.1 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - key: kops.k8s.io/kops-controller-pki + operator: Exists + containers: + - args: + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + command: null + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + - name: KOPS_RUN_TOO_NEW_VERSION + value: "1" + image: registry.k8s.io/kops/kops-controller:1.35.1 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + runAsUser: 10011 + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: null + priorityClassName: system-cluster-critical + serviceAccount: kops-controller + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + type: OnDelete + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + - coordination.k8s.io + resourceNames: + - kops-controller-leader + resources: + - configmaps + - leases + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + - coordination.k8s.io + resources: + - configmaps + - leases + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: kops-controller.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content new file mode 100644 index 0000000000000..bb4d69ef0ef88 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: kubelet-api.rbac.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: kubelet-api.rbac.addons.k8s.io + name: kops:system:kubelet-api-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kubelet-api diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-limit-range.addons.k8s.io_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-limit-range.addons.k8s.io_content new file mode 100644 index 0000000000000..c8939ddffd3ad --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-limit-range.addons.k8s.io_content @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: LimitRange +metadata: + labels: + addon.kops.k8s.io/name: limit-range.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-addon: limit-range.addons.k8s.io + name: limits + namespace: default +spec: + limits: + - defaultRequest: + cpu: 100m + type: Container diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-metrics-server.addons.k8s.io-k8s-1.11_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-metrics-server.addons.k8s.io-k8s-1.11_content new file mode 100644 index 0000000000000..935871b565ec0 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-metrics-server.addons.k8s.io-k8s-1.11_content @@ -0,0 +1,266 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + ports: + - appProtocol: https + name: https + port: 443 + protocol: TCP + targetPort: https + selector: + k8s-app: metrics-server + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + labels: + k8s-app: metrics-server + kops.k8s.io/managed-by: kops + spec: + containers: + - args: + - --secure-port=10250 + - --kubelet-use-node-status-port + - --metric-resolution=15s + - --kubelet-preferred-address-types=InternalIP + - --cert-dir=/tmp + - --kubelet-insecure-tls + image: registry.k8s.io/metrics-server/metrics-server:v0.8.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + name: metrics-server + ports: + - containerPort: 10250 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readyz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /tmp + name: tmp-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: metrics-server + topologySpreadConstraints: + - labelSelector: + matchLabels: + k8s-app: metrics-server + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + - labelSelector: + matchLabels: + k8s-app: metrics-server + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + volumes: + - emptyDir: {} + name: tmp-dir + +--- + +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: metrics-server + namespace: kube-system + version: v1beta1 + versionPriority: 100 + +--- + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + addon.kops.k8s.io/name: metrics-server.addons.k8s.io + app.kubernetes.io/managed-by: kops + k8s-app: metrics-server + name: metrics-server + namespace: kube-system +spec: + minAvailable: 1 + selector: + matchLabels: + k8s-app: metrics-server diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-networking.cilium.io-k8s-1.16_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-networking.cilium.io-k8s-1.16_content new file mode 100644 index 0000000000000..4c9065327ece3 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-networking.cilium.io-k8s-1.16_content @@ -0,0 +1,1194 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium + io.cilium/app: operator + name: cilium-operator + role.kubernetes.io/networking: "1" + name: cilium-operator + namespace: kube-system +spec: + maxUnavailable: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: cilium + namespace: kube-system + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: cilium-operator + namespace: kube-system + +--- + +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + auto-direct-node-routes: "false" + bpf-distributed-lru: "false" + bpf-events-drop-enabled: "true" + bpf-events-policy-verdict-enabled: "true" + bpf-events-trace-enabled: "true" + bpf-lb-acceleration: disabled + bpf-lb-algorithm-annotation: "false" + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode-annotation: "false" + bpf-lb-sock: "false" + bpf-lb-sock-hostns-only: "false" + bpf-lb-source-range-all-types: "false" + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-policy-stats-map-max: "65536" + bpf-root: /sys/fs/bpf + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-name: default + cluster-pool-ipv4-mask-size: "24" + clustermesh-enable-endpoint-sync: "false" + clustermesh-enable-mcs-api: "false" + cni-exclusive: "true" + cni-log-file: /var/run/cilium/cilium-cni.log + custom-cni-conf: "false" + datapath-mode: veth + debug: "false" + debug-verbose: "" + default-lb-service-ipam: lbipam + direct-routing-skip-unreachable: "false" + disable-endpoint-crd: "false" + dnsproxy-enable-transparent-mode: "true" + dnsproxy-socket-linger-timeout: "10" + egress-gateway-reconciliation-trigger-interval: 1s + enable-auto-protect-node-port-range: "true" + enable-bpf-clock-probe: "false" + enable-bpf-masquerade: "false" + enable-endpoint-health-checking: "true" + enable-endpoint-lockdown-on-policy-overflow: "false" + enable-health-check-loadbalancer-ip: "false" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-internal-traffic-policy: "true" + enable-ipv4: "true" + enable-ipv4-big-tcp: "false" + enable-ipv4-masquerade: "true" + enable-ipv6: "false" + enable-ipv6-big-tcp: "false" + enable-ipv6-masquerade: "false" + enable-k8s-networkpolicy: "true" + enable-l2-neigh-discovery: "false" + enable-l7-proxy: "true" + enable-lb-ipam: "true" + enable-local-redirect-policy: "false" + enable-masquerade-to-route-source: "false" + enable-node-port: "true" + enable-node-selector-labels: "false" + enable-non-default-deny-policies: "true" + enable-policy: default + enable-sctp: "false" + enable-service-topology: "false" + enable-source-ip-verification: "true" + enable-svc-source-range-check: "true" + enable-tcx: "true" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + envoy-access-log-buffer-size: "4096" + envoy-base-id: "0" + envoy-keep-cap-netbindservice: "false" + external-envoy-proxy: "false" + health-check-icmp-failure-threshold: "3" + http-retry-count: "3" + http-stream-idle-timeout: "300" + identity-allocation-mode: crd + identity-change-grace-period: 5s + identity-gc-interval: 15m0s + identity-heartbeat-timeout: 30m0s + identity-management-mode: agent + install-no-conntrack-iptables-rules: "false" + ipam: kubernetes + ipam-cilium-node-update-rate: 15s + iptables-random-fully: "false" + k8s-require-ipv4-pod-cidr: "false" + k8s-require-ipv6-pod-cidr: "false" + kube-proxy-replacement: "true" + max-connected-clusters: "255" + mesh-auth-enabled: "true" + mesh-auth-gc-interval: 5m0s + mesh-auth-queue-size: "1024" + mesh-auth-rotated-identities-queue-size: "1024" + metrics-sampling-interval: 5m + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + nat-map-stats-entries: "32" + nat-map-stats-interval: 30s + node-port-bind-protection: "true" + nodeport-addresses: "" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + policy-cidr-match-mode: "" + policy-default-local-cluster: "false" + preallocate-bpf-maps: "false" + procfs: /host/proc + proxy-connect-timeout: "2" + proxy-idle-timeout-seconds: "60" + proxy-initial-fetch-timeout: "30" + proxy-max-concurrent-retries: "128" + proxy-max-connection-duration-seconds: "0" + proxy-max-requests-per-connection: "0" + proxy-xff-num-trusted-hops-egress: "0" + proxy-xff-num-trusted-hops-ingress: "0" + remove-cilium-node-taints: "true" + routing-mode: tunnel + service-no-backend-response: reject + set-cilium-is-up-condition: "true" + set-cilium-node-taints: "true" + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "1000" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-preallocate-identities: "true" + tofqdns-proxy-response-max-delay: 100ms + tunnel-protocol: vxlan + tunnel-source-port-range: 0-0 + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" + write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist +kind: ConfigMap +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + role.kubernetes.io/networking: "1" + name: cilium-config + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - pods + - endpoints + - nodes + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch + - get +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs + - ciliumclusterwideenvoyconfigs + - ciliumclusterwidenetworkpolicies + - ciliumegressgatewaypolicies + - ciliumendpoints + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumidentities + - ciliumlocalredirectpolicies + - ciliumnetworkpolicies + - ciliumnodes + - ciliumnodeconfigs + - ciliumcidrgroups + - ciliuml2announcementpolicies + - ciliumpodippools + verbs: + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + - ciliumendpoints + - ciliumnodes + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + verbs: + - delete + - get +- apiGroups: + - cilium.io + resources: + - ciliumnodes + - ciliumnodes/status + verbs: + - get + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints/status + - ciliumendpoints + - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resourceNames: + - cilium-config + resources: + - configmaps + verbs: + - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - update + - patch +- apiGroups: + - "" + resources: + - namespaces + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + - endpoints + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumclusterwidenetworkpolicies + verbs: + - create + - update + - deletecollection + - patch + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies/status + - ciliumclusterwidenetworkpolicies/status + verbs: + - patch + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpoints + - ciliumidentities + verbs: + - delete + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumidentities + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumnodes + verbs: + - create + - update + - get + - list + - watch + - delete +- apiGroups: + - cilium.io + resources: + - ciliumnodes/status + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumendpointslices + - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs + verbs: + - create + - update + - get + - list + - watch + - delete + - patch +- apiGroups: + - cilium.io + resources: + - ciliumbgpclusterconfigs/status + - ciliumbgppeerconfigs/status + verbs: + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resourceNames: + - ciliumloadbalancerippools.cilium.io + - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io + - ciliumclusterwideenvoyconfigs.cilium.io + - ciliumclusterwidenetworkpolicies.cilium.io + - ciliumegressgatewaypolicies.cilium.io + - ciliumendpoints.cilium.io + - ciliumendpointslices.cilium.io + - ciliumenvoyconfigs.cilium.io + - ciliumidentities.cilium.io + - ciliumlocalredirectpolicies.cilium.io + - ciliumnetworkpolicies.cilium.io + - ciliumnodes.cilium.io + - ciliumnodeconfigs.cilium.io + - ciliumcidrgroups.cilium.io + - ciliuml2announcementpolicies.cilium.io + - ciliumpodippools.cilium.io + - ciliumgatewayclassconfigs.cilium.io + resources: + - customresourcedefinitions + verbs: + - update +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools + - ciliumpodippools + - ciliumbgppeeringpolicies + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides + - ciliumbgppeerconfigs + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumpodippools + verbs: + - create +- apiGroups: + - cilium.io + resources: + - ciliumloadbalancerippools/status + verbs: + - patch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: cilium-operator + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium-config-agent + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/part-of: cilium + role.kubernetes.io/networking: "1" + name: cilium-config-agent + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-config-agent +subjects: +- kind: ServiceAccount + name: cilium + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + k8s-app: cilium + kubernetes.io/cluster-service: "true" + role.kubernetes.io/networking: "1" + name: cilium + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" + template: + metadata: + annotations: + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined + container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined + container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined + kubectl.kubernetes.io/default-container: cilium-agent + labels: + app.kubernetes.io/name: cilium-agent + app.kubernetes.io/part-of: cilium + k8s-app: cilium + kops.k8s.io/managed-by: kops + kubernetes.io/cluster-service: "true" + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: cilium + topologyKey: kubernetes.io/hostname + automountServiceAccountToken: true + containers: + - args: + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + divisor: "1" + resource: limits.memory + - name: KUBE_CLIENT_BACKOFF_BASE + value: "1" + - name: KUBE_CLIENT_BACKOFF_DURATION + value: "120" + - name: KUBERNETES_SERVICE_HOST + value: api.internal.k8s1.equivalence.example.internal + - name: KUBERNETES_SERVICE_PORT + value: "443" + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /cni-uninstall.sh + livenessProbe: + failureThreshold: 10 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + - name: require-k8s-connectivity + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent + ports: null + readinessProbe: + failureThreshold: 3 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 25m + memory: 128Mi + securityContext: + capabilities: + add: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + startupProbe: + failureThreshold: 300 + httpGet: + host: 127.0.0.1 + httpHeaders: + - name: brief + value: "true" + path: /healthz + port: 9879 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 2 + successThreshold: 1 + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/proc/sys/net + name: host-proc-sys-net + - mountPath: /host/proc/sys/kernel + name: host-proc-sys-kernel + - mountPath: /sys/fs/bpf + mountPropagation: HostToContainer + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /var/run/cilium/netns + mountPropagation: HostToContainer + name: cilium-netns + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + - mountPath: /tmp + name: tmp + hostNetwork: true + initContainers: + - command: + - cilium-dbg + - build-config + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: KUBERNETES_SERVICE_HOST + value: api.internal.k8s1.equivalence.example.internal + - name: KUBERNETES_SERVICE_PORT + value: "443" + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: config + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /tmp + name: tmp + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-mount /hostbin/cilium-mount; + nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; + rm /hostbin/cilium-mount + env: + - name: CGROUP_ROOT + value: /run/cilium/cgroupv2 + - name: BIN_PATH + value: /opt/cni/bin + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: mount-cgroup + securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - command: + - sh + - -ec + - | + cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; + nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; + rm /hostbin/cilium-sysctlfix + env: + - name: BIN_PATH + value: /opt/cni/bin + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: apply-sysctl-overwrites + securityContext: + capabilities: + add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /hostproc + name: hostproc + - mountPath: /hostbin + name: cni-path + - args: + - mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf + command: + - /bin/bash + - -c + - -- + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: mount-bpf-fs + securityContext: + privileged: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + name: bpf-maps + - command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + key: write-cni-conf-when-ready + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: api.internal.k8s1.equivalence.example.internal + - name: KUBERNETES_SERVICE_PORT + value: "443" + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: clean-cilium-state + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + - SYS_ADMIN + - SYS_RESOURCE + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /run/cilium/cgroupv2 + mountPropagation: HostToContainer + name: cilium-cgroup + - mountPath: /var/run/cilium + name: cilium-run + - command: + - /install-plugin.sh + image: quay.io/cilium/cilium:v1.18.6 + imagePullPolicy: IfNotPresent + name: install-cni-binaries + resources: + requests: + cpu: 100m + memory: 10Mi + securityContext: + capabilities: + drop: + - ALL + seLinuxOptions: + level: s0 + type: spc_t + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-path + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + restartPolicy: Always + securityContext: + seccompProfile: + type: Unconfined + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - operator: Exists + volumes: + - emptyDir: {} + name: tmp + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + - hostPath: + path: /var/run/netns + type: DirectoryOrCreate + name: cilium-netns + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps + - hostPath: + path: /proc + type: Directory + name: hostproc + - hostPath: + path: /run/cilium/cgroupv2 + type: DirectoryOrCreate + name: cilium-cgroup + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + - hostPath: + path: /lib/modules + name: lib-modules + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock + - name: clustermesh-secrets + projected: + defaultMode: 256 + sources: + - secret: + name: cilium-clustermesh + optional: true + - secret: + items: + - key: tls.key + path: common-etcd-client.key + - key: tls.crt + path: common-etcd-client.crt + - key: ca.crt + path: common-etcd-client-ca.crt + name: clustermesh-apiserver-remote-cert + optional: true + - secret: + items: + - key: tls.key + path: local-etcd-client.key + - key: tls.crt + path: local-etcd-client.crt + - key: ca.crt + path: local-etcd-client-ca.crt + name: clustermesh-apiserver-local-cert + optional: true + - hostPath: + path: /proc/sys/net + type: Directory + name: host-proc-sys-net + - hostPath: + path: /proc/sys/kernel + type: Directory + name: host-proc-sys-kernel + updateStrategy: + type: OnDelete + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + addon.kops.k8s.io/name: networking.cilium.io + app.kubernetes.io/managed-by: kops + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium + io.cilium/app: operator + name: cilium-operator + role.kubernetes.io/networking: "1" + name: cilium-operator + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 50% + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: cilium-operator + app.kubernetes.io/part-of: cilium + io.cilium/app: operator + kops.k8s.io/managed-by: kops + name: cilium-operator + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + io.cilium/app: operator + topologyKey: kubernetes.io/hostname + automountServiceAccountToken: true + containers: + - args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + - --eni-tags=KubernetesCluster=k8s1.equivalence.example.internal + command: + - cilium-operator + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: api.internal.k8s1.equivalence.example.internal + - name: KUBERNETES_SERVICE_PORT + value: "443" + image: quay.io/cilium/operator:v1.18.6 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + name: cilium-operator + readinessProbe: + failureThreshold: 5 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 5 + timeoutSeconds: 3 + resources: + requests: + cpu: 25m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + restartPolicy: Always + securityContext: + seccompProfile: + type: RuntimeDefault + serviceAccountName: cilium-operator + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/master + operator: Exists + - key: node.kubernetes.io/not-ready + operator: Exists + - key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - key: node.cilium.io/agent-not-ready + operator: Exists + volumes: + - configMap: + name: cilium-config + name: cilium-config-path diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-storage-gce.addons.k8s.io-v1.7.0_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-storage-gce.addons.k8s.io-v1.7.0_content new file mode 100644 index 0000000000000..b5b781d7554ae --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_k8s1.equivalence.example.internal-addons-storage-gce.addons.k8s.io-v1.7.0_content @@ -0,0 +1,13 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addon.kops.k8s.io/name: storage-gce.addons.k8s.io + addonmanager.kubernetes.io/mode: EnsureExists + app.kubernetes.io/managed-by: kops + k8s-addon: storage-gce.addons.k8s.io + kubernetes.io/cluster-service: "true" + name: standard +parameters: + type: pd-standard +provisioner: kubernetes.io/gce-pd diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_kops-version.txt_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_kops-version.txt_content new file mode 100644 index 0000000000000..7eee785743325 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_kops-version.txt_content @@ -0,0 +1 @@ +1.35.1 diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-a_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-a_content new file mode 100644 index 0000000000000..37dbd720b2921 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-a_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s-io-etcd-events + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-events --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-b_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-b_content new file mode 100644 index 0000000000000..37dbd720b2921 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-b_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s-io-etcd-events + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-events --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-c_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-c_content new file mode 100644 index 0000000000000..37dbd720b2921 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-c_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-events + name: etcd-manager-events + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/events + --client-urls=https://__name__:4002 --cluster-name=etcd-events --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3997 --peer-urls=https://__name__:2381 + --quarantine-client-urls=https://__name__:3995 --v=6 --volume-name-tag=k8s-io-etcd-events + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-events --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 100m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-events + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd-events.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-a_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-a_content new file mode 100644 index 0000000000000..20eddfa02bcac --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-a_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s-io-etcd-main + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-main --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-b_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-b_content new file mode 100644 index 0000000000000..20eddfa02bcac --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-b_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s-io-etcd-main + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-main --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-c_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-c_content new file mode 100644 index 0000000000000..20eddfa02bcac --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-c_content @@ -0,0 +1,118 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + k8s-app: etcd-manager-main + name: etcd-manager-main + namespace: kube-system +spec: + containers: + - command: + - /bin/sh + - -c + - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /ko-app/etcd-manager + --backup-store=memfs://tests/k8s1.equivalence.example.internal/backups/etcd/main + --client-urls=https://__name__:4001 --cluster-name=etcd --containerized=true + --dns-suffix=.internal.k8s1.equivalence.example.internal --grpc-port=3996 --peer-urls=https://__name__:2380 + --quarantine-client-urls=https://__name__:3994 --v=6 --volume-name-tag=k8s-io-etcd-main + --volume-provider=gce --volume-tag=k8s-io-cluster-name=k8s1-equivalence-example-internal + --volume-tag=k8s-io-etcd-main --volume-tag=k8s-io-role-master=master > /tmp/pipe + 2>&1 + env: + - name: ETCD_MANAGER_DAILY_BACKUPS_RETENTION + value: 90d + image: registry.k8s.io/etcd-manager/etcd-manager-slim:v3.0.20260512 + name: etcd-manager + resources: + requests: + cpu: 200m + memory: 100Mi + securityContext: + privileged: true + volumeMounts: + - mountPath: /rootfs + name: rootfs + - mountPath: /run + name: run + - mountPath: /etc/kubernetes/pki/etcd-manager + name: pki + - mountPath: /opt + name: opt + - mountPath: /opt/etcd-v3.4.13 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.4.3 + name: etcd-v3-4-13 + - mountPath: /opt/etcd-v3.5.0 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.1 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.13 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.17 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.21 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.23 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.24 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.25 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.3 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.30 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.4 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.6 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.7 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.5.9 + name: etcd-v3-5-30 + - mountPath: /opt/etcd-v3.6.11 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.5 + name: etcd-v3-6-11 + - mountPath: /opt/etcd-v3.6.6 + name: etcd-v3-6-11 + - mountPath: /var/log/etcd.log + name: varlogetcd + hostNetwork: true + hostPID: true + priorityClassName: system-cluster-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - hostPath: + path: / + type: Directory + name: rootfs + - hostPath: + path: /run + type: DirectoryOrCreate + name: run + - hostPath: + path: /etc/kubernetes/pki/etcd-manager-main + type: DirectoryOrCreate + name: pki + - emptyDir: {} + name: opt + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.4.13 + name: etcd-v3-4-13 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.5.30 + name: etcd-v3-5-30 + - image: + pullPolicy: IfNotPresent + reference: registry.k8s.io/etcd:v3.6.11 + name: etcd-v3-6-11 + - hostPath: + path: /var/log/etcd.log + type: FileOrCreate + name: varlogetcd +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content new file mode 100644 index 0000000000000..2495c19ac4a4b --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: {} +spec: + containers: + - args: + - --ca-cert=/secrets/ca.crt + - --client-cert=/secrets/client.crt + - --client-key=/secrets/client.key + image: registry.k8s.io/kops/kube-apiserver-healthcheck:1.35.1 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /.kube-apiserver-healthcheck/healthz + port: 3990 + initialDelaySeconds: 5 + timeoutSeconds: 5 + name: healthcheck + resources: {} + securityContext: + runAsNonRoot: true + runAsUser: 10012 + volumeMounts: + - mountPath: /secrets + name: healthcheck-secrets + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/kube-apiserver-healthcheck/secrets + type: Directory + name: healthcheck-secrets +status: {} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-a_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-a_content new file mode 100644 index 0000000000000..0b898c6ef7f31 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-a_content @@ -0,0 +1,310 @@ +APIServerConfig: + API: {} + ClusterDNSDomain: cluster.local + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + enableAdmissionPlugins: + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - NodeRestriction + - ResourceQuota + - RuntimeClass + - ServiceAccount + - ValidatingAdmissionPolicy + - ValidatingAdmissionWebhook + enableAggregatorRouting: true + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: registry.k8s.io/kube-apiserver:v1.35.5 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.k8s1.equivalence.example.internal + serviceAccountJWKSURI: https://api.internal.k8s1.equivalence.example.internal/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9CGwPhRDpZpU4g/c33O + ywpM0HeuzLpYNaDuSYPKfUvF/sHdJQehy2B+W300Uwy79GaeGN7/1TIwAkuRTgRl + pq3Zwl9Q3A7uuTylyRjm47JLM2TEClK5u8Nz71qA+0QuhWzx0E1Qc72OqlNsk6Lb + I/7A/UnJCLNM5RacFTeERfKvaNkolS88LyEogmjRVnc14sZ5l7i7ltpLHwVIhPUs + cpTVRaxnuSxBcRoGkn3XDK8AmGhvfiP9Yb5YiZP5IqxdpHiJZkWBrlgc3rJdkef7 + BVXLnsR+KN17mlhbd6MXaocbpQpghJVAMJGKfMYP0J3PfRfHP3tQQvw5AGizOpMR + iwIDAQAB + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + - ab0b3c97a377733946584f86bd191526c267345feead4d8ecefbc701444d3b26@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-amd64 + - c12e9e476c47a180618cd4ee1bd12c86ee890efd8e74b4526b42e783c71d1a9b@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 + - f969b90f83d5a1445e901bdded0b75aa391bbc4d8569278553f2816205c32082@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-arm64 + - c7a38921fa96a8c8ccf798db9793a7de87801cfdfb0edddc324ecf882a4d1bb6@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIDETCCAfmgAwIBAgIRANK0IaP/wDWEXGUkKcVaIt0wDQYJKoZIhvcNAQELBQAw + IjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3ItY2EwHhcNMjYwNjAyMDA1 + MjUyWhcNMzYwNTMwMDA1MjUyWjAiMSAwHgYDVQQDExdhcGlzZXJ2ZXItYWdncmVn + YXRvci1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFUazdUPdsr + Q44pwSB935u0Rq2blh1iRtAleEs4nmfa/BeFAd4+fLJ6ozJOYkAQImYpFzOIj/PX + ragyDo/fRTPm3Qm2Ljn8ipaI46ndlHeUkKzqV/0ewJ+A+33Pdq2pp0maSTC50M5O + zwwkFblKHjk+MldtyUUxWxv8JB+UBC+nAFxFC1HVZeoBhSguhn5hwj8xgRe//ovq + eZN5XunF9x7Na/XOYnq1usF76KZCPAy2XEZqOB41Bhb/7W6BJNOeVkkQF7q9qD3D + lZUdjvgkoeMPjMxjyn6sEF2tgZr9HGSvzexAmtnm/QDDauR6C9LuHKAVsFt9BQt8 + pAu9shLcq7ECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB + Af8wHQYDVR0OBBYEFGptCok5udjNkXjg9hyJ/F9C/X1BMA0GCSqGSIb3DQEBCwUA + A4IBAQCC4P0xJsrUmwCEPRNisayuA8rf7KNc+OyHEYSMtHMQuy3mgtCd3OFvCHX/ + KjQiPeqOSuBSDQuu6N/f63H1nWuIxi1GmeYC+6iR38sBU2dpakF+8vncA2ln59ln + 5YuvzkIj9kkCdvnui1x10cASATJvLpawBgj48KLrayrnfA4LddTtV4flO8dogFZt + GP9c/1xtgagU4NW4VmJnyV970CEzbmY4ZpBw5PK+bR7wTd21jDMRr2cliwQs3t0f + fem5ZlODRE6r/kgGCkMEvqOpa8emLSZp3VzvbQWEFVWKccjA/Z36gHEOZjKTiF9O + gi+gTnPBYZzP+COI40aPW+etJKML + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIDADCCAeigAwIBAgIQWObTivI7jyAX7UCMUztaeDANBgkqhkiG9w0BAQsFADAa + MRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwHhcNMjYwNjAyMDA1MjUyWhcNMzYw + NTMwMDA1MjUyWjAaMRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwggEiMA0GCSqG + SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAPoMFLJT39dwnzqBg/zxKwWwBDV0H7N9H + xoUJvUyT/7E0BxzUKmajQhBkRNvaoebSFOvH8m6IQ4Zkum2EvyH7MNqUy3avt3GM + CTLhtSEiR8RhMsYDJ5CGgDooyPGBOJpITOY/o6Hie6RYk47A6jdes89xHYOr80Mq + gqtqvVkaxQDt/AIhCQWfml1+d4KUAt8qTt9umiwyIgjc30+ujk/dC5/KqNzQIHtA + pib5SkWabGMwKbHwYEB0ngzHO/tveJyKnrva1VUxk+3QuSSdkZbCSe/3hh3tDXJw + BZj5e67I7AzR5ZAVoYYR4Dad6BJkGR14kYdmoF0Lj5DnKSBIg23bAgMBAAGjQjBA + MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSex9K9 + QG60AcGgXHChRPAaqVztkjANBgkqhkiG9w0BAQsFAAOCAQEAokGJK5B8JPyemlbK + Qef+qt7MEACty5u+gTSvi0HOqW1xb24TaHWI3Y46OBaYcyEJSVkzkeWl47ZrhzED + qGab/ASFQJ8lZ2PEV++5ehitiPps6MZbAnUb8YO9BHoxuHUjrRcev7Lyvo17mB62 + slOGCB+GaYXAdj4RCZYOhrCoX26FURyxkSY9/2+qiIPitOUQKJyoNw/e6aDv+FUH + Jy81NzbzbEPjwEYBOWV094TY1zsA9GcXN+N+DuPrTGJhOYAtVuNYXWdedLdCLizx + Dz0aC4qA6W5zBxU2++jdnJQfIrnFq0MF+ugj+FO8lmKEiYYVoLOC921JChEEYfd5 + b2dkAg== + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDDzCCAfegAwIBAgIRALuXJ1SK65cmULxmNsByJewwDQYJKoZIhvcNAQELBQAw + ITEfMB0GA1UEAxMWZXRjZC1tYW5hZ2VyLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUy + NTJaFw0zNjA1MzAwMDUyNTJaMCExHzAdBgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1l + dmVudHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3Ir75KrvZOxap + /yTFuMCeoh9+WjzYh+Ey3EP+fEef2v0vJw89OEPaxcNVjlpY+4KXQerMua2OJHw2 + l3swhWA//91Pg+6wtN74XtC7AckKsN32LaV6xDEtJkY5vhXsuSYRPSQSH95S+po5 + Nzz7RtoFhlVQc9p946QHpCpys3cuu2fKc8XSsbK71EQePIY/x5sqfQ04v8X7PO7t + eFIwz/oRSwnZYOfj5zp9A3pefG595YSUrnjS2CVfiJMnuU+629pRtyQNuM1JlDyB + ymdrYzUD1bWuCn0q0EQ/iWBNO4QJi4nuX6Do6BsFDlLTspfae3F0lnaqKzpsJYDG + TKeYDDKbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ + MB0GA1UdDgQWBBQbhV5ghNsXkBjIcdZqUtvcf7SSMjANBgkqhkiG9w0BAQsFAAOC + AQEAnyqtKzkNLAN7nKnfBDkU1ZmsoLIkV8glzoQx4nA8rFjycvFPElWPu9yUvocO + BZV44PWXmG1oWfExGlW2PjHPhLkzGWMpvhzWXy9kBANOYDvRmI50F3I1oaZpAtF5 + SUAiWq9Mkrafn20aBBL8hNXUUhugrtrYY1cIb99TBrh+4WAGHLqfhw3qXHVhJTzJ + +P0tuatqwJfMWOpL3qBYWAQci5NSZg4VFvQgVCwaD3h+OoT0Z7wZeYytf8tOENZ2 + Erdb7tMDiXSLLHslVybrVhU8cd8ck7QMXhhK/FIOTJKCgJAIsD6lr+/c9hfh8Fpz + r55rasUA3eHKr6ekstPZOl6CEQ== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQd7hcMdMribsvFHJRe0eUrjANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLW1hbmFnZXItY2EtbWFpbjAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWlu + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArINQkwWh9p9BR9z/lDey + LXE8tmxuLTOXwpgzj2L9ynbFmRCF4cN8V3A8CEjy4f7woxM3sy9C5ckYq3thy0qx + F92Idu6Re/48j2hlOV15ZgQtgsgNtYy1g7RG46DDZAfya4RkQEvg/7Duiv0ZNl0N + sYotP7+g+jc4d2Nl22vh9Ml9ro1K06C/GiafK8ZNgVt+4ivV5sGSVYVzlnr6LcoK + tu9BQVIpyNzKOuJ0iUEZhJrtoI9adl+g/asLBiESa+xKNb+liQZ91Cq+/8H014LV + Qm7y9bk3S1TpqYTt2GlA+uJH5cp8eYxw6ZF0iIDhKayPSULvnjPgI1DhYyHaCUeo + FQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUTmUXDqEcpM0KXRNde162bck8FAYwDQYJKoZIhvcNAQELBQADggEBABRt + 1qNQ2KPQ5gGnoATexGg88VCYqJClFlBKcPU+vIom2SUBgrpddHWAHn8ba0ULhSrj + D4BrK997ss+wu9w31YwoGvXKWepyW+iFRcSDfl9WCzCQP0rdvEqrphtkX7L9Rygv + 3pHh7xtyN4hLUfJTvGjndjmcUD8c93Bzhz6269yIec7MHC0eoWipMXa3oyjCROCP + HSXm6ToU6c5ESZf92R+4FZ0DgIGWFNFVIT4+cN/49dGdb/cYPYMd3AfqkRk89r1+ + OIIs98PXiG6z7XNcTOZqzR7C/TbhIT6g+R6r09HdjasNihWVGNDHdAeYpzXYGtgZ + 7io//596d7+67eEXdgA= + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQH2KH1CnrdSogWVMlcYn7UTANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLXBlZXJzLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRz + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnTaTLSjrlVIhPkDzhW3D + wpkJw9bBZgdNSpcJ+3xnkn252W0jvffBxyGjc/KSxiFIzo/mlEmojUcM8sTVlZsk + gnUUzN2hts2Jl3CAENESJsAPk1IdKitEoODcpBJ2w26HlWDxoPPqeP6I5631pRr+ + u6lYDDBQBdMdaF6fnFary7w8t72Xyqbjcg6yAItWglYJvFQkNK+HZ8beamk6LQSC + AHnZH0iORI84z0t1kaqQtCaZqnWVM4CybcnpJ3GUMQss8ASWzKYclxzjjBPS+xxV + Y95ShdlDxo2X04qjNiGL+TXAYMDLVAMLh5n+GqTRxti6myv1GVJy61/XzrJjZJGg + 9wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUvtzj6D31eItxfnXQjJND5ScuAAQwDQYJKoZIhvcNAQELBQADggEBAEid + BZKaX3EdteUwk6Giq6tN3tLG8IOoEHe+Br8/e1Sa4A/1rqqkmTpcwMsByVXEs5W9 + N8QMP7KjJ37D5U1a664Nr7S+1ZyoRgV50N1arOzomgsIHaUchuuWUv/O4tfVwycf + jdBDqmieXUfw2ecpn5uEC3TRVrheVBm+EENOv/XjVflleDFQtLH1r58LVqdBvwkc + rVF8OtewTsBTpjZIOFVkPOn09NsawnCehWNcqzjaWe8cGue8aWfWyNDRpzC6Xj1l + KVoOym4Yy8X+zdshsqesNpEESfMvymyCE8UJdK7qfwIP5ltOdvy8+1zPazOrl2CZ + 6XQov2y/Ow0g+XkoEv8= + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDBzCCAe+gAwIBAgIRAOQLShrbKun+n9ajvOAfQNgwDQYJKoZIhvcNAQELBQAw + HTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMB4XDTI2MDYwMjAwNTI1MloX + DTM2MDUzMDAwNTI1MlowHTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMIIB + IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFGIqusxOcEA+Y9fPM8SlL09 + JhDEgQYf8fpwS4b+EING6cqJ4GBXpymt6Cayae/5CWLDTUg7qiFYiHG0f0isvWm0 + Mi9ny500D//Kc7LbAcw7NcYfTQ+4olrd4cOb8+wc+WnARC8/9ZsFjWO0RxLA1KL4 + I6o8eLTlCmqN3TeBkJSkIYghS8VPcHMW0Bs48mXBtIa1m+gGC5r7HKJKLaR9YlrA + 6xsAjt+O99jKo/sBVJYUT+AydrgZolMDeXp81XjFkAEEO/vrUjxIhlZAfIpDfHDh + RQqXa9RxlHkZEfUny1HpP1iWPL21IWgD+3HH78v38gN4826DRsKCqbAtW6E2zQID + AQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E + FgQUJhYrtyo80hzjngUdByHKT8DoFi8wDQYJKoZIhvcNAQELBQADggEBADOSz7qO + TU2mxc7ce3IaCCWXY053HW25l7YvEfqzXRu0mnWJTKfNHMQw98bjSnohofmQVkbE + hOLof3mWBLPT3ztq9AbQZVeb+xE4JHpotmUIut97NjV37CcR65hX71BFVsvLyWyJ + RCTqDVwV0e+uEnJDS4QEP7rPff4avIKQ4KVoL7BAtZdozkzJBBLWI9oKwoOEnlxQ + qEo9wdSIM7441EUUAQ2kPDnSOt5ZDThOzhALiVvdPyFmAuqYXTv34eKOft+4xJ71 + eyhmBU7OXZPeEcvyJGHp5xEihC2mPYoBWz1U3l0pGzYd8CbMN8HKvp37AUCn6+sq + Mi3lTsmbQ27eaGo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- +ClusterName: k8s1.equivalence.example.internal +ControlPlaneConfig: + KubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: k8s1.equivalence.example.internal + configureCloudRoutes: false + image: registry.k8s.io/kube-controller-manager:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + KubeScheduler: + image: registry.k8s.io/kube-scheduler:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 +EtcdClusterNames: +- main +- events +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "280073174862227056602006568538268639965" + etcd-clients-ca: "118170582503415779531584811913942096504" + etcd-manager-ca-events: "249350469749551963525358799680093431276" + etcd-manager-ca-main: "159135384049592867611523192322834011310" + etcd-peers-ca-events: "41717667896109287431418128439683185489" + etcd-peers-ca-main: "303122601330476694764145455686559416536" + kubernetes-ca: "238614331772899139602871862646837292042" + service-account: "318642634964669623587285433362208799903" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ImageVolume: "true" + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://tests/k8s1.equivalence.example.internal/addons/bootstrap-channel.yaml +configStore: + keypairs: memfs://tests/k8s1.equivalence.example.internal/pki + secrets: memfs://tests/k8s1.equivalence.example.internal/secrets +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +etcdManifests: +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-a.yaml +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-a.yaml +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-b_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-b_content new file mode 100644 index 0000000000000..ecfbf8e821e31 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-b_content @@ -0,0 +1,310 @@ +APIServerConfig: + API: {} + ClusterDNSDomain: cluster.local + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + enableAdmissionPlugins: + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - NodeRestriction + - ResourceQuota + - RuntimeClass + - ServiceAccount + - ValidatingAdmissionPolicy + - ValidatingAdmissionWebhook + enableAggregatorRouting: true + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: registry.k8s.io/kube-apiserver:v1.35.5 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.k8s1.equivalence.example.internal + serviceAccountJWKSURI: https://api.internal.k8s1.equivalence.example.internal/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9CGwPhRDpZpU4g/c33O + ywpM0HeuzLpYNaDuSYPKfUvF/sHdJQehy2B+W300Uwy79GaeGN7/1TIwAkuRTgRl + pq3Zwl9Q3A7uuTylyRjm47JLM2TEClK5u8Nz71qA+0QuhWzx0E1Qc72OqlNsk6Lb + I/7A/UnJCLNM5RacFTeERfKvaNkolS88LyEogmjRVnc14sZ5l7i7ltpLHwVIhPUs + cpTVRaxnuSxBcRoGkn3XDK8AmGhvfiP9Yb5YiZP5IqxdpHiJZkWBrlgc3rJdkef7 + BVXLnsR+KN17mlhbd6MXaocbpQpghJVAMJGKfMYP0J3PfRfHP3tQQvw5AGizOpMR + iwIDAQAB + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + - ab0b3c97a377733946584f86bd191526c267345feead4d8ecefbc701444d3b26@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-amd64 + - c12e9e476c47a180618cd4ee1bd12c86ee890efd8e74b4526b42e783c71d1a9b@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 + - f969b90f83d5a1445e901bdded0b75aa391bbc4d8569278553f2816205c32082@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-arm64 + - c7a38921fa96a8c8ccf798db9793a7de87801cfdfb0edddc324ecf882a4d1bb6@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIDETCCAfmgAwIBAgIRANK0IaP/wDWEXGUkKcVaIt0wDQYJKoZIhvcNAQELBQAw + IjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3ItY2EwHhcNMjYwNjAyMDA1 + MjUyWhcNMzYwNTMwMDA1MjUyWjAiMSAwHgYDVQQDExdhcGlzZXJ2ZXItYWdncmVn + YXRvci1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFUazdUPdsr + Q44pwSB935u0Rq2blh1iRtAleEs4nmfa/BeFAd4+fLJ6ozJOYkAQImYpFzOIj/PX + ragyDo/fRTPm3Qm2Ljn8ipaI46ndlHeUkKzqV/0ewJ+A+33Pdq2pp0maSTC50M5O + zwwkFblKHjk+MldtyUUxWxv8JB+UBC+nAFxFC1HVZeoBhSguhn5hwj8xgRe//ovq + eZN5XunF9x7Na/XOYnq1usF76KZCPAy2XEZqOB41Bhb/7W6BJNOeVkkQF7q9qD3D + lZUdjvgkoeMPjMxjyn6sEF2tgZr9HGSvzexAmtnm/QDDauR6C9LuHKAVsFt9BQt8 + pAu9shLcq7ECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB + Af8wHQYDVR0OBBYEFGptCok5udjNkXjg9hyJ/F9C/X1BMA0GCSqGSIb3DQEBCwUA + A4IBAQCC4P0xJsrUmwCEPRNisayuA8rf7KNc+OyHEYSMtHMQuy3mgtCd3OFvCHX/ + KjQiPeqOSuBSDQuu6N/f63H1nWuIxi1GmeYC+6iR38sBU2dpakF+8vncA2ln59ln + 5YuvzkIj9kkCdvnui1x10cASATJvLpawBgj48KLrayrnfA4LddTtV4flO8dogFZt + GP9c/1xtgagU4NW4VmJnyV970CEzbmY4ZpBw5PK+bR7wTd21jDMRr2cliwQs3t0f + fem5ZlODRE6r/kgGCkMEvqOpa8emLSZp3VzvbQWEFVWKccjA/Z36gHEOZjKTiF9O + gi+gTnPBYZzP+COI40aPW+etJKML + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIDADCCAeigAwIBAgIQWObTivI7jyAX7UCMUztaeDANBgkqhkiG9w0BAQsFADAa + MRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwHhcNMjYwNjAyMDA1MjUyWhcNMzYw + NTMwMDA1MjUyWjAaMRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwggEiMA0GCSqG + SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAPoMFLJT39dwnzqBg/zxKwWwBDV0H7N9H + xoUJvUyT/7E0BxzUKmajQhBkRNvaoebSFOvH8m6IQ4Zkum2EvyH7MNqUy3avt3GM + CTLhtSEiR8RhMsYDJ5CGgDooyPGBOJpITOY/o6Hie6RYk47A6jdes89xHYOr80Mq + gqtqvVkaxQDt/AIhCQWfml1+d4KUAt8qTt9umiwyIgjc30+ujk/dC5/KqNzQIHtA + pib5SkWabGMwKbHwYEB0ngzHO/tveJyKnrva1VUxk+3QuSSdkZbCSe/3hh3tDXJw + BZj5e67I7AzR5ZAVoYYR4Dad6BJkGR14kYdmoF0Lj5DnKSBIg23bAgMBAAGjQjBA + MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSex9K9 + QG60AcGgXHChRPAaqVztkjANBgkqhkiG9w0BAQsFAAOCAQEAokGJK5B8JPyemlbK + Qef+qt7MEACty5u+gTSvi0HOqW1xb24TaHWI3Y46OBaYcyEJSVkzkeWl47ZrhzED + qGab/ASFQJ8lZ2PEV++5ehitiPps6MZbAnUb8YO9BHoxuHUjrRcev7Lyvo17mB62 + slOGCB+GaYXAdj4RCZYOhrCoX26FURyxkSY9/2+qiIPitOUQKJyoNw/e6aDv+FUH + Jy81NzbzbEPjwEYBOWV094TY1zsA9GcXN+N+DuPrTGJhOYAtVuNYXWdedLdCLizx + Dz0aC4qA6W5zBxU2++jdnJQfIrnFq0MF+ugj+FO8lmKEiYYVoLOC921JChEEYfd5 + b2dkAg== + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDDzCCAfegAwIBAgIRALuXJ1SK65cmULxmNsByJewwDQYJKoZIhvcNAQELBQAw + ITEfMB0GA1UEAxMWZXRjZC1tYW5hZ2VyLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUy + NTJaFw0zNjA1MzAwMDUyNTJaMCExHzAdBgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1l + dmVudHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3Ir75KrvZOxap + /yTFuMCeoh9+WjzYh+Ey3EP+fEef2v0vJw89OEPaxcNVjlpY+4KXQerMua2OJHw2 + l3swhWA//91Pg+6wtN74XtC7AckKsN32LaV6xDEtJkY5vhXsuSYRPSQSH95S+po5 + Nzz7RtoFhlVQc9p946QHpCpys3cuu2fKc8XSsbK71EQePIY/x5sqfQ04v8X7PO7t + eFIwz/oRSwnZYOfj5zp9A3pefG595YSUrnjS2CVfiJMnuU+629pRtyQNuM1JlDyB + ymdrYzUD1bWuCn0q0EQ/iWBNO4QJi4nuX6Do6BsFDlLTspfae3F0lnaqKzpsJYDG + TKeYDDKbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ + MB0GA1UdDgQWBBQbhV5ghNsXkBjIcdZqUtvcf7SSMjANBgkqhkiG9w0BAQsFAAOC + AQEAnyqtKzkNLAN7nKnfBDkU1ZmsoLIkV8glzoQx4nA8rFjycvFPElWPu9yUvocO + BZV44PWXmG1oWfExGlW2PjHPhLkzGWMpvhzWXy9kBANOYDvRmI50F3I1oaZpAtF5 + SUAiWq9Mkrafn20aBBL8hNXUUhugrtrYY1cIb99TBrh+4WAGHLqfhw3qXHVhJTzJ + +P0tuatqwJfMWOpL3qBYWAQci5NSZg4VFvQgVCwaD3h+OoT0Z7wZeYytf8tOENZ2 + Erdb7tMDiXSLLHslVybrVhU8cd8ck7QMXhhK/FIOTJKCgJAIsD6lr+/c9hfh8Fpz + r55rasUA3eHKr6ekstPZOl6CEQ== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQd7hcMdMribsvFHJRe0eUrjANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLW1hbmFnZXItY2EtbWFpbjAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWlu + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArINQkwWh9p9BR9z/lDey + LXE8tmxuLTOXwpgzj2L9ynbFmRCF4cN8V3A8CEjy4f7woxM3sy9C5ckYq3thy0qx + F92Idu6Re/48j2hlOV15ZgQtgsgNtYy1g7RG46DDZAfya4RkQEvg/7Duiv0ZNl0N + sYotP7+g+jc4d2Nl22vh9Ml9ro1K06C/GiafK8ZNgVt+4ivV5sGSVYVzlnr6LcoK + tu9BQVIpyNzKOuJ0iUEZhJrtoI9adl+g/asLBiESa+xKNb+liQZ91Cq+/8H014LV + Qm7y9bk3S1TpqYTt2GlA+uJH5cp8eYxw6ZF0iIDhKayPSULvnjPgI1DhYyHaCUeo + FQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUTmUXDqEcpM0KXRNde162bck8FAYwDQYJKoZIhvcNAQELBQADggEBABRt + 1qNQ2KPQ5gGnoATexGg88VCYqJClFlBKcPU+vIom2SUBgrpddHWAHn8ba0ULhSrj + D4BrK997ss+wu9w31YwoGvXKWepyW+iFRcSDfl9WCzCQP0rdvEqrphtkX7L9Rygv + 3pHh7xtyN4hLUfJTvGjndjmcUD8c93Bzhz6269yIec7MHC0eoWipMXa3oyjCROCP + HSXm6ToU6c5ESZf92R+4FZ0DgIGWFNFVIT4+cN/49dGdb/cYPYMd3AfqkRk89r1+ + OIIs98PXiG6z7XNcTOZqzR7C/TbhIT6g+R6r09HdjasNihWVGNDHdAeYpzXYGtgZ + 7io//596d7+67eEXdgA= + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQH2KH1CnrdSogWVMlcYn7UTANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLXBlZXJzLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRz + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnTaTLSjrlVIhPkDzhW3D + wpkJw9bBZgdNSpcJ+3xnkn252W0jvffBxyGjc/KSxiFIzo/mlEmojUcM8sTVlZsk + gnUUzN2hts2Jl3CAENESJsAPk1IdKitEoODcpBJ2w26HlWDxoPPqeP6I5631pRr+ + u6lYDDBQBdMdaF6fnFary7w8t72Xyqbjcg6yAItWglYJvFQkNK+HZ8beamk6LQSC + AHnZH0iORI84z0t1kaqQtCaZqnWVM4CybcnpJ3GUMQss8ASWzKYclxzjjBPS+xxV + Y95ShdlDxo2X04qjNiGL+TXAYMDLVAMLh5n+GqTRxti6myv1GVJy61/XzrJjZJGg + 9wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUvtzj6D31eItxfnXQjJND5ScuAAQwDQYJKoZIhvcNAQELBQADggEBAEid + BZKaX3EdteUwk6Giq6tN3tLG8IOoEHe+Br8/e1Sa4A/1rqqkmTpcwMsByVXEs5W9 + N8QMP7KjJ37D5U1a664Nr7S+1ZyoRgV50N1arOzomgsIHaUchuuWUv/O4tfVwycf + jdBDqmieXUfw2ecpn5uEC3TRVrheVBm+EENOv/XjVflleDFQtLH1r58LVqdBvwkc + rVF8OtewTsBTpjZIOFVkPOn09NsawnCehWNcqzjaWe8cGue8aWfWyNDRpzC6Xj1l + KVoOym4Yy8X+zdshsqesNpEESfMvymyCE8UJdK7qfwIP5ltOdvy8+1zPazOrl2CZ + 6XQov2y/Ow0g+XkoEv8= + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDBzCCAe+gAwIBAgIRAOQLShrbKun+n9ajvOAfQNgwDQYJKoZIhvcNAQELBQAw + HTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMB4XDTI2MDYwMjAwNTI1MloX + DTM2MDUzMDAwNTI1MlowHTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMIIB + IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFGIqusxOcEA+Y9fPM8SlL09 + JhDEgQYf8fpwS4b+EING6cqJ4GBXpymt6Cayae/5CWLDTUg7qiFYiHG0f0isvWm0 + Mi9ny500D//Kc7LbAcw7NcYfTQ+4olrd4cOb8+wc+WnARC8/9ZsFjWO0RxLA1KL4 + I6o8eLTlCmqN3TeBkJSkIYghS8VPcHMW0Bs48mXBtIa1m+gGC5r7HKJKLaR9YlrA + 6xsAjt+O99jKo/sBVJYUT+AydrgZolMDeXp81XjFkAEEO/vrUjxIhlZAfIpDfHDh + RQqXa9RxlHkZEfUny1HpP1iWPL21IWgD+3HH78v38gN4826DRsKCqbAtW6E2zQID + AQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E + FgQUJhYrtyo80hzjngUdByHKT8DoFi8wDQYJKoZIhvcNAQELBQADggEBADOSz7qO + TU2mxc7ce3IaCCWXY053HW25l7YvEfqzXRu0mnWJTKfNHMQw98bjSnohofmQVkbE + hOLof3mWBLPT3ztq9AbQZVeb+xE4JHpotmUIut97NjV37CcR65hX71BFVsvLyWyJ + RCTqDVwV0e+uEnJDS4QEP7rPff4avIKQ4KVoL7BAtZdozkzJBBLWI9oKwoOEnlxQ + qEo9wdSIM7441EUUAQ2kPDnSOt5ZDThOzhALiVvdPyFmAuqYXTv34eKOft+4xJ71 + eyhmBU7OXZPeEcvyJGHp5xEihC2mPYoBWz1U3l0pGzYd8CbMN8HKvp37AUCn6+sq + Mi3lTsmbQ27eaGo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- +ClusterName: k8s1.equivalence.example.internal +ControlPlaneConfig: + KubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: k8s1.equivalence.example.internal + configureCloudRoutes: false + image: registry.k8s.io/kube-controller-manager:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + KubeScheduler: + image: registry.k8s.io/kube-scheduler:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 +EtcdClusterNames: +- main +- events +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "280073174862227056602006568538268639965" + etcd-clients-ca: "118170582503415779531584811913942096504" + etcd-manager-ca-events: "249350469749551963525358799680093431276" + etcd-manager-ca-main: "159135384049592867611523192322834011310" + etcd-peers-ca-events: "41717667896109287431418128439683185489" + etcd-peers-ca-main: "303122601330476694764145455686559416536" + kubernetes-ca: "238614331772899139602871862646837292042" + service-account: "318642634964669623587285433362208799903" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ImageVolume: "true" + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://tests/k8s1.equivalence.example.internal/addons/bootstrap-channel.yaml +configStore: + keypairs: memfs://tests/k8s1.equivalence.example.internal/pki + secrets: memfs://tests/k8s1.equivalence.example.internal/secrets +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +etcdManifests: +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-b.yaml +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-b.yaml +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-c_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-c_content new file mode 100644 index 0000000000000..eb7a431d239a8 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-control-plane-us-west1-c_content @@ -0,0 +1,310 @@ +APIServerConfig: + API: {} + ClusterDNSDomain: cluster.local + KubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiAudiences: + - kubernetes.svc.default + apiServerCount: 3 + authorizationMode: Node,RBAC + bindAddress: 0.0.0.0 + enableAdmissionPlugins: + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - NodeRestriction + - ResourceQuota + - RuntimeClass + - ServiceAccount + - ValidatingAdmissionPolicy + - ValidatingAdmissionWebhook + enableAggregatorRouting: true + etcdServers: + - https://127.0.0.1:4001 + etcdServersOverrides: + - /events#https://127.0.0.1:4002 + image: registry.k8s.io/kube-apiserver:v1.35.5 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.internal.k8s1.equivalence.example.internal + serviceAccountJWKSURI: https://api.internal.k8s1.equivalence.example.internal/openid/v1/jwks + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + ServiceAccountPublicKeys: | + -----BEGIN RSA PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9CGwPhRDpZpU4g/c33O + ywpM0HeuzLpYNaDuSYPKfUvF/sHdJQehy2B+W300Uwy79GaeGN7/1TIwAkuRTgRl + pq3Zwl9Q3A7uuTylyRjm47JLM2TEClK5u8Nz71qA+0QuhWzx0E1Qc72OqlNsk6Lb + I/7A/UnJCLNM5RacFTeERfKvaNkolS88LyEogmjRVnc14sZ5l7i7ltpLHwVIhPUs + cpTVRaxnuSxBcRoGkn3XDK8AmGhvfiP9Yb5YiZP5IqxdpHiJZkWBrlgc3rJdkef7 + BVXLnsR+KN17mlhbd6MXaocbpQpghJVAMJGKfMYP0J3PfRfHP3tQQvw5AGizOpMR + iwIDAQAB + -----END RSA PUBLIC KEY----- +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + - ab0b3c97a377733946584f86bd191526c267345feead4d8ecefbc701444d3b26@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-amd64 + - c12e9e476c47a180618cd4ee1bd12c86ee890efd8e74b4526b42e783c71d1a9b@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 + - f969b90f83d5a1445e901bdded0b75aa391bbc4d8569278553f2816205c32082@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/protokube,https://github.com/kubernetes/kops/releases/download/v1.35.1/protokube-linux-arm64 + - c7a38921fa96a8c8ccf798db9793a7de87801cfdfb0edddc324ecf882a4d1bb6@https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/channels,https://github.com/kubernetes/kops/releases/download/v1.35.1/channels-linux-arm64 +CAs: + apiserver-aggregator-ca: | + -----BEGIN CERTIFICATE----- + MIIDETCCAfmgAwIBAgIRANK0IaP/wDWEXGUkKcVaIt0wDQYJKoZIhvcNAQELBQAw + IjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3ItY2EwHhcNMjYwNjAyMDA1 + MjUyWhcNMzYwNTMwMDA1MjUyWjAiMSAwHgYDVQQDExdhcGlzZXJ2ZXItYWdncmVn + YXRvci1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFUazdUPdsr + Q44pwSB935u0Rq2blh1iRtAleEs4nmfa/BeFAd4+fLJ6ozJOYkAQImYpFzOIj/PX + ragyDo/fRTPm3Qm2Ljn8ipaI46ndlHeUkKzqV/0ewJ+A+33Pdq2pp0maSTC50M5O + zwwkFblKHjk+MldtyUUxWxv8JB+UBC+nAFxFC1HVZeoBhSguhn5hwj8xgRe//ovq + eZN5XunF9x7Na/XOYnq1usF76KZCPAy2XEZqOB41Bhb/7W6BJNOeVkkQF7q9qD3D + lZUdjvgkoeMPjMxjyn6sEF2tgZr9HGSvzexAmtnm/QDDauR6C9LuHKAVsFt9BQt8 + pAu9shLcq7ECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB + Af8wHQYDVR0OBBYEFGptCok5udjNkXjg9hyJ/F9C/X1BMA0GCSqGSIb3DQEBCwUA + A4IBAQCC4P0xJsrUmwCEPRNisayuA8rf7KNc+OyHEYSMtHMQuy3mgtCd3OFvCHX/ + KjQiPeqOSuBSDQuu6N/f63H1nWuIxi1GmeYC+6iR38sBU2dpakF+8vncA2ln59ln + 5YuvzkIj9kkCdvnui1x10cASATJvLpawBgj48KLrayrnfA4LddTtV4flO8dogFZt + GP9c/1xtgagU4NW4VmJnyV970CEzbmY4ZpBw5PK+bR7wTd21jDMRr2cliwQs3t0f + fem5ZlODRE6r/kgGCkMEvqOpa8emLSZp3VzvbQWEFVWKccjA/Z36gHEOZjKTiF9O + gi+gTnPBYZzP+COI40aPW+etJKML + -----END CERTIFICATE----- + etcd-clients-ca: | + -----BEGIN CERTIFICATE----- + MIIDADCCAeigAwIBAgIQWObTivI7jyAX7UCMUztaeDANBgkqhkiG9w0BAQsFADAa + MRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwHhcNMjYwNjAyMDA1MjUyWhcNMzYw + NTMwMDA1MjUyWjAaMRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwggEiMA0GCSqG + SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAPoMFLJT39dwnzqBg/zxKwWwBDV0H7N9H + xoUJvUyT/7E0BxzUKmajQhBkRNvaoebSFOvH8m6IQ4Zkum2EvyH7MNqUy3avt3GM + CTLhtSEiR8RhMsYDJ5CGgDooyPGBOJpITOY/o6Hie6RYk47A6jdes89xHYOr80Mq + gqtqvVkaxQDt/AIhCQWfml1+d4KUAt8qTt9umiwyIgjc30+ujk/dC5/KqNzQIHtA + pib5SkWabGMwKbHwYEB0ngzHO/tveJyKnrva1VUxk+3QuSSdkZbCSe/3hh3tDXJw + BZj5e67I7AzR5ZAVoYYR4Dad6BJkGR14kYdmoF0Lj5DnKSBIg23bAgMBAAGjQjBA + MA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSex9K9 + QG60AcGgXHChRPAaqVztkjANBgkqhkiG9w0BAQsFAAOCAQEAokGJK5B8JPyemlbK + Qef+qt7MEACty5u+gTSvi0HOqW1xb24TaHWI3Y46OBaYcyEJSVkzkeWl47ZrhzED + qGab/ASFQJ8lZ2PEV++5ehitiPps6MZbAnUb8YO9BHoxuHUjrRcev7Lyvo17mB62 + slOGCB+GaYXAdj4RCZYOhrCoX26FURyxkSY9/2+qiIPitOUQKJyoNw/e6aDv+FUH + Jy81NzbzbEPjwEYBOWV094TY1zsA9GcXN+N+DuPrTGJhOYAtVuNYXWdedLdCLizx + Dz0aC4qA6W5zBxU2++jdnJQfIrnFq0MF+ugj+FO8lmKEiYYVoLOC921JChEEYfd5 + b2dkAg== + -----END CERTIFICATE----- + etcd-manager-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDDzCCAfegAwIBAgIRALuXJ1SK65cmULxmNsByJewwDQYJKoZIhvcNAQELBQAw + ITEfMB0GA1UEAxMWZXRjZC1tYW5hZ2VyLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUy + NTJaFw0zNjA1MzAwMDUyNTJaMCExHzAdBgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1l + dmVudHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3Ir75KrvZOxap + /yTFuMCeoh9+WjzYh+Ey3EP+fEef2v0vJw89OEPaxcNVjlpY+4KXQerMua2OJHw2 + l3swhWA//91Pg+6wtN74XtC7AckKsN32LaV6xDEtJkY5vhXsuSYRPSQSH95S+po5 + Nzz7RtoFhlVQc9p946QHpCpys3cuu2fKc8XSsbK71EQePIY/x5sqfQ04v8X7PO7t + eFIwz/oRSwnZYOfj5zp9A3pefG595YSUrnjS2CVfiJMnuU+629pRtyQNuM1JlDyB + ymdrYzUD1bWuCn0q0EQ/iWBNO4QJi4nuX6Do6BsFDlLTspfae3F0lnaqKzpsJYDG + TKeYDDKbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/ + MB0GA1UdDgQWBBQbhV5ghNsXkBjIcdZqUtvcf7SSMjANBgkqhkiG9w0BAQsFAAOC + AQEAnyqtKzkNLAN7nKnfBDkU1ZmsoLIkV8glzoQx4nA8rFjycvFPElWPu9yUvocO + BZV44PWXmG1oWfExGlW2PjHPhLkzGWMpvhzWXy9kBANOYDvRmI50F3I1oaZpAtF5 + SUAiWq9Mkrafn20aBBL8hNXUUhugrtrYY1cIb99TBrh+4WAGHLqfhw3qXHVhJTzJ + +P0tuatqwJfMWOpL3qBYWAQci5NSZg4VFvQgVCwaD3h+OoT0Z7wZeYytf8tOENZ2 + Erdb7tMDiXSLLHslVybrVhU8cd8ck7QMXhhK/FIOTJKCgJAIsD6lr+/c9hfh8Fpz + r55rasUA3eHKr6ekstPZOl6CEQ== + -----END CERTIFICATE----- + etcd-manager-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQd7hcMdMribsvFHJRe0eUrjANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLW1hbmFnZXItY2EtbWFpbjAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWlu + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArINQkwWh9p9BR9z/lDey + LXE8tmxuLTOXwpgzj2L9ynbFmRCF4cN8V3A8CEjy4f7woxM3sy9C5ckYq3thy0qx + F92Idu6Re/48j2hlOV15ZgQtgsgNtYy1g7RG46DDZAfya4RkQEvg/7Duiv0ZNl0N + sYotP7+g+jc4d2Nl22vh9Ml9ro1K06C/GiafK8ZNgVt+4ivV5sGSVYVzlnr6LcoK + tu9BQVIpyNzKOuJ0iUEZhJrtoI9adl+g/asLBiESa+xKNb+liQZ91Cq+/8H014LV + Qm7y9bk3S1TpqYTt2GlA+uJH5cp8eYxw6ZF0iIDhKayPSULvnjPgI1DhYyHaCUeo + FQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUTmUXDqEcpM0KXRNde162bck8FAYwDQYJKoZIhvcNAQELBQADggEBABRt + 1qNQ2KPQ5gGnoATexGg88VCYqJClFlBKcPU+vIom2SUBgrpddHWAHn8ba0ULhSrj + D4BrK997ss+wu9w31YwoGvXKWepyW+iFRcSDfl9WCzCQP0rdvEqrphtkX7L9Rygv + 3pHh7xtyN4hLUfJTvGjndjmcUD8c93Bzhz6269yIec7MHC0eoWipMXa3oyjCROCP + HSXm6ToU6c5ESZf92R+4FZ0DgIGWFNFVIT4+cN/49dGdb/cYPYMd3AfqkRk89r1+ + OIIs98PXiG6z7XNcTOZqzR7C/TbhIT6g+R6r09HdjasNihWVGNDHdAeYpzXYGtgZ + 7io//596d7+67eEXdgA= + -----END CERTIFICATE----- + etcd-peers-ca-events: | + -----BEGIN CERTIFICATE----- + MIIDCjCCAfKgAwIBAgIQH2KH1CnrdSogWVMlcYn7UTANBgkqhkiG9w0BAQsFADAf + MR0wGwYDVQQDExRldGNkLXBlZXJzLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUyNTJa + Fw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRz + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnTaTLSjrlVIhPkDzhW3D + wpkJw9bBZgdNSpcJ+3xnkn252W0jvffBxyGjc/KSxiFIzo/mlEmojUcM8sTVlZsk + gnUUzN2hts2Jl3CAENESJsAPk1IdKitEoODcpBJ2w26HlWDxoPPqeP6I5631pRr+ + u6lYDDBQBdMdaF6fnFary7w8t72Xyqbjcg6yAItWglYJvFQkNK+HZ8beamk6LQSC + AHnZH0iORI84z0t1kaqQtCaZqnWVM4CybcnpJ3GUMQss8ASWzKYclxzjjBPS+xxV + Y95ShdlDxo2X04qjNiGL+TXAYMDLVAMLh5n+GqTRxti6myv1GVJy61/XzrJjZJGg + 9wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV + HQ4EFgQUvtzj6D31eItxfnXQjJND5ScuAAQwDQYJKoZIhvcNAQELBQADggEBAEid + BZKaX3EdteUwk6Giq6tN3tLG8IOoEHe+Br8/e1Sa4A/1rqqkmTpcwMsByVXEs5W9 + N8QMP7KjJ37D5U1a664Nr7S+1ZyoRgV50N1arOzomgsIHaUchuuWUv/O4tfVwycf + jdBDqmieXUfw2ecpn5uEC3TRVrheVBm+EENOv/XjVflleDFQtLH1r58LVqdBvwkc + rVF8OtewTsBTpjZIOFVkPOn09NsawnCehWNcqzjaWe8cGue8aWfWyNDRpzC6Xj1l + KVoOym4Yy8X+zdshsqesNpEESfMvymyCE8UJdK7qfwIP5ltOdvy8+1zPazOrl2CZ + 6XQov2y/Ow0g+XkoEv8= + -----END CERTIFICATE----- + etcd-peers-ca-main: | + -----BEGIN CERTIFICATE----- + MIIDBzCCAe+gAwIBAgIRAOQLShrbKun+n9ajvOAfQNgwDQYJKoZIhvcNAQELBQAw + HTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMB4XDTI2MDYwMjAwNTI1MloX + DTM2MDUzMDAwNTI1MlowHTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMIIB + IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFGIqusxOcEA+Y9fPM8SlL09 + JhDEgQYf8fpwS4b+EING6cqJ4GBXpymt6Cayae/5CWLDTUg7qiFYiHG0f0isvWm0 + Mi9ny500D//Kc7LbAcw7NcYfTQ+4olrd4cOb8+wc+WnARC8/9ZsFjWO0RxLA1KL4 + I6o8eLTlCmqN3TeBkJSkIYghS8VPcHMW0Bs48mXBtIa1m+gGC5r7HKJKLaR9YlrA + 6xsAjt+O99jKo/sBVJYUT+AydrgZolMDeXp81XjFkAEEO/vrUjxIhlZAfIpDfHDh + RQqXa9RxlHkZEfUny1HpP1iWPL21IWgD+3HH78v38gN4826DRsKCqbAtW6E2zQID + AQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E + FgQUJhYrtyo80hzjngUdByHKT8DoFi8wDQYJKoZIhvcNAQELBQADggEBADOSz7qO + TU2mxc7ce3IaCCWXY053HW25l7YvEfqzXRu0mnWJTKfNHMQw98bjSnohofmQVkbE + hOLof3mWBLPT3ztq9AbQZVeb+xE4JHpotmUIut97NjV37CcR65hX71BFVsvLyWyJ + RCTqDVwV0e+uEnJDS4QEP7rPff4avIKQ4KVoL7BAtZdozkzJBBLWI9oKwoOEnlxQ + qEo9wdSIM7441EUUAQ2kPDnSOt5ZDThOzhALiVvdPyFmAuqYXTv34eKOft+4xJ71 + eyhmBU7OXZPeEcvyJGHp5xEihC2mPYoBWz1U3l0pGzYd8CbMN8HKvp37AUCn6+sq + Mi3lTsmbQ27eaGo= + -----END CERTIFICATE----- + kubernetes-ca: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- +ClusterName: k8s1.equivalence.example.internal +ControlPlaneConfig: + KubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: external + clusterCIDR: 100.96.0.0/11 + clusterName: k8s1.equivalence.example.internal + configureCloudRoutes: false + image: registry.k8s.io/kube-controller-manager:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + KubeScheduler: + image: registry.k8s.io/kube-scheduler:v1.35.5 + leaderElection: + leaderElect: true + logLevel: 2 +EtcdClusterNames: +- main +- events +FileAssets: +- content: | + apiVersion: kubescheduler.config.k8s.io/v1 + clientConnection: + kubeconfig: /var/lib/kube-scheduler/kubeconfig + kind: KubeSchedulerConfiguration + path: /var/lib/kube-scheduler/config.yaml +Hooks: +- null +- null +KeypairIDs: + apiserver-aggregator-ca: "280073174862227056602006568538268639965" + etcd-clients-ca: "118170582503415779531584811913942096504" + etcd-manager-ca-events: "249350469749551963525358799680093431276" + etcd-manager-ca-main: "159135384049592867611523192322834011310" + etcd-peers-ca-events: "41717667896109287431418128439683185489" + etcd-peers-ca-main: "303122601330476694764145455686559416536" + kubernetes-ca: "238614331772899139602871862646837292042" + service-account: "318642634964669623587285433362208799903" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ImageVolume: "true" + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kops.k8s.io/kops-controller-pki: "" + node-role.kubernetes.io/control-plane: "" + node.kubernetes.io/exclude-from-external-load-balancers: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s + taints: + - node-role.kubernetes.io/control-plane=:NoSchedule +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +channels: +- memfs://tests/k8s1.equivalence.example.internal/addons/bootstrap-channel.yaml +configStore: + keypairs: memfs://tests/k8s1.equivalence.example.internal/pki + secrets: memfs://tests/k8s1.equivalence.example.internal/secrets +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +etcdManifests: +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-c.yaml +- memfs://tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-c.yaml +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +staticManifests: +- key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-a_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-a_content new file mode 100644 index 0000000000000..2926586ed8bad --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-a_content @@ -0,0 +1,57 @@ +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 +CAs: {} +ClusterName: k8s1.equivalence.example.internal +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "238614331772899139602871862646837292042" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + node-role.kubernetes.io/node: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-b_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-b_content new file mode 100644 index 0000000000000..2926586ed8bad --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-b_content @@ -0,0 +1,57 @@ +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 +CAs: {} +ClusterName: k8s1.equivalence.example.internal +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "238614331772899139602871862646837292042" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + node-role.kubernetes.io/node: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-c_content b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-c_content new file mode 100644 index 0000000000000..2926586ed8bad --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/aws_s3_object_nodeupconfig-nodes-us-west1-c_content @@ -0,0 +1,57 @@ +Assets: + amd64: + - 4ba1f892bd52fdae7cd2abb1b1111054ff274c71f88af0a85fd9f1107e8d46c8@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubelet + - 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/kubectl + - 3a3cb26c58744e036c9e82acfecbaf283177184573134f11eeb83210464bc66c@https://dl.k8s.io/release/v1.35.5/bin/linux/amd64/mounter + - b5852509ab5c950485794afd49a3fd1c3d6166db988c15b683d2373b7055300f@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/amd64/auth-provider-gcp + - 62e77f6294e432dca5b56ad7e7d6085b5bbb526ebba0a51d832714f9b04cfdfd@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-amd64.tar.gz + - 66fa8390be8fb3b23dfbb60c767368bb5b51f1acfa88692bbff1a82953d4d9e9@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.amd64 + arm64: + - a100cfe89d7dd7a49a07164c6ccb78df02563483ef5b37330e1150e37e8868b3@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubelet + - ac69e06fd6860d69786692f5af1c3a1208ed54f8366a4d97ab15c172e99765ee@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/kubectl + - b576052af9da1eb95502ae92c1b6a1f603839cac317999c884dde9331b5b8f94@https://dl.k8s.io/release/v1.35.5/bin/linux/arm64/mounter + - 510aee27ef32a27b76e461e9084719365210d56166057b37db290a72dfecbffa@https://artifacts.k8s.io/binaries/cloud-provider-gcp/v35.0.0/auth-provider-gcp/linux/arm64/auth-provider-gcp + - d0897c8e27c96a7b7d4c73e9b278ea9f559dda619d497d88b323a528bd1412c3@https://github.com/containerd/containerd/releases/download/v2.2.4/containerd-2.2.4-linux-arm64.tar.gz + - bd843d75a788e612c9df286b1fa519a44fcbb7a7b8d01e2268431433cc7c718c@https://github.com/opencontainers/runc/releases/download/v1.3.5/runc.arm64 +CAs: {} +ClusterName: k8s1.equivalence.example.internal +Hooks: +- null +- null +KeypairIDs: + kubernetes-ca: "238614331772899139602871862646837292042" +KubeProxy: null +KubeletConfig: + anonymousAuth: false + cgroupDriver: systemd + cgroupRoot: / + cloudProvider: external + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hairpinMode: promiscuous-bridge + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + node-role.kubernetes.io/node: "" + podManifestPath: /etc/kubernetes/manifests + protectKernelDefaults: true + shutdownGracePeriod: 30s + shutdownGracePeriodCriticalPods: 10s +KubernetesVersion: 1.35.5 +Networking: + cilium: {} + nonMasqueradeCIDR: 100.64.0.0/10 + serviceClusterIPRange: 100.64.0.0/13 +UpdatePolicy: automatic +containerdConfig: + logLevel: info + runc: + version: 1.3.5 + sandboxImage: registry.k8s.io/pause:3.10.1 + version: 2.2.4 +multizone: true +nodeTags: k8s1-equivalence-example-internal-k8s-io-role-node +usesLegacyGossip: false +usesNoneDNS: true diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..4ea5e4734d8f9 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,133 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigBase: memfs://tests/k8s1.equivalence.example.internal +InstanceGroupName: control-plane-us-west1-a +InstanceGroupRole: ControlPlane +NodeupConfigHash: WCtyNLbWeYMF3qLfTJ4S3fkyCuoC1guak2BeAsnZTF4= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..348e97fd6d868 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,133 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigBase: memfs://tests/k8s1.equivalence.example.internal +InstanceGroupName: control-plane-us-west1-b +InstanceGroupRole: ControlPlane +NodeupConfigHash: Mc8WRghZsP8XfhXGjQdij0dg4kxnJHSqZC1ERctYC9M= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..602e408028509 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_control-plane-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,133 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigBase: memfs://tests/k8s1.equivalence.example.internal +InstanceGroupName: control-plane-us-west1-c +InstanceGroupRole: ControlPlane +NodeupConfigHash: nZS1Timw6MYebJcOHvcSaFZqTsKqB90VT5HsZ8U7oks= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..1061858b82064 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,155 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.k8s1.equivalence.example.internal:3988/ +InstanceGroupName: nodes-us-west1-a +InstanceGroupRole: Node +NodeupConfigHash: 3ajQo0hnMHkcQF3lYRv7SFX1IKzPrLYaFHZNSQJIZZU= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..69ecb00224156 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,155 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.k8s1.equivalence.example.internal:3988/ +InstanceGroupName: nodes-us-west1-b +InstanceGroupRole: Node +NodeupConfigHash: 3ajQo0hnMHkcQF3lYRv7SFX1IKzPrLYaFHZNSQJIZZU= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data new file mode 100644 index 0000000000000..21db62abf934b --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/data/google_compute_instance_template_nodes-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data @@ -0,0 +1,155 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-amd64 +NODEUP_HASH_AMD64=b6f5dded0fabdda70d1939911a802dd52ef38e221b24b478f9fd78920223a891 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.35.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.35.1/nodeup-linux-arm64 +NODEUP_HASH_ARM64=96097ccc589a9c5dfeddb1bbe75d54194cccf2b50674025297f9177bcb52b1d3 + + + + + +sysctl -w net.core.rmem_max=16777216 || true +sysctl -w net.core.wmem_max=16777216 || true +sysctl -w net.ipv4.tcp_rmem='4096 87380 16777216' || true +sysctl -w net.ipv4.tcp_wmem='4096 87380 16777216' || true + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, urls +download-or-bust() { + echo "== Downloading $1 with hash $2 from $3 ==" + local -r file="$1" + local -r hash="$2" + local -a urls + IFS=, read -r -a urls <<< "$3" + + if [[ -f "${file}" ]]; then + if ! validate-hash "${file}" "${hash}"; then + rm -f "${file}" + else + return 0 + fi + fi + + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10" + "curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10" + "wget -O ${file} --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "== Downloading ${url} using ${cmd} ==" + if ! (${cmd} "${url}"); then + echo "== Failed to download ${url} using ${cmd} ==" + continue + fi + if ! validate-hash "${file}" "${hash}"; then + echo "== Failed to validate hash for ${url} ==" + rm -f "${file}" + else + echo "== Downloaded ${url} with hash ${hash} ==" + return 0 + fi + done + done + + echo "== All downloads failed; sleeping before retrying ==" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum "${file}" | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + cd ${INSTALL_DIR}/bin + download-or-bust nodeup "${NODEUP_HASH}" "${NODEUP_URL}" + + chmod +x nodeup + + echo "== Running nodeup ==" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured ==" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +CloudProvider: gce +ClusterName: k8s1.equivalence.example.internal +ConfigServer: + CACertificates: | + -----BEGIN CERTIFICATE----- + MIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw + GDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1 + MzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3 + DQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa + Lhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw + 7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L + Cfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D + nEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx + ybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G + A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr + ZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU + sayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe + QAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px + KBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd + ZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt + IsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG + bw== + -----END CERTIFICATE----- + servers: + - https://kops-controller.internal.k8s1.equivalence.example.internal:3988/ +InstanceGroupName: nodes-us-west1-c +InstanceGroupRole: Node +NodeupConfigHash: 3ajQo0hnMHkcQF3lYRv7SFX1IKzPrLYaFHZNSQJIZZU= + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/golden-pki.json b/tests/integration/update_cluster/gce-kops-module-equivalence/golden-pki.json new file mode 100644 index 0000000000000..51cc7c249badc --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/golden-pki.json @@ -0,0 +1,33 @@ +{ + "ca_certificates_pem": { + "apiserver-aggregator-ca": "-----BEGIN CERTIFICATE-----\nMIIDETCCAfmgAwIBAgIRANK0IaP/wDWEXGUkKcVaIt0wDQYJKoZIhvcNAQELBQAw\nIjEgMB4GA1UEAxMXYXBpc2VydmVyLWFnZ3JlZ2F0b3ItY2EwHhcNMjYwNjAyMDA1\nMjUyWhcNMzYwNTMwMDA1MjUyWjAiMSAwHgYDVQQDExdhcGlzZXJ2ZXItYWdncmVn\nYXRvci1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALFUazdUPdsr\nQ44pwSB935u0Rq2blh1iRtAleEs4nmfa/BeFAd4+fLJ6ozJOYkAQImYpFzOIj/PX\nragyDo/fRTPm3Qm2Ljn8ipaI46ndlHeUkKzqV/0ewJ+A+33Pdq2pp0maSTC50M5O\nzwwkFblKHjk+MldtyUUxWxv8JB+UBC+nAFxFC1HVZeoBhSguhn5hwj8xgRe//ovq\neZN5XunF9x7Na/XOYnq1usF76KZCPAy2XEZqOB41Bhb/7W6BJNOeVkkQF7q9qD3D\nlZUdjvgkoeMPjMxjyn6sEF2tgZr9HGSvzexAmtnm/QDDauR6C9LuHKAVsFt9BQt8\npAu9shLcq7ECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB\nAf8wHQYDVR0OBBYEFGptCok5udjNkXjg9hyJ/F9C/X1BMA0GCSqGSIb3DQEBCwUA\nA4IBAQCC4P0xJsrUmwCEPRNisayuA8rf7KNc+OyHEYSMtHMQuy3mgtCd3OFvCHX/\nKjQiPeqOSuBSDQuu6N/f63H1nWuIxi1GmeYC+6iR38sBU2dpakF+8vncA2ln59ln\n5YuvzkIj9kkCdvnui1x10cASATJvLpawBgj48KLrayrnfA4LddTtV4flO8dogFZt\nGP9c/1xtgagU4NW4VmJnyV970CEzbmY4ZpBw5PK+bR7wTd21jDMRr2cliwQs3t0f\nfem5ZlODRE6r/kgGCkMEvqOpa8emLSZp3VzvbQWEFVWKccjA/Z36gHEOZjKTiF9O\ngi+gTnPBYZzP+COI40aPW+etJKML\n-----END CERTIFICATE-----\n", + "etcd-clients-ca": "-----BEGIN CERTIFICATE-----\nMIIDADCCAeigAwIBAgIQWObTivI7jyAX7UCMUztaeDANBgkqhkiG9w0BAQsFADAa\nMRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwHhcNMjYwNjAyMDA1MjUyWhcNMzYw\nNTMwMDA1MjUyWjAaMRgwFgYDVQQDEw9ldGNkLWNsaWVudHMtY2EwggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDAPoMFLJT39dwnzqBg/zxKwWwBDV0H7N9H\nxoUJvUyT/7E0BxzUKmajQhBkRNvaoebSFOvH8m6IQ4Zkum2EvyH7MNqUy3avt3GM\nCTLhtSEiR8RhMsYDJ5CGgDooyPGBOJpITOY/o6Hie6RYk47A6jdes89xHYOr80Mq\ngqtqvVkaxQDt/AIhCQWfml1+d4KUAt8qTt9umiwyIgjc30+ujk/dC5/KqNzQIHtA\npib5SkWabGMwKbHwYEB0ngzHO/tveJyKnrva1VUxk+3QuSSdkZbCSe/3hh3tDXJw\nBZj5e67I7AzR5ZAVoYYR4Dad6BJkGR14kYdmoF0Lj5DnKSBIg23bAgMBAAGjQjBA\nMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSex9K9\nQG60AcGgXHChRPAaqVztkjANBgkqhkiG9w0BAQsFAAOCAQEAokGJK5B8JPyemlbK\nQef+qt7MEACty5u+gTSvi0HOqW1xb24TaHWI3Y46OBaYcyEJSVkzkeWl47ZrhzED\nqGab/ASFQJ8lZ2PEV++5ehitiPps6MZbAnUb8YO9BHoxuHUjrRcev7Lyvo17mB62\nslOGCB+GaYXAdj4RCZYOhrCoX26FURyxkSY9/2+qiIPitOUQKJyoNw/e6aDv+FUH\nJy81NzbzbEPjwEYBOWV094TY1zsA9GcXN+N+DuPrTGJhOYAtVuNYXWdedLdCLizx\nDz0aC4qA6W5zBxU2++jdnJQfIrnFq0MF+ugj+FO8lmKEiYYVoLOC921JChEEYfd5\nb2dkAg==\n-----END CERTIFICATE-----\n", + "etcd-manager-ca-events": "-----BEGIN CERTIFICATE-----\nMIIDDzCCAfegAwIBAgIRALuXJ1SK65cmULxmNsByJewwDQYJKoZIhvcNAQELBQAw\nITEfMB0GA1UEAxMWZXRjZC1tYW5hZ2VyLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUy\nNTJaFw0zNjA1MzAwMDUyNTJaMCExHzAdBgNVBAMTFmV0Y2QtbWFuYWdlci1jYS1l\ndmVudHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3Ir75KrvZOxap\n/yTFuMCeoh9+WjzYh+Ey3EP+fEef2v0vJw89OEPaxcNVjlpY+4KXQerMua2OJHw2\nl3swhWA//91Pg+6wtN74XtC7AckKsN32LaV6xDEtJkY5vhXsuSYRPSQSH95S+po5\nNzz7RtoFhlVQc9p946QHpCpys3cuu2fKc8XSsbK71EQePIY/x5sqfQ04v8X7PO7t\neFIwz/oRSwnZYOfj5zp9A3pefG595YSUrnjS2CVfiJMnuU+629pRtyQNuM1JlDyB\nymdrYzUD1bWuCn0q0EQ/iWBNO4QJi4nuX6Do6BsFDlLTspfae3F0lnaqKzpsJYDG\nTKeYDDKbAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/\nMB0GA1UdDgQWBBQbhV5ghNsXkBjIcdZqUtvcf7SSMjANBgkqhkiG9w0BAQsFAAOC\nAQEAnyqtKzkNLAN7nKnfBDkU1ZmsoLIkV8glzoQx4nA8rFjycvFPElWPu9yUvocO\nBZV44PWXmG1oWfExGlW2PjHPhLkzGWMpvhzWXy9kBANOYDvRmI50F3I1oaZpAtF5\nSUAiWq9Mkrafn20aBBL8hNXUUhugrtrYY1cIb99TBrh+4WAGHLqfhw3qXHVhJTzJ\n+P0tuatqwJfMWOpL3qBYWAQci5NSZg4VFvQgVCwaD3h+OoT0Z7wZeYytf8tOENZ2\nErdb7tMDiXSLLHslVybrVhU8cd8ck7QMXhhK/FIOTJKCgJAIsD6lr+/c9hfh8Fpz\nr55rasUA3eHKr6ekstPZOl6CEQ==\n-----END CERTIFICATE-----\n", + "etcd-manager-ca-main": "-----BEGIN CERTIFICATE-----\nMIIDCjCCAfKgAwIBAgIQd7hcMdMribsvFHJRe0eUrjANBgkqhkiG9w0BAQsFADAf\nMR0wGwYDVQQDExRldGNkLW1hbmFnZXItY2EtbWFpbjAeFw0yNjA2MDIwMDUyNTJa\nFw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtbWFuYWdlci1jYS1tYWlu\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArINQkwWh9p9BR9z/lDey\nLXE8tmxuLTOXwpgzj2L9ynbFmRCF4cN8V3A8CEjy4f7woxM3sy9C5ckYq3thy0qx\nF92Idu6Re/48j2hlOV15ZgQtgsgNtYy1g7RG46DDZAfya4RkQEvg/7Duiv0ZNl0N\nsYotP7+g+jc4d2Nl22vh9Ml9ro1K06C/GiafK8ZNgVt+4ivV5sGSVYVzlnr6LcoK\ntu9BQVIpyNzKOuJ0iUEZhJrtoI9adl+g/asLBiESa+xKNb+liQZ91Cq+/8H014LV\nQm7y9bk3S1TpqYTt2GlA+uJH5cp8eYxw6ZF0iIDhKayPSULvnjPgI1DhYyHaCUeo\nFQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV\nHQ4EFgQUTmUXDqEcpM0KXRNde162bck8FAYwDQYJKoZIhvcNAQELBQADggEBABRt\n1qNQ2KPQ5gGnoATexGg88VCYqJClFlBKcPU+vIom2SUBgrpddHWAHn8ba0ULhSrj\nD4BrK997ss+wu9w31YwoGvXKWepyW+iFRcSDfl9WCzCQP0rdvEqrphtkX7L9Rygv\n3pHh7xtyN4hLUfJTvGjndjmcUD8c93Bzhz6269yIec7MHC0eoWipMXa3oyjCROCP\nHSXm6ToU6c5ESZf92R+4FZ0DgIGWFNFVIT4+cN/49dGdb/cYPYMd3AfqkRk89r1+\nOIIs98PXiG6z7XNcTOZqzR7C/TbhIT6g+R6r09HdjasNihWVGNDHdAeYpzXYGtgZ\n7io//596d7+67eEXdgA=\n-----END CERTIFICATE-----\n", + "etcd-peers-ca-events": "-----BEGIN CERTIFICATE-----\nMIIDCjCCAfKgAwIBAgIQH2KH1CnrdSogWVMlcYn7UTANBgkqhkiG9w0BAQsFADAf\nMR0wGwYDVQQDExRldGNkLXBlZXJzLWNhLWV2ZW50czAeFw0yNjA2MDIwMDUyNTJa\nFw0zNjA1MzAwMDUyNTJaMB8xHTAbBgNVBAMTFGV0Y2QtcGVlcnMtY2EtZXZlbnRz\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnTaTLSjrlVIhPkDzhW3D\nwpkJw9bBZgdNSpcJ+3xnkn252W0jvffBxyGjc/KSxiFIzo/mlEmojUcM8sTVlZsk\ngnUUzN2hts2Jl3CAENESJsAPk1IdKitEoODcpBJ2w26HlWDxoPPqeP6I5631pRr+\nu6lYDDBQBdMdaF6fnFary7w8t72Xyqbjcg6yAItWglYJvFQkNK+HZ8beamk6LQSC\nAHnZH0iORI84z0t1kaqQtCaZqnWVM4CybcnpJ3GUMQss8ASWzKYclxzjjBPS+xxV\nY95ShdlDxo2X04qjNiGL+TXAYMDLVAMLh5n+GqTRxti6myv1GVJy61/XzrJjZJGg\n9wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV\nHQ4EFgQUvtzj6D31eItxfnXQjJND5ScuAAQwDQYJKoZIhvcNAQELBQADggEBAEid\nBZKaX3EdteUwk6Giq6tN3tLG8IOoEHe+Br8/e1Sa4A/1rqqkmTpcwMsByVXEs5W9\nN8QMP7KjJ37D5U1a664Nr7S+1ZyoRgV50N1arOzomgsIHaUchuuWUv/O4tfVwycf\njdBDqmieXUfw2ecpn5uEC3TRVrheVBm+EENOv/XjVflleDFQtLH1r58LVqdBvwkc\nrVF8OtewTsBTpjZIOFVkPOn09NsawnCehWNcqzjaWe8cGue8aWfWyNDRpzC6Xj1l\nKVoOym4Yy8X+zdshsqesNpEESfMvymyCE8UJdK7qfwIP5ltOdvy8+1zPazOrl2CZ\n6XQov2y/Ow0g+XkoEv8=\n-----END CERTIFICATE-----\n", + "etcd-peers-ca-main": "-----BEGIN CERTIFICATE-----\nMIIDBzCCAe+gAwIBAgIRAOQLShrbKun+n9ajvOAfQNgwDQYJKoZIhvcNAQELBQAw\nHTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMB4XDTI2MDYwMjAwNTI1MloX\nDTM2MDUzMDAwNTI1MlowHTEbMBkGA1UEAxMSZXRjZC1wZWVycy1jYS1tYWluMIIB\nIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzFGIqusxOcEA+Y9fPM8SlL09\nJhDEgQYf8fpwS4b+EING6cqJ4GBXpymt6Cayae/5CWLDTUg7qiFYiHG0f0isvWm0\nMi9ny500D//Kc7LbAcw7NcYfTQ+4olrd4cOb8+wc+WnARC8/9ZsFjWO0RxLA1KL4\nI6o8eLTlCmqN3TeBkJSkIYghS8VPcHMW0Bs48mXBtIa1m+gGC5r7HKJKLaR9YlrA\n6xsAjt+O99jKo/sBVJYUT+AydrgZolMDeXp81XjFkAEEO/vrUjxIhlZAfIpDfHDh\nRQqXa9RxlHkZEfUny1HpP1iWPL21IWgD+3HH78v38gN4826DRsKCqbAtW6E2zQID\nAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E\nFgQUJhYrtyo80hzjngUdByHKT8DoFi8wDQYJKoZIhvcNAQELBQADggEBADOSz7qO\nTU2mxc7ce3IaCCWXY053HW25l7YvEfqzXRu0mnWJTKfNHMQw98bjSnohofmQVkbE\nhOLof3mWBLPT3ztq9AbQZVeb+xE4JHpotmUIut97NjV37CcR65hX71BFVsvLyWyJ\nRCTqDVwV0e+uEnJDS4QEP7rPff4avIKQ4KVoL7BAtZdozkzJBBLWI9oKwoOEnlxQ\nqEo9wdSIM7441EUUAQ2kPDnSOt5ZDThOzhALiVvdPyFmAuqYXTv34eKOft+4xJ71\neyhmBU7OXZPeEcvyJGHp5xEihC2mPYoBWz1U3l0pGzYd8CbMN8HKvp37AUCn6+sq\nMi3lTsmbQ27eaGo=\n-----END CERTIFICATE-----\n", + "kubernetes-ca": "-----BEGIN CERTIFICATE-----\nMIIC/TCCAeWgAwIBAgIRALODctwtXhWbZEKuQYKzLAowDQYJKoZIhvcNAQELBQAw\nGDEWMBQGA1UEAxMNa3ViZXJuZXRlcy1jYTAeFw0yNjA2MDIwMDUyNTJaFw0zNjA1\nMzAwMDUyNTJaMBgxFjAUBgNVBAMTDWt1YmVybmV0ZXMtY2EwggEiMA0GCSqGSIb3\nDQEBAQUAA4IBDwAwggEKAoIBAQCineuw1m/n8m3VtgB6tgNksNqETzNGwKNNSjTa\nLhbndl/kkQppaZyDdlUtEqo+a/m3tegceB3Dc6SNpl94zjDyspWmVVE7QfBjQSVw\n7tlA9fc3ptzA2TMkaw/eOVL9UZdVrUWRM2vmDF3i3jmPA9LzXdV/nbV5AX+CJd5L\nCfjaf+/oeU7lsF0/L5u5EunalGk7HpWHOsx9FmVNsHpgODRLwHzcQ6Nx8WMpd26D\nnEI4ftM8ivxKbZerdgXP6pj+A8HzyPWeUZJskQk3PsrxfcSFt/c7ja2ov8pOMHrx\nybvhFMua8WysZH6mj9z7+ynZk6lLo8DZ/66i+T/EYcqPdcVBAgMBAAGjQjBAMA4G\nA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSOOR6BFiYr\nZQr0/oIORH1cykiedDANBgkqhkiG9w0BAQsFAAOCAQEAVPkiUhh5qCeMOiDeTRUU\nsayl6lMXPV6ZOv+fqXscKfxfqOMG2N7lZo3oIn0UQAtFsgHleKasMDsoIOAKOnJe\nQAmOIHAHSkdEL2b3OQoPF1OvP+Mpit2U1WaFvQfkb43AZmzfb73D9CQGEexTT/px\nKBMjleg4QviGHlcs2B0OCQMWoRX10cEdEArExyjz/JhOY8EXpxNChZbW//XQrWZd\nZujeTRJi1IJmq/CZemzFkHdoe0GzyIjcmLfASjFHjqU07U3zVyp65BvSiqzw2uQt\nIsEENsL6WHZDqutbYeUcQrTeUoEXfgT3+zYUG+SMtfqyFbvq0mFgCSDxIl2/hJHG\nbw==\n-----END CERTIFICATE-----\n", + "service-account": "-----BEGIN CERTIFICATE-----\nMIIDATCCAemgAwIBAgIRAO+4VtddIjaQvwJsCOgsPJ8wDQYJKoZIhvcNAQELBQAw\nGjEYMBYGA1UEAwwPc2VydmljZS1hY2NvdW50MB4XDTI2MDYwMjAwNTI1MloXDTM2\nMDUzMDAwNTI1MlowGjEYMBYGA1UEAwwPc2VydmljZS1hY2NvdW50MIIBIjANBgkq\nhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9CGwPhRDpZpU4g/c33OywpM0HeuzLpY\nNaDuSYPKfUvF/sHdJQehy2B+W300Uwy79GaeGN7/1TIwAkuRTgRlpq3Zwl9Q3A7u\nuTylyRjm47JLM2TEClK5u8Nz71qA+0QuhWzx0E1Qc72OqlNsk6LbI/7A/UnJCLNM\n5RacFTeERfKvaNkolS88LyEogmjRVnc14sZ5l7i7ltpLHwVIhPUscpTVRaxnuSxB\ncRoGkn3XDK8AmGhvfiP9Yb5YiZP5IqxdpHiJZkWBrlgc3rJdkef7BVXLnsR+KN17\nmlhbd6MXaocbpQpghJVAMJGKfMYP0J3PfRfHP3tQQvw5AGizOpMRiwIDAQABo0Iw\nQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUOlks\nQcaPHYiRM6hMIBZ0hZYfVZUwDQYJKoZIhvcNAQELBQADggEBACE7L0Koc8lcA6k5\nfMz1e2UJPvE+9E7KU6Q25HOdNYpatFjNeY6zbSDvXTA8U74j88WMdAiLmLB+ctKa\nk599LLZ4eUneEHv9L+vy9+9f8spQlyLHRnFVpFvBlJJTdKJadqTz5HhvYwAAVfIP\nHirz85rHhq3GQCQ0EvKTJrl83wth0dPhuyHwXFS2UI5zBiczCQZIvkk4tC+Cxz1Z\nGpZBusfgBTq8OK7PHRfx3e/P7UciRqsM75Ch82eWpRpCJczO6nDoFkoDz0VvnLpB\nBbBQeFw5K9aQXDX+KR22sdlrhE0h9hKiDVwMdE+/x17L7TpdfgKdlsne6KWWY4Ki\nbWp+N7M=\n-----END CERTIFICATE-----\n" + }, + "keypair_ids": { + "apiserver-aggregator-ca": "280073174862227056602006568538268639965", + "etcd-clients-ca": "118170582503415779531584811913942096504", + "etcd-manager-ca-events": "249350469749551963525358799680093431276", + "etcd-manager-ca-main": "159135384049592867611523192322834011310", + "etcd-peers-ca-events": "41717667896109287431418128439683185489", + "etcd-peers-ca-main": "303122601330476694764145455686559416536", + "kubernetes-ca": "238614331772899139602871862646837292042", + "service-account": "318642634964669623587285433362208799903" + }, + "service_account_public_keys": "-----BEGIN RSA PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAr9CGwPhRDpZpU4g/c33O\nywpM0HeuzLpYNaDuSYPKfUvF/sHdJQehy2B+W300Uwy79GaeGN7/1TIwAkuRTgRl\npq3Zwl9Q3A7uuTylyRjm47JLM2TEClK5u8Nz71qA+0QuhWzx0E1Qc72OqlNsk6Lb\nI/7A/UnJCLNM5RacFTeERfKvaNkolS88LyEogmjRVnc14sZ5l7i7ltpLHwVIhPUs\ncpTVRaxnuSxBcRoGkn3XDK8AmGhvfiP9Yb5YiZP5IqxdpHiJZkWBrlgc3rJdkef7\nBVXLnsR+KN17mlhbd6MXaocbpQpghJVAMJGKfMYP0J3PfRfHP3tQQvw5AGizOpMR\niwIDAQAB\n-----END RSA PUBLIC KEY-----\n", + "private_keys_pem": { + "apiserver-aggregator-ca": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDTcR4z4y7dG7Nn\nqQ6thuxjHmotXs/cw6kwZitp+d0yj+8kIuC2bkL32eLfAOV8fk4K5FdN3tjFP88G\ntK04u8dWSe6OInBMPkpu8RenPtjJYg8qaS9sZRkKC4zJFi5tUOs+aZo2AIfXlTBV\nGISmiq5Yk/Hlic1O22j+UMcLU6LnUQkTMxkuCN79SrVRRui9UaZdwHrc3kx9OgS4\nEOcq/qUBJ8yPFro4b0SilNoRa4afxFq0ft+gv583BUR/y4vQzygM1D022UXnzI4j\nP4ubjps15cEDikVsqewEbYmXXV7vLPLwcvkYdG++ejwvEbh7B4D1sXaOyIFzyS0N\nIDD0WCDnAgMBAAECggEACPN9Z/pbtxqR0i4J2uQxNomRzNoxHIopGZZuQhKykiBn\nNFDa0aLdnzHrvhsVLklQ158zVdNPKIlo2rA/s1hw3pxwD+5WKLQi82aJLfPKcV9/\nNkqQPK3KaJMrveSzNr5eXfgtpqvjXXV3P6CzYFfS6fTF/BgXNRnX7HaI+Oo17o97\nPub9NT6d/tC+To8vGPDh1pQsCKEqbcgsMwdaKofBl4sn8oiNkPIsEaPgP6v2mog8\nbTYsRIPN00Ee7OtMm84INpqCLoGsQLSKyseqvvTyxnJwGY02J7GwzmxTt8iI0RWw\nEhtHqaWKRKvMOkNaIEBEsnphX7X03lSSlq4kQyvysQKBgQDwvNm0i1ca+R7s7lO2\n3NQICGRR/biP6ESUoBbGtAdqC5eDSOjrqROPpoer9tY2E+LnNVoUJqrg6agMAn6/\n1aWeP4lYseeQI+oAr5pwpPu6jdMenBfoLLbxibODorEzSRU1nd9PAJ/dHHhXl5YF\npj0L4EH7MO/Ou1ezhrs397j73wKBgQDg2MyQVlzXP3Wq5c+s+0pxBneWN94aNzVY\nEkVxFMG6YnDKI8tvwFofsZ66fXp9s073vvdrR6r+2N69XpsL/pwE1UeoF+taMnwB\nqE16a2d4CYSXdt3p6ACkIR50P8+Pe103+FRcfI8Fc5a/vleBaRLr4vgYuF+mlIJQ\nqj82bHl7+QKBgCVm0dQRCflw1vMsm+j/KhoNY7ixWKlTBsSokqFQew5ouqN1AmsK\nm7VnxBBdiy5frJMLYPFciJPk+WzUPmkDgnWxnV8gVL+6NLLilXYGy+1R94j762Lw\n/4NLR46eCjXTlu37lyQpDq1FrRS+wloiV72um9Gw6rENvjRwLEonvJ+1AoGACUbo\nqcvmaLslj7eLqgd0OtWZeYk0kMBWYxvd4ZyJobcOEEkJy8KT9VrOwcMLmsQbLf2j\nF7oNRBaOmiGdJBGNPmXOXxEdlMsgcrEPAU4NJta9scIlxdDctejn6MP5PGJKplSI\npog9Hmx2e9LOqXg8X58tvVdeGvFlrw5XLVesa4ECgYA8PkZEYcXuF+4uhWLjo2VU\nXUzFVcKQv8/qkwyLiEqvvHkQHqftoFiAbriB3Jka6YaLjjU+CHtP+O9qBhKHS0lB\noSWYTOcGPEKsbwB6x+zZuYxubDo7onAbC9pXscZatc5cMrYs50zWTxYcu2VJZCRV\nrqwj4IeJBjMydIYOQztOhQ==\n-----END PRIVATE KEY-----\n", + "etcd-clients-ca": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCZewVQwEQP2oIS\n2ARXjtmKWVrpTnJv+umVWsYrcyFGK+kO1m7diyV/fAW5Dr1JpLJ7lAGZsB+kjMv9\n/PESsmqe94xBhjXjqrfNj2dhSEWGSpntVRMbWOgBv2VKdCrW4m7xHCJinkKFPquE\nT/8oc9ZLAIZV7nqfLHTv8tXV2JDW+ZYsjzcSEIYGFqz1UeUQFMdnQtXqzTey+/n9\nHInPggzRwmfrLhUY7fFr2L0DIuSOu94+8dDYGI34kwjnkhK15cVP/ezyQN5AkMtX\nV4MSdm1RJB0FUMb0lYYMPfajheaNQeLxvnt3O0ySwy3xRYdilVCMQZaRiEOCD2gm\ngpwdk20tAgMBAAECggEACSPkp+UKA0ivyjf/oDLuEnlqUC50uGWrX1KR2S4Te0vr\npqTPYusMZLh0gyC/1et/Q/FHRsll76/WiS3KXI2atb9dJLIoXO0tN4p0rWbiBGfx\nHUty+8QHwxeMdx/jxOV6Fmqmwz6UtxM0SzOpaNW8hhMbV5xAGgvKJgJinjLgw1Ca\nTvigkgPEiGL4TCUpHb8A1kwRnw8mPU8/BjSoTd4D9liPhWzfx8Ta6nZDZSlhVlXv\nuznXAjywnrSSs3OHdZlL7Wfk4regdIloPLBJ+Bw2QKWdFpmqE9qOM1PgzozMZcvQ\n0qtFXeszjQwJ3Yz8fimpEEUruzpjL8b8bTKZpcxz0QKBgQDWgDirdiKTWSFAjSX4\nR0KE+BgEONXUykjo6CXxJYrXgINRoPcC4X22fTkTkK/LwBVP2tfos1l45EVutXdU\nOFFQ1gyBjCndELWPULjhSVXWo3bVWgZNQyvPanye1KK5qjL7C1gKK+4aBxG7TAJ/\n1zPF1Rdw6v49CQ1ZhgbFxZWKMQKBgQC3LJdK9uC+lHtVc/9OJTzwfBWIzwLlAgTX\n7r0SUEUAJzpRiWoVL6bB/3xHaIyL8fs0JzSPXX9t4wO2sZ89gw5y6STid960WJ1K\nz4EZipWbszNqoGQ8Ypyi9Nro2mW5I2PUJwCJoJPbvui9/QzNzYU96W5Ub2gi38eC\nJk5WaGIXvQKBgQCsbGuhYJgQjAAPzKZt+pI9ItTjZ/BzWZvcJOiC/OOrsXFeB58n\noK0WKXjdAlxd020OQON3+dt9Mzmg8ZwXhiHPchChflMUkd7wcjn7VqD6pXHbtL8G\nEm3AEGBWqKSZzHeol4jQvFlscKxPO51yuEG00ENEkJGH7KPfh74f8mDvsQKBgQCi\n3PTP5F3DVYWrXu2a76UVNV54O4cZUwg3bb+SLuiK0FYb9Q/rMmkUiDJZ021TaJ2v\nq0QiTEFm1LeSkq9BI+uO5Bi1e1r6AugnggQJT5zhyR1oKp+0yQD4KrkQ3E4hpIO4\nN81UkP+mHEIX2oyt15QmjgxY6o0upMVgxOpWDY1aCQKBgBcdd/JIJ0zOPUZ7LXbJ\n6yHNclfIiiz2DCq65m5AEW1dwMatc0xb5s1NU3eTTyM4kzhAZLnwvkSSXADFIHOX\nefQn1/aVjx8GBGR2BPq/rSXx1Ed9WW58oroA2ad66G/fVAtF8+sglbjlcWifQ0y9\nU9UWBLY71u4Gv5NbKNFM6Brt\n-----END PRIVATE KEY-----\n", + "etcd-manager-ca-events": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCd0Iu0tiGKGCqG\naK736HDML4/c2X0Uw9QcObagHNcmNaQ7DhLaLA8UU3RpCSBWxVnzilZQRq1Lw1SA\nQ4JcvGew4aNE4v8aSVepEaFUN6bVjGaFTLHyjES651BI7IJkXrZoTd/qKMCi0mMf\nQ1jw6yVwsk1j5aGFEVh8nWA7JHS/eREwXOJf1MW9sA7CFx29+RDjl0J4W69WYjqj\nbea74EqPmQG3vp49WGjBeFPmZgH22lccDllN+sntK7U1/CtVCP5KncFM5T1/pWLs\nPfsm6Xz7gw6ZciPum7i+r7fPcVADkMtOhkPWd3CJahGWOUSUQ3I4DOFDn/T0/ztx\nWL93WOD9AgMBAAECggEAByTDHNjCuhF1I/+V7fxhq2h6wAyV7CGfgXCTsE87O6Gh\nem0yaW769sNT/r4uQ9Hr9rQmyzxLyLpsjIrMb54KqTfUMBXWENlMOce7Gwi9iVHx\nQRbzQM9cfBkPlLbPVzOaA6zaFit76X2RsD8C8L2K2hKuzKG+zDsjBi8J2wYqhRE9\nFkOtOgyNI0FnN1x4XMBJ8gjzTzL1yjAe7zDghLeD5IxuCdkw3K0ijtJOb5ih/JeP\nT4ezalH/xMJCl9eWilkj62PqwNjgoTB/kJfJCjolPFQacl+QuNUEepHn5ZeOzsW7\n4FdnkuQs99utGQL4G54KYR7jcvTVEReqVa1sVtNMGQKBgQDSB3Pda1NF/PCCE0a1\nUCzTZnb+HdmYKiN25sYSTxmQWU3/Y0EOzaDEIFaN3zJiJAOLknjukTmvxbjR/D9B\nA/ZiylJBbe2QoUJShhi0nACA+16yQqGfEb2lCjz/7O8bw/EMcjNkDQ9CjEbOJp0n\n3kMhN26OyyMJHAiGvCXxAhalpQKBgQDAW1zT0dPQq7Xdi/w/ctrFY57f4veJ0u5K\no4YxWyXQhVqoKC/rkTqJ9gHUXIbToZqexYV8ZalmQg3FNTEdJlHYbf411+inoomw\nDOpjvrxNJ9gf3O/SQZpYSYDPCmqj0+G0dKU03K3weOaED8RJTapeCIqnxZbmsNLq\nENe07epeeQKBgQCo4vtFchcNMV7Iie2SwLy65SsMHj6pwECyTNG7SUXaMCWrFknr\nOGhtLT/MZ0zhY+pB4m9CDx9e+1Qp7neCtUf7motYN6gPY8ysBU56dPNmxDsahr/l\n5GMmAjyY4xJGtmHvyykN6/54WcpZ3O0EcRJz9Fn/ktNCcPteNI8h5LcHTQKBgCYt\n/C7O5wv0CRUARAET+vhS3aXyls90XnfSHUCOpuLMlUHrOBe5q7TWb2aHNvuqh9Yb\nvnMCeq0hTzplZGo0wOoEawCk6jjbuhiHd750jUuDLRKl97Pz8X60mCyAFMqPs6zt\nuhzT8HJajOaJepxSHg6sA1lpjpobXsG2thnjXzGBAoGAF6FrRHs02VomYZhZPry0\nOna5DhrVRf7OIkX5rC7GaBJdeT8U7c5kTILq5hbSY9a3QnfKpxRkObpb4t3WceCY\n4yqxda4YMq/r8uBVKJarOQlgaQ5m3Z4K8E2uv/C11/rz5rdvQMf6ml8bAx4tvSMB\n4n77YDoactRBtCLseCmRD8Y=\n-----END PRIVATE KEY-----\n", + "etcd-manager-ca-main": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCN6ImVn1yy/JPC\nm32zZtFbA8U2ArDuHIbN0GtNBAote2vBIqTJhCkR2uAbXk02mtJt83/d+/BW2g1d\nIY54b4RmzvU503LSgEJ1kGjPRsupiMtCsNioznCGJe1F41OY1xT2VZ0bthBZnwyP\nyQn9oR1UsSrJeo06TFO+WetCYBqLfM/Srd3m0AouwvgxEEMAT8BMkGXxk50yorUr\ntKirw0AdoXXuUDVBWgUQMKa26bccUX9CJGgdxVDsyw2x7V1s1GsXVqDUjZIUQIP+\nk39YAvx9li8vBJTN8lzlxzM9C1jHHYLVSoUNnSRP/wWIvM/SZBdLSFuQaSjJeetn\n5aIzgzJzAgMBAAECggEADfEbWOdVikIu9CiM0qwJ0OYC1JN/TxpFxjR+FKZBp7lF\n9Pofc94eBaKNPP/ofxjHDsVX05CifeEhyfuqs+6YDlx/Ep832zJKe6L97eTrMmzk\nAPMMV8oizpMtITe8NiDMO7oqjspPoRzZ25bpRxFMQLTzp334ud8KcgIC9bwoyRMk\nu9So9TIqx7MBgXkDDIXno+Bp1nHiPzu/7Gf7Pzxbmu2BDPPfcn9C92V55J6nsulw\nMWobnlig5+ZnvaKTxorlgTObMJ9eHMT4jQ7kqENmugxUdH6ohXf6z/fdgR/DkHxQ\ntbmYo78VDTfhyD9+cv27eJH4Wv6PVvGXatv7/YNU8QKBgQDBBr6ksHF3y73Is3bV\nkk8u5ykYfACqe4EVbUh4u74dCllvpqn47uVgbA8v28uNHmxu+tnU5OmxLgrW+EXp\nllTkiFhXZjpvC0j4upXrPU3rJMYzsFHva/tzQvaFSPrwCFn8RHUiXzBa/6YUwVdg\nbakh/OCe9PXMXwzabdV79JNXkQKBgQC8NH+ZUzCtpLET8Sr0YVk8vJ0xMBnrlvpX\n30igzm9ZYCPlVz38a/P+mEnjZVD0xl9+vIfEiq1hrODi9kx2V4n5/hzp+p9BU414\n+6RqxJ0RvA7bqpiKDAEWfqyxtw3llZTkTikTqlpKUVkYeD1luHb3hVHL4FZkDKbe\nzmAMW8YPwwKBgBxjyJBiXx32emBVfOnmjfzhBNtUCakJZy/mld0kB88voX3ZbnBt\nepEW85jfY+3Khro/7N/yFyj3E6MQev/jG3NNVmjsYTXJ3B6xGrGxn48xJnGdwj9L\ni30lYSqzVC0PnWSMEQpQI8WOijfNQ/OrzU9xaPwYHTrE1BggRWDo1O7xAoGALi1z\nYVtYBkLyb8Isl+mKy6w7pWX39iYFJqpSV6JPD87W/954Bwc1xrH27yRcyLIjnCUF\n6uNbxytzwjxMZnz0DbKbxY0Ipi6p95VGsiurnj5WJu+knbp0SeCq3q/baQlajune\nikSyAqoTAj3+UocAKZs8hpl0mPfBuclJJ78cyaECgYBMo6xd3bCGz3QM0d9/FwYQ\n20LdVFlL/H8XB9bbvYwByLOTdvHxpUDF6rYvOyns5kKoGjWufHwTHIVuDljHtXkD\nvayVvY6Vld9cpDkxQUkIpenyygHGN2tgsskirwcV1E/7zXWz1o29ylyEfESp7bIT\n5kncUTLrairQYYIqW9701w==\n-----END PRIVATE KEY-----\n", + "etcd-peers-ca-events": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGqWJF2hj3XRNn\nF+vHZ8yupcJAacL1WYVAc+jpSLrImIL0T9BOuWGrCdOzpXGqM6RTAqwzi8mdD6lP\nbjD8UgxB4P+/iIxjcbh1cWSyZP+eWQ8KDUpNu/YnkJxjw8/qS5kPunNA7HUljIvS\n6EodzZGrjgtjpJN53sxEtkkh9FnrvYZLKzFdPXyDd+c2vrk6HnoY/jJjgIvH1c35\nVxhZdnaCTSuimid7o+x24gnE9kcOkkNyIa0P401Xz7lxpyCVt9qpGpODDh+Czy6M\nxXg4tAjgq7Y6bMojrZibo4YsWHcSXxTn9/2W19NJt8ZoWC2OGTlgvaKYaKCMHHBk\nd2fUSJ3PAgMBAAECggEAMv/8BxOw8hv2ZbJ56JSosrNCFZgxIRxsvXmkMvNlZCbn\nA4JT50nnHWol64oPDpM4XckjXQ5HYWdKX2ZZ9AS6OdVhwpFy/PqOULQeFkAgcU3K\nsxlrWF5nGIwi4ee8ghkLP8LtYJ7zigK1SJ0ynckO2vxZxyhSZeiBByjGyI064m4u\nfNhyerZLvGTGdzGtoy1hIXYmfa+YD31tgyinWKOvTpnl0h5+P9Wcp+VPFAWx1gxx\nU8RU5rvuiWzxW7CvjAPdNRr6QwJEdX9DtqS9NQCiDB8yWVp7mSHFvi3R8D+k0Y0B\nDHTXjlBtUS54oL6BtfCikSoj/FHVwPg5HBJS/yDKdQKBgQD3CbniQQuqCFu+jET2\nNY46yE58wPzsNACcQ7Xd+tk1ioGCrAI6+fWPWfkUHumhsAPcYGaVWTEQrPGsJa+f\nVTWVHorkCD+KfH8WShJySIt8JWQCpiY6niDCM1Vyqu1a/bkHP1oIpb13/fYabOeg\nXG5BkYldnGabOux5ujY/ED6O0wKBgQDN3mFlVpD13sYlPii1j8Igw5mnn5tLqRT8\noKMNyFOIlbnV9NixPkTnBa0EU7JfRgoIotuTdq96kWN/rDTtimA3DmGm2j2c8rgP\n1RO4+O7MCJDIrtKEa4Ci4xqfL77q+dk5EfsxK+2LNYtmSkGI+uKklNkX7kNXbshb\nc2MRHQZvlQKBgQDxXW9gwGzFFKWO5z350RnGFS+adHSp06VFCGMJH65ZwZoIMkd7\nl/10slPKdOmQMImVKBsSkLJ/uHnn2GpfIsHXkJ/jbKuogzsoa/bZ8kdWZLHyZKO1\nYgpftAgooRiHbp75lcxB5q3YLynlZ8Jny+dt6ORtsgIOaOPjiyy1MQnuQQKBgAkh\nsxpGo7l0Pz/JqQ6cqNTBOGCaEM5jWp32FiV3XFUzpwGZkb8S7wV8V/dYKxJUzf8i\nEjFRU2lRZ6NzJQngWHI4vhYjmhYLIiT5o51R1W3Rd3KfZpEfH56aCkVDXemYWY1R\ngd3lf+zr3BoWcx+7WTBJtfN8yxnmvRZpYNySeAWdAoGBAOnJPhIONPNGIkCiorlW\nnGLkRvf5V94FtrnFTsPKKnupGHgkSDUh3LQFxiih5q2x+dvTUHLB261BiBtVw+ZX\n1U9DdNvPBksqZO+riCi4QFHd/3wDv0RVMGJjU7JZqzLrBIojsyWkPXzk51vLTLqO\n+hvS6eQketO5NOTD/z8OwVOr\n-----END PRIVATE KEY-----\n", + "etcd-peers-ca-main": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDeFZmkx6oFoqcX\n4rW0ihQo8FjxxJwKNu4vn2mpgmI87uCNytqpSY2uXLGKbrIrFW4Kb8kzCNAM1FwA\nTDAmK+ikpbRmKM9l1dsp1gVEuG8XllioCfS8papVXLmhLufNEbTXOeWFkTdkQ2LI\nN7F/P9H5OsT1bTCkzVgji4n8SjpNrxk8xeGWMVQ2/S3RlQdPybS+J1bMNh8q8Z2+\nJ6ZukPOL1vaitN/5jl5eHyvAuOe5/dExEDGHpRUaYDjav1pLjXv0g9rIHiQ3BZIY\n5RFnRQGF9q/aZhExwMIxoelcOpmDES+d4LXR6SS4TZIujLxJpYDH/08dQNXk66cF\nlQq5KTXrAgMBAAECggEAWUq6AcS9mZJyOGpQwIwb2vH38Q0p5/l5/u/LXfosYW9y\nHdCH5jTKoGZog85KGIR/P/b+IGTUsbPYYc5ANbjFyx1M0R5rdPhtK/7/E0hMcmFi\nt/14xzicqcGkaSggzxTlbvu6St++M7PcTcAGv7D7TYfRlRQX0Ms3OuNp0WRO2Nhk\nNkIsSX8/zhQaqu7xR36e8tjGyFcUY5t+RheDKuXTDlpzWeNrwMTFx0LFLFUZXnOG\nr82XgJ+ZlKDC/LzKEow0po62MStv7Xsr1fCj5zP62G/FYFOoTWBPcXZELesqBxET\nqbA0fdVZvvXwF//PgrWEHS2/RkG0wnr9MkmHaysJIQKBgQDzwXcDE3w8P0DEYQSC\nTcwIZa+kMX7BjXUe6s8OTal+Zn702jduNB3ZaEr5TgCElNWu+hhUMilNpCORC5H6\n084+e91i6CL5W00AOB8ILggwkKXDmNLAz6IhBF0YZIs76d90hrZjCA3vAwEbiK3T\nRtV2DoIAv0jOYr6Npb7nvvbR+QKBgQDpPXTP5uCbF8c1Rwt5p+4uwB0idM7onbOB\nRyLMN+aGfuyduyIzfm87TEpU7U5gRhVLUTdo7BmS/ySBv9RBdCZO/YsVQ9vVm1Eb\nl6Gzutd8Bb+fInTwCR7zs463AZngMK9c5vp2ty9UXev1zaV6SiIIGDwwdo8EB1Fr\nFnCER5rAAwKBgFfPdQklptapTrKv5Ijwb7UCtMBomCovxGZr3kVf0tRXJhsc1WUc\n3zfnq0KeR+97EnNjYwpJu7ldjPZ+fYsnbB/0TS+JQrgjnNycWwl+U7LzGNQapKmS\n4uwT/1l/Nu1Rdg8rTF7mkuZsZaS+mygGV3WdClYskpTO26h5cglYMOA5AoGAO3gl\n/pR5lFHdwyHYNS6yNk1ZoF10HC+Jg17l6ngeWRCnuBfswoBsUAxFr53KLBXye/+7\nRw9b/fT8E/EbZ4z774wKGGPy+2XcqlJhMnhjVQYKM8MPWdyZfb28l8DawSTosaFb\nH0XZ1YMVFNlYZfPc9X9oaVrxYC+VfN591dyEAo0CgYEAqbu0Qf01PDqd+bUQbgKC\nzJB1o8OHOEG93gPi7XYN5Et2LtcinCdYsvvp5kL+cmCAzjSj35G+avS//f6NY7I/\n0xi6Q50fmXqL8y90hgEvO86eR+KULIdBOnW+wfX9E8pOHnO9DXCm/quod54PF1F8\na6tI1/ubuLZ4LyN+CByGg4k=\n-----END PRIVATE KEY-----\n", + "kubernetes-ca": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCj4lFZWKgr0KJi\nfwuRforw9nhzFRmOMLYARY0G0Xyy0KoFddNe9rm0F0QxRLkp0U/5RDoh/cc/EBbu\n8pHub5Ng8g9rvFlcZRVzRqKnI63BHVBO5VM7Hm8XEQu2aATCvkakjK111qqhsn9t\nN0shzZAplyWOS6+4/dgAHU5i3zUaByj3iAjQWS18yzdMu2CGy6iPJRwrB5xV6JXM\nrIhkqKZx1/5fFir/YDzOTFtX1VPfZweW2XeDcpb6CE8+WDDV2Ovrfj/bxgoCFUzP\nWfatfWwKz6WpYfjwyYD39ynvQ4ED/eZvQ9wC2RdiiR6Ka6w5ez+nIIWh9WnyGIbc\nAGhVc5qZAgMBAAECggEAC+tlz67Gd7FYx8f5VzkYV+0SCgskObBKfHvFuK1urJhh\nCInVhTGpHg0XKGvMLKhACp9iVWvEz+eWrQU3HsI+VBQm+C04MdbBGApKKDsZfYZp\nSEoe1jNUQ+qfqtunHpboqNOTUvlf0OgccBzpoDX4R0SEA1tD45uRPnWwAgbwymMf\nGJe5KVOUfK1YakPgkLsGG8VxZMY69XJpJyAIznf/5qmYY6E+1hjcG+DJHoWZmhRG\nH5Rm/CguRbQ7OMo93RtAzZOlzSSSZRvvzkJxdx5spP5ldCXrdEXYUA1fp4NWUNYZ\nw4UiR3An/TCMAlg7N9XacUKlUSxUdx5CvEJD+q5UBQKBgQDXabxjBUmmqVOvgZEx\nAguIcU7J1Cgizzsc8qcn0CYxMWiVKLoArwBLukxGB9n07tkbEdaZaJfEXKvqTKcY\n0WYoqnIiQk8saLjE1pqy726OXiVGZJnihbS5Pfd5GUXo4hyskm+yzncRi3aN8U+I\nO5Msd8OYichvbsW2Tx0RY68MJwKBgQDCwyAkW0cn27Fus3xbExXh/Q6ZBxLgPfW9\ng6Dn4lSuBEcz/4B0+wd0IyxAh7OykWnlpybBrLFWe3+ZD25rRB35kmDo09LmvxQF\nPPlTcsnHddQUF7DKEZmTtiCTQpjyGpMiz3jQ6Wft24MdBr7MvFvPN8/OhfuLZrkc\n6lPX1VybPwKBgEfp9cl8/2yTROvnrogU7bK8WRhKKqgMVOjCHYnOFKIBo6+ALDCE\nO+4MIOCUSHBbuMEvwFvwrhIQ+he7RWGkrp1O4dGMH5Bs2ZplJror8tW2mm0YssML\n7hyVkmOsS0mHCxmVvZKNGPUTgR27/Ng3Q1fqF7N59B83dOv7F3IxvEOrAoGAChea\nXsfsFACqXRYn54BBrggy7XMUY1JSnZNojxuHT80g82zV9zna1QXNkTp28VXfqurS\nc07DkbusjP3jtuM8umX/Z5EyaIqWOv9B3DUih5dveesQWig3URQq8/aLfmAqJ6+g\njxLJM+YgESz8ThekmkRGHGmM81hLV1g9J/4CYnkCgYBUG+JWZiEoJKfl18V4i5TF\n+x6F3z+7qmmW5WBBN9fLR7KKYmPy7Dn0S9j5DXl9+FEeasahXfFCr2xs6QRAYO7k\ngPN3ucVaTZnqETJCDuZYA5zz9lO5MIg0qBIKfFqF6JAHYKSi5oogvJ+3C3RW1AQD\nmyYdUoU8/QwIuKrUgZX4XA==\n-----END PRIVATE KEY-----\n", + "service-account": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCv0IbA+FEOlmlT\niD9zfc7LCkzQd67Mulg1oO5Jg8p9S8X+wd0lB6HLYH5bfTRTDLv0Zp4Y3v/VMjAC\nS5FOBGWmrdnCX1DcDu65PKXJGObjskszZMQKUrm7w3PvWoD7RC6FbPHQTVBzvY6q\nU2yTotsj/sD9SckIs0zlFpwVN4RF8q9o2SiVLzwvISiCaNFWdzXixnmXuLuW2ksf\nBUiE9SxylNVFrGe5LEFxGgaSfdcMrwCYaG9+I/1hvliJk/kirF2keIlmRYGuWBze\nsl2R5/sFVcuexH4o3XuaWFt3oxdqhxulCmCElUAwkYp8xg/Qnc99F8c/e1BC/DkA\naLM6kxGLAgMBAAECggEAHNIlj536m6uTLzsB5nV/NXrwxf54od2s9s2fgl71DhPN\nLKABwpTntFE90ChMMDkW2U+2fFlCEw7xHnneI1ib2Uuyi3yDOC/Hzoc0niyo3OFY\nwTS0NlpX/CxkOvPZJQIzph0KiZ0YTSuL0ZFva8b3VC/qLxtYtfET7+dZmtjI+YuI\nyDPfHSSsEZmOgIHj9Tnirk3RMdTPdC0qJOQDNw6XM5JbYKcDYUWvobQKFig/KhBq\ndX82dgec0YoaceHdgyZ0bTLD5HSeKpF2YQ2KO2fHdnzEpr/nqrA30mBWCdr/dsc0\nagAkN0U+hBr9IUS34LTrKHo96mULhHmLjWbOE1SvwQKBgQDYJ9xi42BOYKXkzF7w\ncmcb4pI4QU9kJ7aFP6uTD6WNP2LaE3y89YCsycnDqzhUgWLkn4gTk6wH74lg3I1g\ni0RM9WUWb2QUZiEMvbaVW3qSTVbMFtm3YeR8WYuKdwsPQ/LGvi6yP4+Dg0EanvDJ\n8PUY6i0GclI0g21s/bkf6u8/sQKBgQDQOQNVX57EGY0JzkVgLFjcYAwbmekEXAvP\nOlc2VI8gkfsGg7Pyj7rfgZewYkBpLtYAaY/2P7G7LmZvsY/M6TghwOc6jneAVSu+\n4GC4w9OpScpnpGcrHSPcWowWc1gfnpCiNIrdSi6p+qBcYmdlKqMTsfC/pHhRMxoR\nN+UTUNhP+wKBgE+ywsujUosTj5dA03CMfgAGFiuvyPoAuod+5mAe0Me3q1cFAGUb\nkAoljMMcinR+EFyDhcTSGggDBgA00dva/Z/yiwcuoxPyZBUyjfOb+yGeanZtpnC7\nI2fGDHFNW0KRpx4oWUdXsC6pYoPSPq32rig1j8xksnKyUtJw6qUeFoDBAoGAGDxK\nkwlKJXED5XxIvuTqgtv7DMthP8ym4zvbIsu8zQGwwhVaQIVSLumaikElVhjJd8rR\nosvPGAZSjfWQ0zzh96vIPhyO9DT1noXnPBou7KlWEbIk//OoVOGFJKfQPf3qv7Z1\n40PbIpW1TIt3lWLdEFHJv8SY7HqSMibbVuA2DTkCgYEAz5O8Y8cA91Jce2Rrno1j\nhNlaplo41q+2SvTzcej3kUH+3oO5hTJGWRQuQB2ydu6Rl39Hz4cGhYB69sk7frrx\n6Ze8HijLckAr56NOIHLywT/cycflerMpSbw575p5moHMOuGa6Bvm1HLS3ifPOx6I\njQ+n2EEwCtZGq1UogVfVpxY=\n-----END PRIVATE KEY-----\n" + } +} diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/in-v1alpha2.yaml b/tests/integration/update_cluster/gce-kops-module-equivalence/in-v1alpha2.yaml new file mode 100644 index 0000000000000..d2269b3ed43e6 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/in-v1alpha2.yaml @@ -0,0 +1,203 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: k8s1.equivalence.example.internal +spec: + api: + loadBalancer: + type: Public + authorization: + rbac: {} + channel: stable + cloudConfig: {} + cloudProvider: gce + configBase: memfs://tests/k8s1.equivalence.example.internal + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: control-plane-us-west1-a + name: a + - instanceGroup: control-plane-us-west1-b + name: b + - instanceGroup: control-plane-us-west1-c + name: c + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: control-plane-us-west1-a + name: a + - instanceGroup: control-plane-us-west1-b + name: b + - instanceGroup: control-plane-us-west1-c + name: c + manager: + backupRetentionDays: 90 + memoryRequest: 100Mi + name: events + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubeProxy: + enabled: false + kubernetesApiAccess: + - 0.0.0.0/0 + - ::/0 + kubernetesVersion: 1.35.5 + metricsServer: + enabled: true + insecure: true + networkID: default + networking: + cilium: + enableNodePort: true + nonMasqueradeCIDR: 100.64.0.0/10 + podCIDR: 100.96.0.0/11 + project: example-project + serviceClusterIPRange: 100.64.0.0/13 + sshAccess: [] + subnets: + - cidr: 10.0.16.0/20 + name: us-west1 + region: us-west1 + type: Private + topology: + dns: + type: None + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: control-plane-us-west1-a +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Master + rootVolume: + size: 64 + subnets: + - us-west1 + zones: + - us-west1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: control-plane-us-west1-b +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Master + rootVolume: + size: 64 + subnets: + - us-west1 + zones: + - us-west1-b + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: control-plane-us-west1-c +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Master + rootVolume: + size: 64 + subnets: + - us-west1 + zones: + - us-west1-c + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: nodes-us-west1-a +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Node + rootVolume: + size: 128 + subnets: + - us-west1 + zones: + - us-west1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: nodes-us-west1-b +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Node + rootVolume: + size: 128 + subnets: + - us-west1 + zones: + - us-west1-b + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: k8s1.equivalence.example.internal + name: nodes-us-west1-c +spec: + image: ubuntu-os-cloud/ubuntu-2404-noble-amd64-v20250606 + machineType: e2-medium + maxSize: 1 + minSize: 1 + role: Node + rootVolume: + size: 128 + subnets: + - us-west1 + zones: + - us-west1-c diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/kubernetes.tf b/tests/integration/update_cluster/gce-kops-module-equivalence/kubernetes.tf new file mode 100644 index 0000000000000..78e453bf6e21e --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/kubernetes.tf @@ -0,0 +1,1136 @@ +locals { + cluster_name = "k8s1.equivalence.example.internal" + project = "example-project" + region = "us-west1" +} + +output "cluster_name" { + value = "k8s1.equivalence.example.internal" +} + +output "project" { + value = "example-project" +} + +output "region" { + value = "us-west1" +} + +provider "google" { + project = "example-project" + region = "us-west1" +} + +provider "aws" { + alias = "files" + region = "us-test-1" +} + +resource "aws_s3_object" "cluster-completed-spec" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_cluster-completed.spec_content") + key = "tests/k8s1.equivalence.example.internal/cluster-completed.spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-events" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-events_content") + key = "tests/k8s1.equivalence.example.internal/backups/etcd/events/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "etcd-cluster-spec-main" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_etcd-cluster-spec-main_content") + key = "tests/k8s1.equivalence.example.internal/backups/etcd/main/control/etcd-cluster-spec" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-bootstrap" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-bootstrap_content") + key = "tests/k8s1.equivalence.example.internal/addons/bootstrap-channel.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-coredns-addons-k8s-io-k8s-1-12" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-coredns.addons.k8s.io-k8s-1.12_content") + key = "tests/k8s1.equivalence.example.internal/addons/coredns.addons.k8s.io/k8s-1.12.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-gcp-cloud-controller-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-cloud-controller.addons.k8s.io-k8s-1.23_content") + key = "tests/k8s1.equivalence.example.internal/addons/gcp-cloud-controller.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-gcp-pd-csi-driver-addons-k8s-io-k8s-1-23" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-gcp-pd-csi-driver.addons.k8s.io-k8s-1.23_content") + key = "tests/k8s1.equivalence.example.internal/addons/gcp-pd-csi-driver.addons.k8s.io/k8s-1.23.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-kops-controller-addons-k8s-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kops-controller.addons.k8s.io-k8s-1.16_content") + key = "tests/k8s1.equivalence.example.internal/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-kubelet-api-rbac-addons-k8s-io-k8s-1-9" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-kubelet-api.rbac.addons.k8s.io-k8s-1.9_content") + key = "tests/k8s1.equivalence.example.internal/addons/kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-limit-range-addons-k8s-io" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-limit-range.addons.k8s.io_content") + key = "tests/k8s1.equivalence.example.internal/addons/limit-range.addons.k8s.io/v1.5.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-metrics-server-addons-k8s-io-k8s-1-11" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-metrics-server.addons.k8s.io-k8s-1.11_content") + key = "tests/k8s1.equivalence.example.internal/addons/metrics-server.addons.k8s.io/k8s-1.11.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-networking-cilium-io-k8s-1-16" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-networking.cilium.io-k8s-1.16_content") + key = "tests/k8s1.equivalence.example.internal/addons/networking.cilium.io/k8s-1.16-v1.15.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "k8s1-equivalence-example-internal-addons-storage-gce-addons-k8s-io-v1-7-0" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_k8s1.equivalence.example.internal-addons-storage-gce.addons.k8s.io-v1.7.0_content") + key = "tests/k8s1.equivalence.example.internal/addons/storage-gce.addons.k8s.io/v1.7.0.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "kops-version-txt" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_kops-version.txt_content") + key = "tests/k8s1.equivalence.example.internal/kops-version.txt" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-control-plane-us-west1-a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-a_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-control-plane-us-west1-b" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-b_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-b.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-events-control-plane-us-west1-c" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-events-control-plane-us-west1-c_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/events-control-plane-us-west1-c.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-control-plane-us-west1-a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-a_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-a.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-control-plane-us-west1-b" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-b_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-b.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-etcdmanager-main-control-plane-us-west1-c" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-etcdmanager-main-control-plane-us-west1-c_content") + key = "tests/k8s1.equivalence.example.internal/manifests/etcd/main-control-plane-us-west1-c.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "manifests-static-kube-apiserver-healthcheck" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_manifests-static-kube-apiserver-healthcheck_content") + key = "tests/k8s1.equivalence.example.internal/manifests/static/kube-apiserver-healthcheck.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-control-plane-us-west1-a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-control-plane-us-west1-a_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/control-plane/control-plane-us-west1-a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-control-plane-us-west1-b" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-control-plane-us-west1-b_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/control-plane/control-plane-us-west1-b/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-control-plane-us-west1-c" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-control-plane-us-west1-c_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/control-plane/control-plane-us-west1-c/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes-us-west1-a" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes-us-west1-a_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/node/nodes-us-west1-a/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes-us-west1-b" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes-us-west1-b_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/node/nodes-us-west1-b/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "aws_s3_object" "nodeupconfig-nodes-us-west1-c" { + bucket = "testingBucket" + content = file("${path.module}/data/aws_s3_object_nodeupconfig-nodes-us-west1-c_content") + key = "tests/k8s1.equivalence.example.internal/igconfig/node/nodes-us-west1-c/nodeupconfig.yaml" + provider = aws.files + server_side_encryption = "AES256" +} + +resource "google_compute_address" "api-k8s1-equivalence-example-internal" { + name = "api-k8s1-equivalence-example-internal" +} + +resource "google_compute_address" "api-us-west1-k8s1-equivalence-example-internal" { + address_type = "INTERNAL" + name = "api-us-west1-k8s1-equivalence-example-internal" + purpose = "SHARED_LOADBALANCER_VIP" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name +} + +resource "google_compute_disk" "a-etcd-events-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-events" = "a-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "a-etcd-events-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-a" +} + +resource "google_compute_disk" "a-etcd-main-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-main" = "a-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "a-etcd-main-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-a" +} + +resource "google_compute_disk" "b-etcd-events-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-events" = "b-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "b-etcd-events-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-b" +} + +resource "google_compute_disk" "b-etcd-main-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-main" = "b-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "b-etcd-main-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-b" +} + +resource "google_compute_disk" "c-etcd-events-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-events" = "c-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "c-etcd-events-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-c" +} + +resource "google_compute_disk" "c-etcd-main-k8s1-equivalence-example-internal" { + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-etcd-main" = "c-2fa-2cb-2cc" + "k8s-io-role-master" = "master" + } + name = "c-etcd-main-k8s1-equivalence-example-internal" + size = 20 + type = "pd-ssd" + zone = "us-west1-c" +} + +resource "google_compute_firewall" "https-api-ipv6-k8s1-equivalence-example-internal" { + allow { + ports = ["443"] + protocol = "tcp" + } + disabled = false + name = "https-api-ipv6-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["::/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane"] +} + +resource "google_compute_firewall" "https-api-k8s1-equivalence-example-internal" { + allow { + ports = ["443"] + protocol = "tcp" + } + disabled = false + name = "https-api-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["0.0.0.0/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane"] +} + +resource "google_compute_firewall" "kops-controller-ipv6-k8s1-equivalence-example-internal" { + allow { + ports = ["3988"] + protocol = "tcp" + } + disabled = false + name = "kops-controller-ipv6-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["::/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane"] +} + +resource "google_compute_firewall" "kops-controller-k8s1-equivalence-example-internal" { + allow { + ports = ["3988"] + protocol = "tcp" + } + disabled = false + name = "kops-controller-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["0.0.0.0/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane"] +} + +resource "google_compute_firewall" "lb-health-checks-k8s1-equivalence-example-internal" { + allow { + protocol = "tcp" + } + disabled = false + name = "lb-health-checks-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["35.191.0.0/16", "130.211.0.0/22", "209.85.204.0/22", "209.85.152.0/22"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane"] +} + +resource "google_compute_firewall" "master-to-master-k8s1-equivalence-example-internal" { + allow { + protocol = "tcp" + } + allow { + protocol = "udp" + } + allow { + protocol = "icmp" + } + allow { + protocol = "esp" + } + allow { + protocol = "ah" + } + allow { + protocol = "sctp" + } + disabled = false + name = "master-to-master-k8s1-equivalence-example-internal" + network = "default" + source_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_firewall" "master-to-node-k8s1-equivalence-example-internal" { + allow { + protocol = "tcp" + } + allow { + protocol = "udp" + } + allow { + protocol = "icmp" + } + allow { + protocol = "esp" + } + allow { + protocol = "ah" + } + allow { + protocol = "sctp" + } + disabled = false + name = "master-to-node-k8s1-equivalence-example-internal" + network = "default" + source_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_firewall" "node-to-master-k8s1-equivalence-example-internal" { + allow { + ports = ["443"] + protocol = "tcp" + } + allow { + ports = ["10250"] + protocol = "tcp" + } + allow { + ports = ["3988"] + protocol = "tcp" + } + allow { + ports = ["8472"] + protocol = "udp" + } + disabled = false + name = "node-to-master-k8s1-equivalence-example-internal" + network = "default" + source_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_firewall" "node-to-node-k8s1-equivalence-example-internal" { + allow { + protocol = "tcp" + } + allow { + protocol = "udp" + } + allow { + protocol = "icmp" + } + allow { + protocol = "esp" + } + allow { + protocol = "ah" + } + allow { + protocol = "sctp" + } + disabled = false + name = "node-to-node-k8s1-equivalence-example-internal" + network = "default" + source_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_firewall" "nodeport-external-to-node-ipv6-k8s1-equivalence-example--5pdea1" { + allow { + ports = ["30000-32767"] + protocol = "tcp" + } + allow { + ports = ["30000-32767"] + protocol = "udp" + } + disabled = true + name = "nodeport-external-to-node-ipv6-k8s1-equivalence-example--5pdea1" + network = "default" + source_ranges = ["::/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_firewall" "nodeport-external-to-node-k8s1-equivalence-example-internal" { + allow { + ports = ["30000-32767"] + protocol = "tcp" + } + allow { + ports = ["30000-32767"] + protocol = "udp" + } + disabled = true + name = "nodeport-external-to-node-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["0.0.0.0/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_firewall" "ssh-external-to-master-ipv6-k8s1-equivalence-example-internal" { + allow { + ports = ["22"] + protocol = "tcp" + } + disabled = true + name = "ssh-external-to-master-ipv6-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["::/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_firewall" "ssh-external-to-master-k8s1-equivalence-example-internal" { + allow { + ports = ["22"] + protocol = "tcp" + } + disabled = true + name = "ssh-external-to-master-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["0.0.0.0/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_firewall" "ssh-external-to-node-ipv6-k8s1-equivalence-example-internal" { + allow { + ports = ["22"] + protocol = "tcp" + } + disabled = true + name = "ssh-external-to-node-ipv6-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["::/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_firewall" "ssh-external-to-node-k8s1-equivalence-example-internal" { + allow { + ports = ["22"] + protocol = "tcp" + } + disabled = true + name = "ssh-external-to-node-k8s1-equivalence-example-internal" + network = "default" + source_ranges = ["0.0.0.0/0"] + target_tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_forwarding_rule" "api-k8s1-equivalence-example-internal" { + ip_address = google_compute_address.api-k8s1-equivalence-example-internal.address + ip_protocol = "TCP" + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "name" = "api" + } + load_balancing_scheme = "EXTERNAL" + name = "api-k8s1-equivalence-example-internal" + port_range = "443-443" + target = google_compute_target_pool.api-k8s1-equivalence-example-internal.self_link +} + +resource "google_compute_forwarding_rule" "api-us-west1-k8s1-equivalence-example-internal" { + backend_service = google_compute_region_backend_service.api-k8s1-equivalence-example-internal.id + ip_address = google_compute_address.api-us-west1-k8s1-equivalence-example-internal.address + ip_protocol = "TCP" + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "name" = "api-us-west1" + } + load_balancing_scheme = "INTERNAL" + name = "api-us-west1-k8s1-equivalence-example-internal" + network = "default" + ports = ["443"] + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name +} + +resource "google_compute_forwarding_rule" "kops-controller-us-west1-k8s1-equivalence-example-internal" { + backend_service = google_compute_region_backend_service.api-k8s1-equivalence-example-internal.id + ip_address = google_compute_address.api-us-west1-k8s1-equivalence-example-internal.address + ip_protocol = "TCP" + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "name" = "kops-controller-us-west1" + } + load_balancing_scheme = "INTERNAL" + name = "kops-controller-us-west1-k8s1-equivalence-example-internal" + network = "default" + ports = ["3988"] + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name +} + +resource "google_compute_http_health_check" "api-k8s1-equivalence-example-internal" { + name = "api-k8s1-equivalence-example-internal" + port = 3990 + request_path = "/healthz" +} + +resource "google_compute_instance_group_manager" "a-control-plane-us-west1-a-k8s1-equivalence-example-internal" { + base_instance_name = "control-plane-us-west1-a" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "a-control-plane-us-west1-a-k8s1-equivalence-example-internal" + target_pools = [google_compute_target_pool.api-k8s1-equivalence-example-internal.self_link] + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.control-plane-us-west1-a-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-a" +} + +resource "google_compute_instance_group_manager" "a-nodes-us-west1-a-k8s1-equivalence-example-internal" { + base_instance_name = "nodes-us-west1-a" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "a-nodes-us-west1-a-k8s1-equivalence-example-internal" + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.nodes-us-west1-a-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-a" +} + +resource "google_compute_instance_group_manager" "b-control-plane-us-west1-b-k8s1-equivalence-example-internal" { + base_instance_name = "control-plane-us-west1-b" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "b-control-plane-us-west1-b-k8s1-equivalence-example-internal" + target_pools = [google_compute_target_pool.api-k8s1-equivalence-example-internal.self_link] + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.control-plane-us-west1-b-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-b" +} + +resource "google_compute_instance_group_manager" "b-nodes-us-west1-b-k8s1-equivalence-example-internal" { + base_instance_name = "nodes-us-west1-b" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "b-nodes-us-west1-b-k8s1-equivalence-example-internal" + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.nodes-us-west1-b-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-b" +} + +resource "google_compute_instance_group_manager" "c-control-plane-us-west1-c-k8s1-equivalence-example-internal" { + base_instance_name = "control-plane-us-west1-c" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "c-control-plane-us-west1-c-k8s1-equivalence-example-internal" + target_pools = [google_compute_target_pool.api-k8s1-equivalence-example-internal.self_link] + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.control-plane-us-west1-c-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-c" +} + +resource "google_compute_instance_group_manager" "c-nodes-us-west1-c-k8s1-equivalence-example-internal" { + base_instance_name = "nodes-us-west1-c" + lifecycle { + ignore_changes = [target_size] + } + list_managed_instances_results = "PAGINATED" + name = "c-nodes-us-west1-c-k8s1-equivalence-example-internal" + target_size = 1 + update_policy { + max_unavailable_fixed = 1 + minimal_action = "REPLACE" + type = "OPPORTUNISTIC" + } + version { + instance_template = google_compute_instance_template.nodes-us-west1-c-k8s1-equivalence-example-internal.self_link + } + zone = "us-west1-c" +} + +resource "google_compute_instance_template" "control-plane-us-west1-a-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 64 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "control-plane-us-west1-a" + "k8s-io-role-control-plane" = "control-plane" + "k8s-io-role-master" = "master" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "control-plane-us-west1-a" + "user-data" = file("${path.module}/data/google_compute_instance_template_control-plane-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "control-plane-us-west1-a--rc6ris-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.control-plane.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_instance_template" "control-plane-us-west1-b-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 64 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "control-plane-us-west1-b" + "k8s-io-role-control-plane" = "control-plane" + "k8s-io-role-master" = "master" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "control-plane-us-west1-b" + "user-data" = file("${path.module}/data/google_compute_instance_template_control-plane-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "control-plane-us-west1-b--h02o8r-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.control-plane.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_instance_template" "control-plane-us-west1-c-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 64 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "control-plane-us-west1-c" + "k8s-io-role-control-plane" = "control-plane" + "k8s-io-role-master" = "master" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "control-plane-us-west1-c" + "user-data" = file("${path.module}/data/google_compute_instance_template_control-plane-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "control-plane-us-west1-c--gbuta4-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.control-plane.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-control-plane", "k8s1-equivalence-example-internal-k8s-io-role-master"] +} + +resource "google_compute_instance_template" "nodes-us-west1-a-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 128 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "nodes-us-west1-a" + "k8s-io-role-node" = "node" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "nodes-us-west1-a" + "kube-env" = "AUTOSCALER_ENV_VARS: os_distribution=ubuntu;arch=amd64;os=linux" + "user-data" = file("${path.module}/data/google_compute_instance_template_nodes-us-west1-a-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "nodes-us-west1-a-k8s1-equ-082h6h-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.node.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_only"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_instance_template" "nodes-us-west1-b-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 128 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "nodes-us-west1-b" + "k8s-io-role-node" = "node" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "nodes-us-west1-b" + "kube-env" = "AUTOSCALER_ENV_VARS: os_distribution=ubuntu;arch=amd64;os=linux" + "user-data" = file("${path.module}/data/google_compute_instance_template_nodes-us-west1-b-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "nodes-us-west1-b-k8s1-equ-01o9jq-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.node.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_only"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_instance_template" "nodes-us-west1-c-k8s1-equivalence-example-internal" { + can_ip_forward = true + disk { + auto_delete = true + boot = true + device_name = "persistent-disks-0" + disk_name = "" + disk_size_gb = 128 + disk_type = "pd-standard" + interface = "" + mode = "READ_WRITE" + provisioned_iops = 0 + provisioned_throughput = 0 + source = "" + source_image = "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/ubuntu-2404-noble-amd64-v20250606" + type = "PERSISTENT" + } + labels = { + "k8s-io-cluster-name" = "k8s1-equivalence-example-internal" + "k8s-io-instance-group" = "nodes-us-west1-c" + "k8s-io-role-node" = "node" + } + lifecycle { + create_before_destroy = true + } + machine_type = "e2-medium" + metadata = { + "cluster-name" = "k8s1.equivalence.example.internal" + "kops-k8s-io-instance-group-name" = "nodes-us-west1-c" + "kube-env" = "AUTOSCALER_ENV_VARS: os_distribution=ubuntu;arch=amd64;os=linux" + "user-data" = file("${path.module}/data/google_compute_instance_template_nodes-us-west1-c-k8s1-equivalence-example-internal_metadata_user-data") + } + name_prefix = "nodes-us-west1-c-k8s1-equ-k6036m-" + network_interface { + network = "default" + stack_type = "IPV4_ONLY" + subnetwork = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + } + scheduling { + automatic_restart = true + on_host_maintenance = "MIGRATE" + preemptible = false + provisioning_model = "STANDARD" + } + service_account { + email = google_service_account.node.email + scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_only"] + } + tags = ["k8s1-equivalence-example-internal-k8s-io-role-node"] +} + +resource "google_compute_region_backend_service" "api-k8s1-equivalence-example-internal" { + backend { + balancing_mode = "CONNECTION" + group = google_compute_instance_group_manager.a-control-plane-us-west1-a-k8s1-equivalence-example-internal.instance_group + } + backend { + balancing_mode = "CONNECTION" + group = google_compute_instance_group_manager.b-control-plane-us-west1-b-k8s1-equivalence-example-internal.instance_group + } + backend { + balancing_mode = "CONNECTION" + group = google_compute_instance_group_manager.c-control-plane-us-west1-c-k8s1-equivalence-example-internal.instance_group + } + health_checks = [google_compute_region_health_check.api-k8s1-equivalence-example-internal.id] + load_balancing_scheme = "INTERNAL" + name = "api-k8s1-equivalence-example-internal" + protocol = "TCP" +} + +resource "google_compute_region_health_check" "api-k8s1-equivalence-example-internal" { + name = "api-k8s1-equivalence-example-internal" + tcp_health_check { + port = 443 + } +} + +resource "google_compute_router" "nat-k8s1-equivalence-example-internal" { + name = "nat-k8s1-equivalence-example-internal" + network = "default" +} + +resource "google_compute_router_nat" "nat-k8s1-equivalence-example-internal" { + name = "nat-k8s1-equivalence-example-internal" + nat_ip_allocate_option = "AUTO_ONLY" + region = "us-west1" + router = google_compute_router.nat-k8s1-equivalence-example-internal.name + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + subnetwork { + name = google_compute_subnetwork.us-west1-k8s1-equivalence-example-internal.name + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } +} + +resource "google_compute_subnetwork" "us-west1-k8s1-equivalence-example-internal" { + ip_cidr_range = "10.0.16.0/20" + name = "us-west1-k8s1-equivalence-example-internal" + network = "default" + region = "us-west1" + stack_type = "IPV4_ONLY" +} + +resource "google_compute_target_pool" "api-k8s1-equivalence-example-internal" { + health_checks = [google_compute_http_health_check.api-k8s1-equivalence-example-internal.self_link] + name = "api-k8s1-equivalence-example-internal" +} + +resource "google_project_iam_member" "serviceaccount-control-plane" { + member = format("serviceAccount:%s", google_service_account.control-plane.email) + project = "example-project" + role = "roles/container.serviceAgent" +} + +resource "google_project_iam_member" "serviceaccount-nodes" { + member = format("serviceAccount:%s", google_service_account.node.email) + project = "example-project" + role = "roles/compute.viewer" +} + +resource "google_service_account" "control-plane" { + account_id = "control-plane-k8s1-equi-5pdea1" + description = "kubernetes control-plane instances" + display_name = "control-plane" + project = "example-project" +} + +resource "google_service_account" "node" { + account_id = "node-k8s1-equivalence-e-5pdea1" + description = "kubernetes worker nodes" + display_name = "node" + project = "example-project" +} + +terraform { + required_version = ">= 0.15.0" + required_providers { + aws = { + "configuration_aliases" = [aws.files] + "source" = "hashicorp/aws" + "version" = ">= 5.0.0" + } + google = { + "source" = "hashicorp/google" + "version" = ">= 5.11.0" + } + } +} From f728baf2e8a536d6ea7f56e9308d343afe5d6fff Mon Sep 17 00:00:00 2001 From: Rauf Guliyev Date: Thu, 23 Jul 2026 22:19:10 +0000 Subject: [PATCH 5/5] Document GCE Terraform migration safeguards Explain the state transition from authoritative IAM bindings and mark the deterministic PKI as public test material. --- docs/terraform.md | 21 +++++++++++++++++++ .../gce-kops-module-equivalence/README.md | 17 +++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/integration/update_cluster/gce-kops-module-equivalence/README.md diff --git a/docs/terraform.md b/docs/terraform.md index 926174c63794b..39d62540904da 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -83,6 +83,27 @@ $ terraform apply Wait for the cluster to initialize. If all goes well, you should have a working Kubernetes cluster! +#### Migrating GCE project IAM resources + +GCE Terraform output now manages the control-plane and node service-account +roles with additive `google_project_iam_member` resources instead of +authoritative `google_project_iam_binding` resources. Existing Terraform state +does not automatically associate the new resource type with the old addresses. + +After regenerating the Terraform configuration, but before planning or applying +it, remove any old addresses that are present from Terraform state: + +``` +$ terraform state rm 'google_project_iam_binding.serviceaccount-control-plane' +$ terraform state rm 'google_project_iam_binding.serviceaccount-nodes' +``` + +Removing these addresses from Terraform state does not remove the IAM grants +from GCP. Review the next plan and confirm it creates the corresponding +`google_project_iam_member` resources without destroying project IAM bindings. +This prevents the old authoritative resources from removing the grants after +the additive resources have been created. + #### Editing the cluster It's possible to use Terraform to make changes to your infrastructure as defined by kOps. In the example below we'd like to change some cluster configs: diff --git a/tests/integration/update_cluster/gce-kops-module-equivalence/README.md b/tests/integration/update_cluster/gce-kops-module-equivalence/README.md new file mode 100644 index 0000000000000..04042402edf66 --- /dev/null +++ b/tests/integration/update_cluster/gce-kops-module-equivalence/README.md @@ -0,0 +1,17 @@ +# GCE kops-module equivalence fixture + +This offline fixture exercises the GCE topology used by kops-module with +generic cluster and project identities. Regenerate it with: + +``` +HACK_UPDATE_EXPECTED_IN_PLACE=1 go test ./cmd/kops \ + -run '^TestGCEKopsModuleEquivalence$' -count=1 +``` + +`golden-pki.json` contains public, deterministic test keys. They are committed +only to make generated node configuration reproducible and must never be used +for a real cluster. + +The `aws_s3_object_*` files represent objects in the integration harness's +in-memory state store. They are expected for GCE Terraform fixtures and do not +exercise AWS infrastructure.