From 6d62f2d5c2283862ffda9e1cc390a8e090825811 Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Wed, 29 Jul 2026 13:41:01 +0200 Subject: [PATCH 1/3] remove special case deployer switch --- cmd/deploy_test.go | 31 ++++++++++--------- pkg/deployers/deployers.go | 13 ++++---- pkg/deployers/deployers_test.go | 54 ++++++++++++++++++--------------- pkg/functions/client.go | 2 -- pkg/functions/client_test.go | 6 ++-- 5 files changed, 56 insertions(+), 50 deletions(-) diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index 0b2446b1fe..ec79c69db7 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -2664,26 +2664,27 @@ func TestDeploy_DeployerGlobalConfig(t *testing.T) { } } -// TestDeploy_DeployerSwitch pins the CLI-only part of the switch guard. +// TestDeploy_DeployerSwitch pins two things no lower-level test reaches: that a +// blocked switch surfaces its error through the whole CLI path, and the CLI's +// own resolution of a deployer ValidateSwitch would call unknown. func TestDeploy_DeployerSwitch(t *testing.T) { for _, tt := range []struct { name string - deployedDep string // Deploy.Deployer already deployed ("" = legacy) - requested string // --deployer flag value + deployedDep string // Deploy.Deployer already deployed ("" = unrecorded) + requested string // --deployer flag value ("" = flag omitted) wantBlocked bool }{ - // A blocked and an allowed switch, proving the guard is reached and its - // error surfaced through the full CLI path (flag -> config -> client). - {"keda2raw blocked", keda.KedaDeployerName, k8s.KubernetesDeployerName, true}, - {"raw2keda safe switch", k8s.KubernetesDeployerName, keda.KedaDeployerName, false}, - // Legacy (deployed before the deployer was persisted, so empty): the CLI - // treats it as knative, so an explicit knative is allowed but any other - // deployer is blocked. This normalization is the CLI-only bit here. - {"legacy empty is treated as knative", "", deployers.Knative, false}, - {"legacy empty to raw is blocked", "", k8s.KubernetesDeployerName, true}, - // A function deployed by an older binary has its deployer only in state, - // not intent. A flag-less redeploy must reuse it. - {"legacy keda reused on flag-less redeploy", keda.KedaDeployerName, "", false}, + // the policy is pinned in pkg/deployers. We test that the guard is + // reached at all through flag -> config -> client, and that its error + // reaches the user. + {"cross-deployer switch is blocked", keda.KedaDeployerName, k8s.KubernetesDeployerName, true}, + // ValidateSwitch treats an empty deployer as "not known" and allows it. + // The CLI never lets one through: a deployed function without a recorded + // deployer is resolved to knative, and an omitted flag is resolved to + // whatever is already deployed. + {"unrecorded deployer is treated as knative", "", deployers.Knative, false}, + {"unrecorded deployer to raw is blocked", "", k8s.KubernetesDeployerName, true}, + {"omitted flag reuses the deployed one", keda.KedaDeployerName, "", false}, } { t.Run(tt.name, func(t *testing.T) { root := FromTempDirectory(t) diff --git a/pkg/deployers/deployers.go b/pkg/deployers/deployers.go index a24af9302d..e64c75fc01 100644 --- a/pkg/deployers/deployers.go +++ b/pkg/deployers/deployers.go @@ -15,18 +15,19 @@ const ( Default = Knative ) -// ValidateSwitch reports an error if redeploying an already-deployed function -// with deployer 'to' would strand the previous deployer's resources on the -// cluster. The only safe cross-deployer change is raw -> keda, because the keda -// deployer embeds the raw one; same-deployer redeploys are always allowed. +// ValidateSwitch reports an error if an already-deployed function is being +// redeployed with a different deployer. Any change of deployer is refused +// because switching is not supported: nothing reconciles one deployer's +// resources into another's, so the user has to run func delete first and then +// redeploy. Same-deployer redeploys are allowed. // 'from' is the deployer the function is currently deployed with. 'to' is the // one to deploy to. An empty value on either side means "not known" -> returns nil. func ValidateSwitch(from, to string) error { if from == "" || to == "" { return nil } - if from == to || (from == Kubernetes && to == Keda) { + if from == to { return nil } - return fmt.Errorf("function was deployed with the %q deployer; redeploying with %q would orphan the old deployer's resources on the cluster - run func delete first to remove them, then redeploy", from, to) + return fmt.Errorf("function was deployed with the %q deployer; redeploying with %q is not supported. Run func delete first, then redeploy", from, to) } diff --git a/pkg/deployers/deployers_test.go b/pkg/deployers/deployers_test.go index 053e5ece83..ad3e826c46 100644 --- a/pkg/deployers/deployers_test.go +++ b/pkg/deployers/deployers_test.go @@ -3,35 +3,41 @@ package deployers import "testing" // TestValidateSwitch covers the deployer-switch policy: the same deployer is -// always allowed, raw -> keda is the one safe cross-switch (keda embeds raw), -// and every other change is blocked. The undeployed case (any deployer allowed) -// is the caller's responsibility and is covered by the cmd-level deploy tests. +// always allowed and any change of deployer is blocked. The undeployed case +// (any deployer allowed) is the caller's responsibility and is covered by the +// cmd-level deploy tests. func TestValidateSwitch(t *testing.T) { + // policy: any re-deployment of a function with different deployer is blocked + all := []string{Knative, Kubernetes, Keda} + + for _, from := range all { + for _, to := range all { + t.Run(from+" to "+to, func(t *testing.T) { + err := ValidateSwitch(from, to) + if from == to && err != nil { + t.Fatalf("expected the same deployer to be a no-op, got: %v", err) + } + if from != to && err == nil { + t.Fatalf("expected %q->%q to be blocked, got nil", from, to) + } + }) + } + } + + // An empty deployer means "not known", not "a deployer named empty": + // no switch can be established, so none is reported. Guards library + // callers, which have no CLI to resolve either side for them. for _, tt := range []struct { - name string - from string - to string - wantErr bool + name string + from string + to string }{ - {"same deployer is a no-op", Keda, Keda, false}, - {"raw to keda is the one safe switch", Kubernetes, Keda, false}, - {"keda to raw is blocked", Keda, Kubernetes, true}, - {"knative to raw is blocked", Knative, Kubernetes, true}, - {"knative to keda is blocked", Knative, Keda, true}, - {"keda to knative is blocked", Keda, Knative, true}, - // An empty deployer means "not known", not "a deployer named empty": - // no switch can be established, so none is reported. Guards library - // callers, which have no CLI to resolve either side for them. - {"unknown deployed-with is not a switch", "", Keda, false}, - {"unknown requested is not a switch", Keda, "", false}, - {"both unknown is not a switch", "", "", false}, + {"unknown deployed-with is not a switch", "", Keda}, + {"unknown requested is not a switch", Keda, ""}, + {"both unknown is not a switch", "", ""}, } { t.Run(tt.name, func(t *testing.T) { - err := ValidateSwitch(tt.from, tt.to) - if tt.wantErr && err == nil { - t.Fatalf("expected %q->%q to be blocked, got nil", tt.from, tt.to) - } - if !tt.wantErr && err != nil { + if err := ValidateSwitch(tt.from, tt.to); err != nil { t.Fatalf("expected %q->%q to be allowed, got: %v", tt.from, tt.to, err) } }) diff --git a/pkg/functions/client.go b/pkg/functions/client.go index a4b2902ff9..ca48686e19 100644 --- a/pkg/functions/client.go +++ b/pkg/functions/client.go @@ -867,8 +867,6 @@ func (c *Client) Deploy(ctx context.Context, f Function, oo ...DeployOption) (Fu // deployed would leave stranded resources on cluster. We error clearly // and expect the user to undeploy first, which removes the resources // correctly. - // One special case is raw -> keda switch which works because keda embeds - // 'raw' deployer, working with the same resources. if f.Deploy.Namespace != "" { if err := deployers.ValidateSwitch(f.Deploy.Deployer, f.Deployer); err != nil { return f, fmt.Errorf("function %q: %w", f.Name, err) diff --git a/pkg/functions/client_test.go b/pkg/functions/client_test.go index 4cc49242ec..6da4a0bdd9 100644 --- a/pkg/functions/client_test.go +++ b/pkg/functions/client_test.go @@ -2580,8 +2580,8 @@ func absPath(p string) string { // TestClient_Deploy_BlocksDeployerSwitch ensures the deployer-switch guard is // enforced by the client itself, so every API consumer is protected, not only // the CLI. Redeploying an already-deployed function with a different deployer -// would strand the previous deployer's resources; raw -> keda is the one safe -// change because the keda deployer embeds the raw one. +// would strand the previous deployer's resources, so every change of deployer +// is refused and the user is told to run func delete first. // // A blocked switch must also fail BEFORE the deployer runs: the point of the // guard is that nothing on the cluster is touched. @@ -2595,7 +2595,7 @@ func TestClient_Deploy_BlocksDeployerSwitch(t *testing.T) { }{ {"keda2raw blocked", deployers.Keda, deployers.Kubernetes, "ns", true}, {"knative2keda blocked", deployers.Knative, deployers.Keda, "ns", true}, - {"raw2keda is safe switch", deployers.Kubernetes, deployers.Keda, "ns", false}, + {"raw2keda blocked", deployers.Kubernetes, deployers.Keda, "ns", true}, {"same deployer is not a switch", deployers.Keda, deployers.Keda, "ns", false}, {"undeployed is never blocked", "", deployers.Keda, "", false}, From 9b60176681413e25a5e9093207bb1da8ba281241 Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Sun, 5 Jul 2026 20:13:47 +0200 Subject: [PATCH 2/3] feat: expose raw k8s deployer via OpenShift Route by default, gated to OCP --- cmd/completion_util.go | 20 ++ cmd/deploy.go | 42 ++- cmd/deploy_test.go | 210 +++++++++++++ cmd/errors.go | 29 ++ docs/reference/func_deploy.md | 1 + e2e/e2e_metadata_test.go | 2 +- e2e/e2e_trigger_sync_test.go | 6 +- .../testing/integration_test_helper.go | 15 +- .../testing/integration_test_helper.go | 1 + pkg/functions/client.go | 8 +- pkg/functions/client_test.go | 36 +++ pkg/functions/errors.go | 1 + pkg/functions/function.go | 11 + pkg/functions/function_expose.go | 21 ++ pkg/functions/function_expose_unit_test.go | 30 ++ pkg/k8s/deployer.go | 223 +++++++++++++- pkg/k8s/deployer_test.go | 84 +++++ pkg/k8s/describer.go | 16 +- pkg/k8s/lister.go | 7 +- pkg/k8s/route.go | 268 ++++++++++++++++ pkg/k8s/route_test.go | 290 ++++++++++++++++++ pkg/keda/deployer.go | 5 + pkg/knative/deployer.go | 2 +- pkg/lister/testing/integration_test_helper.go | 4 + pkg/mock/deployer.go | 3 + pkg/pipelines/tekton/pipelines_provider.go | 13 +- .../testing/integration_test_helper.go | 1 + schema/func_yaml-schema.json | 4 + 28 files changed, 1319 insertions(+), 34 deletions(-) create mode 100644 pkg/functions/function_expose.go create mode 100644 pkg/functions/function_expose_unit_test.go create mode 100644 pkg/k8s/route.go create mode 100644 pkg/k8s/route_test.go diff --git a/cmd/completion_util.go b/cmd/completion_util.go index cffb092534..014f4136b1 100644 --- a/cmd/completion_util.go +++ b/cmd/completion_util.go @@ -190,3 +190,23 @@ func CompleteDeployerList(cmd *cobra.Command, args []string, complete string) (m return } + +func CompleteExposeList(cmd *cobra.Command, args []string, complete string) (matches []string, d cobra.ShellCompDirective) { + values := []string{"none", "route"} + + d = cobra.ShellCompDirectiveNoFileComp + matches = []string{} + + if len(complete) == 0 { + matches = values + return + } + + for _, v := range values { + if strings.HasPrefix(v, complete) { + matches = append(matches, v) + } + } + + return +} diff --git a/cmd/deploy.go b/cmd/deploy.go index 50ec7647c6..169a1890b9 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -132,10 +132,10 @@ EXAMPLES SuggestFor: []string{"delpoy", "deplyo"}, PreRunE: bindEnv("build", "build-timestamp", "builder", "builder-image", "base-image", "confirm", "domain", "env", "git-branch", "git-dir", - "git-url", "image", "image-pull-secret", "management-disabled", "namespace", "path", "platform", "push", "pvc-size", - "service-account", "deployer", "registry", "registry-insecure", - "registry-authfile", "remote", "username", "password", "token", "verbose", - "remote-storage-class"), + "git-url", "image", "image-pull-secret", "management-disabled", + "namespace", "path", "platform", "push", "pvc-size", "service-account", + "deployer", "expose", "registry", "registry-insecure", "registry-authfile", + "remote", "username", "password", "token", "verbose", "remote-storage-class"), RunE: func(cmd *cobra.Command, args []string) error { return runDeploy(cmd, newClient) }, @@ -200,6 +200,9 @@ EXAMPLES "Service account to be used in the deployed function ($FUNC_SERVICE_ACCOUNT)") cmd.Flags().String("image-pull-secret", f.Deploy.ImagePullSecret, "Image pull secret to use when the function's image is in a private registry ($FUNC_IMAGE_PULL_SECRET)") + cmd.Flags().String("expose", f.Deploy.Expose, + "External exposure mode: 'route' (create a Route; OpenShift cluster only), "+ + "'none' (cluster-local opt-out). Raw and keda deployers only. ") // Static Flags: // Options which have static defaults only (not globally configurable nor // persisted with the function) @@ -240,6 +243,10 @@ EXAMPLES fmt.Println("internal: error while calling RegisterFlagCompletionFunc: ", err) } + if err := cmd.RegisterFlagCompletionFunc("expose", CompleteExposeList); err != nil { + fmt.Println("internal: error while calling RegisterFlagCompletionFunc: ", err) + } + return cmd } @@ -285,6 +292,9 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) (err error) { // Warn if registry changed but registryInsecure is still true warnRegistryInsecureChange(cmd.OutOrStderr(), cfg.Registry, f) + // Warn if expose flag is used with deployer where it has no effect + warnExposeIgnore(cmd.OutOrStderr(), cfg.Expose, cfg.Deployer) + // Back-compat: a function deployed before the deployer was recorded has a // namespace but no deployer, which historically could only mean knative. if f.Deploy.Namespace != "" && f.Deploy.Deployer == "" { @@ -570,6 +580,11 @@ type deployConfig struct { // ManagementDisabled disables automatic Function CR sync after deploy. ManagementDisabled bool + + // Expose controls external access - how/if the function should be + // exposed externally. Defaults to exposed on OpenShift, cluster-local + // elsewhere; "none" opts out explicitly. + Expose string } // newDeployConfig creates a buildConfig populated from command flags and @@ -592,6 +607,7 @@ func newDeployConfig(cmd *cobra.Command) deployConfig { ImagePullSecret: viper.GetString("image-pull-secret"), Deployer: viper.GetString("deployer"), ManagementDisabled: viper.GetBool("management-disabled"), + Expose: viper.GetString("expose"), } // NOTE: .Env should be viper.GetStringSlice, but this returns unparsed // results and appears to be an open issue since 2017: @@ -629,6 +645,7 @@ func (c deployConfig) Configure(f fn.Function) (fn.Function, error) { f.Deploy.ImagePullSecret = c.ImagePullSecret f.Deployer = c.Deployer f.Deploy.ManagementDisabled = c.ManagementDisabled + f.Deploy.Expose = c.Expose f.Local.Remote = c.Remote // PVCSize @@ -748,6 +765,11 @@ func (c deployConfig) Validate(cmd *cobra.Command) (err error) { } } + // Validate expose flag if provided + if err = fn.ValidateExpose(c.Expose); err != nil { + return err + } + // Check Image Digest was included var digest bool if c.Image != "" { @@ -909,3 +931,15 @@ func isDigested(v string) (validDigest bool, err error) { _, ok := ref.(name.Digest) return ok, nil } + +// warnExposeIgnore warns when non raw|keda deployer is used with Expose flag +// where it is simply ignored and has no effect. An empty deployer means the +// default (knative), which also ignores expose. Knative has its own exposure +// mechanism. +func warnExposeIgnore(w io.Writer, expose, deployer string) { + if expose != "" && deployer != k8s.KubernetesDeployerName && + deployer != keda.KedaDeployerName { + fmt.Fprintf(w, "warning: deploy.expose %q is ignored - only the raw and keda deployers "+ + "support external exposure via this field.\n", expose) + } +} diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index ec79c69db7..f7bdac3f96 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "reflect" "strings" @@ -2730,3 +2731,212 @@ func TestDeploy_DeployerSwitch(t *testing.T) { }) } } + +// TestDeploy_ExposeEmptyVsUnset: an explicitly empty --expose="" +// clears the persisted deploy.expose key reverting to the default at deploy +// time, while a deploy without the flag leaves the persisted value untouched. +func TestDeploy_ExposeEmptyVsUnset(t *testing.T) { + // newFn initializes a Go function in a temp directory and returns its root. + newFn := func(t *testing.T) string { + t.Helper() + root := FromTempDirectory(t) + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + return root + } + + // deploy runs `func deploy` with args against mock builder/deployer, + // failing the test on error and returning the command's combined output. + deploy := func(t *testing.T, args ...string) string { + t.Helper() + cmd := NewDeployCmd(NewTestClient( + fn.WithBuilder(mock.NewBuilder()), + fn.WithDeployer(mock.NewDeployer()), + fn.WithRegistry(TestRegistry), + )) + cmd.SetArgs(args) + var out strings.Builder + cmd.SetOut(&out) + cmd.SetErr(&out) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + return out.String() + } + + // loadFn re-reads the function from disk. + loadFn := func(t *testing.T, root string) fn.Function { + t.Helper() + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + return f + } + + t.Run(`--expose="" clears a previously-persisted "none"`, func(t *testing.T) { + root := newFn(t) + + deploy(t, "--deployer", "raw", "--expose", "none") + if f := loadFn(t, root); f.Deploy.Expose != "none" { + t.Fatalf("setup: expected expose 'none' to be persisted, got %q", f.Deploy.Expose) + } + + deploy(t, "--deployer", "raw", "--expose=") + // unmarshalled yaml would not be able to distinguish between the value + // being empty and gone (not in the file) + raw, err := os.ReadFile(filepath.Join(root, "func.yaml")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "expose") { + t.Errorf("expected NO expose key in func.yaml, got:\n%s", raw) + } + }) + + t.Run("plain deploy without the flag still works and leaves expose unpersisted", func(t *testing.T) { + root := newFn(t) + deploy(t, "--deployer", "raw") + if f := loadFn(t, root); f.Deploy.Expose != "" { + t.Errorf("expected expose to remain unpersisted (empty), got %q", f.Deploy.Expose) + } + }) + + t.Run("persisted none + no flag round-trips untouched", func(t *testing.T) { + root := newFn(t) + + deploy(t, "--deployer", "raw", "--expose", "none") + if f := loadFn(t, root); f.Deploy.Expose != "none" { + t.Fatalf("expected expose 'none' to be persisted, got %q", f.Deploy.Expose) + } + + // redeploy without changing the flag should keep it as is + deploy(t, "--deployer", "raw") + if f := loadFn(t, root); f.Deploy.Expose != "none" { + t.Errorf("expected persisted 'none' to round-trip untouched, got %q", f.Deploy.Expose) + } + }) +} + +// TestDeploy_ExposeInvalidValueError: a malformed --expose value fails the +// deploy (any deployer) with the CLI's typed ErrInvalidExpose. +func TestDeploy_ExposeInvalidValueError(t *testing.T) { + root := FromTempDirectory(t) + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + cmd := NewDeployCmd(NewTestClient( + fn.WithBuilder(mock.NewBuilder()), + fn.WithDeployer(mock.NewDeployer()), + fn.WithRegistry(TestRegistry), + )) + cmd.SetArgs([]string{"--expose", "bogus"}) + var want *ErrInvalidExpose + if err := cmd.Execute(); !errors.As(err, &want) { + t.Errorf("expected ErrInvalidExpose, got %v", err) + } +} + +// TestDeploy_ExposeRoutePersists ensures "route" round-trips through +// --expose into f.Deploy.Expose end-to-end. +func TestDeploy_ExposeRoutePersists(t *testing.T) { + root := FromTempDirectory(t) + + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + + cmd := NewDeployCmd(NewTestClient( + fn.WithBuilder(mock.NewBuilder()), + fn.WithDeployer(mock.NewDeployer()), + fn.WithRegistry(TestRegistry), + )) + cmd.SetArgs([]string{"--deployer", "raw", "--expose=route"}) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Deploy.Expose != "route" { + t.Fatalf("expected expose 'route' to be persisted, got %q", f.Deploy.Expose) + } +} + +// TestDeploy_ExposeIgnoredByDeployerNote: a deployer that ignores a set +// deploy.expose warns and proceeds +func TestDeploy_ExposeIgnoredByDeployerNote(t *testing.T) { + tests := []struct { + name string + args []string + wantWarning string // distinguishing substring of the warning; "" means silent + }{ + { + name: "raw+route: silent", + args: []string{"--deployer", "raw", "--expose", "route"}, + }, + { + name: "knative+empty: silent", + args: []string{"--deployer", "knative"}, + }, + { + name: "knative+route: warns, proceeds", + args: []string{"--deployer", "knative", "--expose", "route"}, + wantWarning: `deploy.expose "route" is ignored - only the raw and keda deployers support external exposure via this field.`, + }, + { + name: "knative+none: warns, proceeds", + args: []string{"--deployer", "knative", "--expose", "none"}, + wantWarning: `deploy.expose "none" is ignored - only the raw and keda deployers support external exposure via this field.`, + }, + { + name: "keda+route: silent, keda supports expose too", + args: []string{"--deployer", "keda", "--expose", "route"}, + }, + { + name: "keda+none: silent, keda supports expose too", + args: []string{"--deployer", "keda", "--expose", "none"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := FromTempDirectory(t) + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + + builder := mock.NewBuilder() + cmd := NewDeployCmd(NewTestClient( + fn.WithBuilder(builder), + fn.WithDeployer(mock.NewDeployer()), + fn.WithRegistry(TestRegistry), + )) + cmd.SetArgs(tt.args) + var stderr strings.Builder + cmd.SetOut(&stderr) + cmd.SetErr(&stderr) + err := cmd.Execute() + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !builder.BuildInvoked { + t.Error("expected the deploy to proceed to build") + } + + if tt.wantWarning == "" { + if strings.Contains(stderr.String(), "deploy.expose") { + t.Errorf("expected no warning on stderr, got:\n%s", stderr.String()) + } + return + } + if !strings.Contains(stderr.String(), tt.wantWarning) { + t.Errorf("expected stderr to contain:\n%s\ngot:\n%s", tt.wantWarning, stderr.String()) + } + }) + } +} diff --git a/cmd/errors.go b/cmd/errors.go index e820df1d96..f99ed9db3b 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -31,6 +31,9 @@ Internal error during error-wrapping: specified cmd '%s' not supported`, cmd) if errors.Is(err, fn.ErrPlatformNotSupported) { return NewErrPlatformNotSupported(err, cmd) } + if errors.Is(err, fn.ErrInvalidExpose) { + return NewErrInvalidExpose(err) + } return err } @@ -217,6 +220,32 @@ func (e *ErrInvalidDomain) Unwrap() error { // -------------------------------------------------------------------------- // +type ErrInvalidExpose struct { + Err error +} + +func NewErrInvalidExpose(err error) error { + return &ErrInvalidExpose{Err: err} +} + +func (e *ErrInvalidExpose) Error() string { + return fmt.Sprintf(`%v + +Try this: + func deploy --expose=route Create an OpenShift Route (OpenShift clusters only) + func deploy --expose=none Cluster-local opt-out, no external exposure + +deploy.expose takes effect with the raw and keda deployers only (--deployer=raw or --deployer=keda), +which expose by default when the platform and deployer support it. +For more options, run 'func deploy --help'`, e.Err) +} + +func (e *ErrInvalidExpose) Unwrap() error { + return e.Err +} + +// -------------------------------------------------------------------------- // + type ErrInvalidKubeconfig struct { Err error } diff --git a/docs/reference/func_deploy.md b/docs/reference/func_deploy.md index b8257073ad..9ecd104022 100644 --- a/docs/reference/func_deploy.md +++ b/docs/reference/func_deploy.md @@ -122,6 +122,7 @@ func deploy --deployer string Type of deployment to use: 'knative' for Knative Service, 'raw' for Kubernetes Deployment, or 'keda' for Deployment with a KEDA HTTP scaler ($FUNC_DEPLOYER) (default "knative") --domain string Domain to use for the function's route. Cluster must be configured with domain matching for the given domain (ignored if unrecognized) ($FUNC_DOMAIN) -e, --env stringArray Environment variable to set in the form NAME=VALUE. You may provide this flag multiple times for setting multiple environment variables. To unset, specify the environment variable name followed by a "-" (e.g., NAME-). + --expose string External exposure mode: 'route' (create a Route; OpenShift clusters only), 'none' (cluster-local opt-out). Raw and keda deployers only. Defaults to exposed on OpenShift, cluster-local elsewhere. An explicitly empty value (--expose="") clears the persisted deploy.expose key and returns to the default. ($FUNC_EXPOSE) -t, --git-branch string Git revision (branch) to be used when deploying via the Git repository ($FUNC_GIT_BRANCH) -d, --git-dir string Directory in the Git repository containing the function (default is the root) ($FUNC_GIT_DIR) -g, --git-url string Repository url containing the function to build ($FUNC_GIT_URL) diff --git a/e2e/e2e_metadata_test.go b/e2e/e2e_metadata_test.go index 63c376558b..98ad6da5b0 100644 --- a/e2e/e2e_metadata_test.go +++ b/e2e/e2e_metadata_test.go @@ -662,7 +662,7 @@ func TestMetadata_Subscriptions_Raw(t *testing.T) { } // Deploy with raw deployer to test trigger creation - if err := newCmd(t, "deploy", "--deployer", "raw").Run(); err != nil { + if err := newCmd(t, "deploy", "--deployer", "raw", "--expose", "none").Run(); err != nil { t.Fatal(err) } defer clean(t, subscriberName, Namespace) diff --git a/e2e/e2e_trigger_sync_test.go b/e2e/e2e_trigger_sync_test.go index 72953b0d1c..019eef09db 100644 --- a/e2e/e2e_trigger_sync_test.go +++ b/e2e/e2e_trigger_sync_test.go @@ -50,7 +50,7 @@ func TestMetadata_TriggerSync(t *testing.T) { if err := f.Write(); err != nil { t.Fatal(err) } - if err := newCmd(t, "deploy", "--deployer", "raw").Run(); err != nil { + if err := newCmd(t, "deploy", "--deployer", "raw", "--expose", "none").Run(); err != nil { t.Fatal(err) } } @@ -109,7 +109,7 @@ func TestMetadata_TriggerSync(t *testing.T) { t.Logf("Created manual trigger: %s", manualTriggerName) // Redeploy (no changes to subscriptions) - if err := newCmd(t, "deploy", "--deployer", "raw").Run(); err != nil { + if err := newCmd(t, "deploy", "--deployer", "raw", "--expose", "none").Run(); err != nil { t.Fatal(err) } time.Sleep(5 * time.Second) @@ -171,7 +171,7 @@ func TestMetadata_TriggerSync(t *testing.T) { // (AlreadyExists-tolerated) cluster path; trigger-name determinism is // already exhaustively unit-tested (pkg/k8s/deployer_test.go:157-333), // so the previous ×3 loop is reduced to ×1. - if err := newCmd(t, "deploy", "--deployer", "raw").Run(); err != nil { + if err := newCmd(t, "deploy", "--deployer", "raw", "--expose", "none").Run(); err != nil { t.Fatal(err) } time.Sleep(3 * time.Second) diff --git a/pkg/deployer/testing/integration_test_helper.go b/pkg/deployer/testing/integration_test_helper.go index 1c5b9158c3..9a4aaff3c3 100644 --- a/pkg/deployer/testing/integration_test_helper.go +++ b/pkg/deployer/testing/integration_test_helper.go @@ -62,6 +62,10 @@ func TestInt_Deploy(t *testing.T, deployer fn.Deployer, remover fn.Remover, desc Runtime: "go", Namespace: ns, Registry: Registry(), + // Explicit opt-out: keeps this integration deploy cluster-local and + // platform-deterministic under exposed-by-default; ignored entirely + // by the knative deployer. + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -165,6 +169,7 @@ func TestInt_Metadata(t *testing.T, deployer fn.Deployer, remover fn.Remover, de Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -320,6 +325,7 @@ func TestInt_Events(t *testing.T, deployer fn.Deployer, remover fn.Remover, desc Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -403,6 +409,7 @@ func TestInt_Scale(t *testing.T, deployer fn.Deployer, remover fn.Remover, descr Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -518,6 +525,7 @@ func TestInt_EnvsUpdate(t *testing.T, deployer fn.Deployer, remover fn.Remover, Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -731,10 +739,11 @@ func TestInt_FullPath(t *testing.T, deployer fn.Deployer, remover fn.Remover, li // * application also prints the same info to stderr on startup Created: now, Deploy: fn.DeploySpec{ - // TODO: gauron99 - is it okay to have this explicitly set to deploy.image already? - // With this I skip the logic of setting the .Deploy.Image field but it should be fine for this test + // pinned prebuilt image: these tests exercise deployment, not the + // build/image-resolution flow Image: "quay.io/mvasek/func-test-service@sha256:2eca4de00d7569c8791634bdbb0c4d5ec8fb061b001549314591e839dabd5269", Namespace: namespace, + Expose: "none", Labels: []fn.Label{{Key: ptr("my-label"), Value: ptr("my-label-value")}}, Options: fn.Options{ Scale: &fn.ScaleOptions{ @@ -927,6 +936,7 @@ func TestInt_ResourceValidationOnFirstDeploy(t *testing.T, deployer fn.Deployer, Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) @@ -1234,6 +1244,7 @@ func TestInt_OperatorSync(t *testing.T, deployer fn.Deployer, remover fn.Remover Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) diff --git a/pkg/describer/testing/integration_test_helper.go b/pkg/describer/testing/integration_test_helper.go index 45638f86fb..5d57cbdc0b 100644 --- a/pkg/describer/testing/integration_test_helper.go +++ b/pkg/describer/testing/integration_test_helper.go @@ -38,6 +38,7 @@ func TestInt_Describe(t *testing.T, describer fn.Describer, deployer fn.Deployer Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) diff --git a/pkg/functions/client.go b/pkg/functions/client.go index ca48686e19..da54dc9a9f 100644 --- a/pkg/functions/client.go +++ b/pkg/functions/client.go @@ -187,8 +187,8 @@ type Describer interface { type Instance struct { // Route is the primary route of a function instance. Route string - // Routes is the primary route plus any other route at which the function - // can be contacted. + // Routes is the primary route first (external when exposed), plus any + // other route at which the function can be contacted. Routes []string `json:"routes" yaml:"routes"` Name string `json:"name" yaml:"name"` Image string `json:"image" yaml:"image"` @@ -917,9 +917,9 @@ func (c *Client) Deploy(ctx context.Context, f Function, oo ...DeployOption) (Fu switch result.Status { case Deployed: - fmt.Fprintf(os.Stderr, "✅ Function deployed in namespace %q and exposed at URL: \n %v\n", result.Namespace, result.URL) + fmt.Fprintf(os.Stderr, "✅ Function deployed in namespace %q at URL: \n %v\n", result.Namespace, result.URL) case Updated: - fmt.Fprintf(os.Stderr, "✅ Function updated in namespace %q and exposed at URL: \n %v\n", result.Namespace, result.URL) + fmt.Fprintf(os.Stderr, "✅ Function updated in namespace %q at URL: \n %v\n", result.Namespace, result.URL) default: } diff --git a/pkg/functions/client_test.go b/pkg/functions/client_test.go index 6da4a0bdd9..f509c2ecdc 100644 --- a/pkg/functions/client_test.go +++ b/pkg/functions/client_test.go @@ -1603,6 +1603,42 @@ func TestClient_Deploy_UnbuiltErrors(t *testing.T) { } } +// TestClient_Deploy_PrintsResultMessage asserts Deploy prints the deploy +// status message (namespace + URL) to stderr on success. +func TestClient_Deploy_PrintsResultMessage(t *testing.T) { + root, rm := Mktemp(t) + defer rm() + f, err := fn.New().Init(fn.Function{Runtime: TestRuntime, Name: "f", Root: root}) + if err != nil { + t.Fatal(err) + } + + deployer := mock.NewDeployerWithResult(fn.DeploymentResult{ + Status: fn.Deployed, + Namespace: TestNamespace, + URL: "http://f.example.com", + }) + client := fn.New(fn.WithDeployer(deployer)) + + old := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + + _, err = client.Deploy(t.Context(), f, fn.WithDeploySkipBuildCheck(true)) + + w.Close() + os.Stderr = old + if err != nil { + t.Fatal(err) + } + + var buf [4096]byte + n, _ := r.Read(buf[:]) + if output := string(buf[:n]); !strings.Contains(output, "deployed in namespace") || !strings.Contains(output, "http://f.example.com") { + t.Errorf("expected stderr to contain the deploy message and URL, got: %q", output) + } +} + // TestClient_New_BuilderImagesPersisted Asserts that the client preserves user- // provided Builder Images func TestClient_New_BuildersPersisted(t *testing.T) { diff --git a/pkg/functions/errors.go b/pkg/functions/errors.go index 1829bddb03..b52cc3d137 100644 --- a/pkg/functions/errors.go +++ b/pkg/functions/errors.go @@ -10,6 +10,7 @@ import ( var ( ErrEnvironmentNotFound = errors.New("environment not found") ErrFunctionNotFound = errors.New("function not found") + ErrInvalidExpose = errors.New("invalid deploy.expose value") ErrMismatchedName = errors.New("name passed the function source") ErrNameRequired = errors.New("name required") ErrNamespaceRequired = errors.New("namespace required") diff --git a/pkg/functions/function.go b/pkg/functions/function.go index 02bd45d2e4..f9273760b2 100644 --- a/pkg/functions/function.go +++ b/pkg/functions/function.go @@ -277,6 +277,17 @@ type DeploySpec struct { // for operator management after deploy. The zero value (false) means // the function is managed by default when the func-operator is installed. ManagementDisabled bool `yaml:"managementDisabled,omitempty"` + + // Expose controls external access for the raw and keda deployers (the + // knative deployer manages its own exposure and ignores it). Optional. + // Values: "route" (create an OpenShift Route; OpenShift clusters only - + // a hard error elsewhere), "none" (cluster-local only, explicit + // opt-out). Defaults to "route" behavior on OpenShift - a deployed + // function being externally reachable is the expected outcome - and to + // cluster-local on any other cluster, since a Route is an + // OpenShift-only mechanism and the unset default must not impose a + // platform requirement. + Expose string `yaml:"expose,omitempty"` } // HealthEndpoints specify the liveness and readiness endpoints for a Runtime diff --git a/pkg/functions/function_expose.go b/pkg/functions/function_expose.go new file mode 100644 index 0000000000..104bddcb74 --- /dev/null +++ b/pkg/functions/function_expose.go @@ -0,0 +1,21 @@ +package functions + +import ( + "fmt" +) + +// ValidateExpose reports whether expose is a valid deploy.expose value: "" +// (default - exposed via an OpenShift Route, since a deployed function +// being reachable is the expected outcome; cluster-local on non-OpenShift +// clusters, since a Route is an OpenShift-only mechanism), "none" +// (cluster-local, explicit opt-out), or "route" (explicit request for an +// OpenShift Route). There is no ref suffix: an OpenShift Route has no +// concept of "which ingress controller to attach to" - the cluster's +// IngressController picks the router, and the Route object doesn't +// reference one. Any other value is rejected. +func ValidateExpose(expose string) error { + if expose == "" || expose == "none" || expose == "route" { + return nil + } + return fmt.Errorf("%w: %q", ErrInvalidExpose, expose) +} diff --git a/pkg/functions/function_expose_unit_test.go b/pkg/functions/function_expose_unit_test.go new file mode 100644 index 0000000000..0af532c640 --- /dev/null +++ b/pkg/functions/function_expose_unit_test.go @@ -0,0 +1,30 @@ +package functions + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +func Test_ValidateExpose(t *testing.T) { + for _, v := range []string{"", "route", "none"} { + t.Run(v, func(t *testing.T) { + if err := ValidateExpose(v); err != nil { + t.Fatalf("ValidateExpose(%q): unexpected error: %v", v, err) + } + }) + } + + for _, v := range []string{"auto", "bogus", "ingress"} { + t.Run(v, func(t *testing.T) { + err := ValidateExpose(v) + if !errors.Is(err, ErrInvalidExpose) { + t.Fatalf("ValidateExpose(%q): expected errors.Is(err, ErrInvalidExpose), got %v", v, err) + } + if !strings.Contains(err.Error(), fmt.Sprintf("%q", v)) { + t.Errorf("ValidateExpose(%q): expected error to quote the bad value, got %v", v, err) + } + }) + } +} diff --git a/pkg/k8s/deployer.go b/pkg/k8s/deployer.go index b7102557a7..231cd44a68 100644 --- a/pkg/k8s/deployer.go +++ b/pkg/k8s/deployer.go @@ -19,8 +19,10 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" clienteventingv1 "knative.dev/client/pkg/eventing/v1" eventingv1 "knative.dev/eventing/pkg/apis/eventing/v1" eventingv1client "knative.dev/eventing/pkg/client/clientset/versioned/typed/eventing/v1" @@ -38,6 +40,11 @@ const ( DefaultReadinessEndpoint = "/health/readiness" DefaultHTTPPort = 8080 + // RouteHostnameAnnotation records the externally-exposed hostname (if + // any) on the function's Service, so lister/describer can read it back + // without re-deriving or re-querying the Route. + RouteHostnameAnnotation = "function.knative.dev/route-hostname" + // managedByAnnotation identifies triggers managed by this deployer managedByAnnotation = "func.knative.dev/managed-by" managedByValue = "func-raw-deployer" @@ -48,6 +55,11 @@ type DeployerOpt func(*Deployer) type Deployer struct { verbose bool decorator deployer.DeployDecorator + + // exposureDisabled marks a Deployer embedded by another deployer (keda) + // whose functions must stay cluster-local: a Route pointed at the + // raw ClusterIP Service would bypass keda's scale-to-zero interceptor. + exposureDisabled bool } func NewDeployer(opts ...DeployerOpt) *Deployer { @@ -64,6 +76,16 @@ func WithDeployerVerbose(verbose bool) DeployerOpt { } } +// WithDeployerExposureDisabled turns off this Deployer's own OpenShift +// Route exposure; for deployers that embed this Deployer but manage +// exposure themselves (eg. keda, whose functions stay behind its own +// interceptor and mint their own Route separately). +func WithDeployerExposureDisabled() DeployerOpt { + return func(d *Deployer) { + d.exposureDisabled = true + } +} + func WithDeployerDecorator(decorator deployer.DeployDecorator) DeployerOpt { return func(d *Deployer) { d.decorator = decorator @@ -133,6 +155,11 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, err } + dynClient, err := dynamic.NewForConfig(config) + if err != nil { + return fn.DeploymentResult{}, fmt.Errorf("failed to create dynamic client: %w", err) + } + // Check if Dapr is installed daprInstalled := false _, err = clientset.CoreV1().Namespaces().Get(ctx, "dapr-system", metav1.GetOptions{}) @@ -161,7 +188,15 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("failed to validate referenced resources: %w", err) } - svc, err := d.generateService(f, namespace, daprInstalled, existingDeployment) + existingService, svcGetErr := serviceClient.Get(ctx, f.Name, metav1.GetOptions{}) + if svcGetErr != nil { + if !errors.IsNotFound(svcGetErr) { + return fn.DeploymentResult{}, fmt.Errorf("failed to get existing service: %w", svcGetErr) + } + existingService = nil + } + + svc, err := d.generateService(f, namespace, daprInstalled, existingDeployment, existingService) if err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to generate service resources: %w", err) } @@ -173,19 +208,17 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("failed to update deployment: %w", err) } - existingService, err := serviceClient.Get(ctx, f.Name, metav1.GetOptions{}) - if err == nil { + // update/create service + if svcGetErr == nil { svc.ResourceVersion = existingService.ResourceVersion if _, err = serviceClient.Update(ctx, svc, metav1.UpdateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to update service: %w", err) } - } else if errors.IsNotFound(err) { - // Service doesn't exist, create it + } else { + // Confirmed IsNotFound above the generateService() if _, err = serviceClient.Create(ctx, svc, metav1.CreateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to create service: %w", err) } - } else { - return fn.DeploymentResult{}, fmt.Errorf("failed to get existing service: %w", err) } status = fn.Updated @@ -215,7 +248,7 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("failed to create deployment: %w", err) } - svc, err := d.generateService(f, namespace, daprInstalled, deployment) + svc, err := d.generateService(f, namespace, daprInstalled, deployment, nil) if err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to generate service resources: %w", err) } @@ -234,6 +267,12 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("deployment did not become ready: %w", err) } + // External exposure via an OpenShift Route; see resolveExposure(). + url, _, err := d.resolveExposure(ctx, f, namespace, clientset, dynClient) + if err != nil { + return fn.DeploymentResult{}, err + } + // Sync triggers eventingClient, err := newEventingClient(config, namespace) if err != nil { @@ -243,8 +282,6 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, err } - url := fmt.Sprintf("http://%s.%s.svc", f.Name, namespace) - return fn.DeploymentResult{ Status: status, URL: url, @@ -253,6 +290,160 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu }, nil } +// resolveExposure keeps a raw-deployer function's external exposure (an +// OpenShift Route) in sync with what's currently wanted. This is symmetric +// for both directions: create/update the Route when exposure is wanted, +// remove it when it isn't - so toggling expose:route on and off across +// redeploys just works. +// Removal is unconditional whenever exposure isn't currently wanted, so a +// stale Route from a prior raw deploy never survives a raw -> keda deployer +// switch (the only cross-deployer path that still runs this code, since +// keda embeds this deployer with exposure disabled). +// Functions are exposed BY DEFAULT: a deployed function being reachable is +// the expected outcome, matching what a plain "func deploy" already implies +// for every other deployer, so the unset value behaves the same as +// explicit expose:route, not like expose:none. This is only meaningful on +// OpenShift, since a Route is an OpenShift-only mechanism: IsOpenShift() +// keeps plain-Kubernetes deploys safe without requiring any flag - an +// explicit expose:route request off OpenShift is still a hard error (the +// user asked for something impossible), but the unset default just quietly +// degrades to cluster-local there rather than failing an ordinary deploy. +func (d *Deployer) resolveExposure(ctx context.Context, f fn.Function, namespace string, clientset kubernetes.Interface, dynClient dynamic.Interface) (string, bool, error) { + defaultURL := fmt.Sprintf("http://%s.%s.svc", f.Name, namespace) + + if err := fn.ValidateExpose(f.Deploy.Expose); err != nil { + return "", false, err + } + tech := f.Deploy.Expose + + if tech == "route" && !IsOpenShift() { + return "", false, fmt.Errorf( + "expose:route requires an OpenShift cluster: route.openshift.io Routes are an " + + "OpenShift-specific resource, and this does not appear to be an OpenShift cluster") + } + + if d.exposureDisabled { + if err := d.removeExposure(ctx, clientset, dynClient, namespace, f.Name, false); err != nil { + return "", false, err + } + return defaultURL, false, nil + } + + wantsRoute := tech == "route" || (tech == "" && IsOpenShift()) + if !wantsRoute { + // expose:none (explicit opt-out): enforce=true, a hard error if + // removal fails to verify/clear. Unset on a non-OpenShift cluster + // (default gracefully declined, not requested): enforce=false, + // since nothing was actually asked for here. + if err := d.removeExposure(ctx, clientset, dynClient, namespace, f.Name, tech == "none"); err != nil { + return "", false, err + } + return defaultURL, false, nil + } + + url, err := d.ensureExposure(ctx, f, namespace, clientset, dynClient) + if err != nil { + return "", false, fmt.Errorf("external exposure failed: %w", err) + } + return url, true, nil +} + +// removeExposure deletes the managed Route (never a user-authored route +// sharing the function's name) and clears the recorded exposure state. +// Missing Route API support needs no special-casing: the GET reports +// NotFound either way, meaning nothing to remove. +// +// enforce selects the failure posture: +// - true (unset or expose:none): failing to verify/remove is a hard error; +// - false (deployer switched away from raw): an RBAC 403 on the route +// GET/DELETE prints a warning and the deploy continues, since keda +// users without Route permissions must stay green. +func (d *Deployer) removeExposure(ctx context.Context, clientset kubernetes.Interface, dynClient dynamic.Interface, namespace, name string, enforce bool) error { + if _, err := RemoveManagedRoute(ctx, dynClient, namespace, name); err != nil { + if !enforce && errors.IsForbidden(err) { + fmt.Fprintf(os.Stderr, "⚠️ cannot remove Route %q (forbidden) - leaving it in place\n", name) + } else { + return fmt.Errorf("failed to remove Route: %w", err) + } + } + + if err := writeRouteHostnameAnnotation(ctx, clientset, namespace, name, ""); err != nil { + if !enforce { + fmt.Fprintf(os.Stderr, "⚠️ failed to clear exposure state: %v\n", err) + return nil + } + return fmt.Errorf("failed to clear exposure state: %w", err) + } + return nil +} + +// ensureExposure creates or updates the Route exposing f, waits for it to +// be admitted by a router, and records the minted hostname. +func (d *Deployer) ensureExposure(ctx context.Context, f fn.Function, namespace string, clientset kubernetes.Interface, dynClient dynamic.Interface) (string, error) { + deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, f.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get deployment for owner reference: %w", err) + } + + route, err := GenerateRoute(f, f.Name, deployment, d.decorator, KubernetesDeployerName) + if err != nil { + return "", fmt.Errorf("failed to generate Route: %w", err) + } + + fmt.Fprintf(os.Stderr, "🌐 Exposing function externally -> %s\n", f.Name) + + if err := EnsureRoute(ctx, dynClient, namespace, route); err != nil { + return "", err + } + + // Wait for a router to accept the route - enforced, never downgraded to a warning. + host, err := WaitForRouteAdmitted(ctx, dynClient, namespace, f.Name, 30*time.Second) + if err != nil { + return "", fmt.Errorf("route was not admitted: %w", err) + } + + if err := writeRouteHostnameAnnotation(ctx, clientset, namespace, f.Name, host); err != nil { + return "", err + } + + // The Route redirects http to https (see GenerateRoute's tls stanza). + return fmt.Sprintf("https://%s", host), nil +} + +// writeRouteHostnameAnnotation records (hostname != "") or clears +// (hostname == "") the exposed hostname on the function's Service: no-op +// when already current, retried on write conflicts. A missing Service is +// tolerated only when clearing; recording against one that doesn't exist +// is a real error. +func writeRouteHostnameAnnotation(ctx context.Context, clientset kubernetes.Interface, namespace, name, hostname string) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + svc, err := clientset.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + if svc.Annotations[RouteHostnameAnnotation] == hostname { + return nil + } + if hostname == "" { + delete(svc.Annotations, RouteHostnameAnnotation) + } else { + if svc.Annotations == nil { + svc.Annotations = map[string]string{} + } + svc.Annotations[RouteHostnameAnnotation] = hostname + } + _, err = clientset.CoreV1().Services(namespace).Update(ctx, svc, metav1.UpdateOptions{}) + return err + }) + if err != nil { + if hostname == "" && errors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to update exposure state on service %q: %w", name, err) + } + return nil +} + // generateTriggerName creates a deterministic trigger name based on subscription content func generateTriggerName(functionName, broker string, filters map[string]string) string { filterKeys := make([]string, 0, len(filters)) @@ -489,13 +680,23 @@ func (d *Deployer) generateDeployment(f fn.Function, namespace string, daprInsta return deployment, nil } -func (d *Deployer) generateService(f fn.Function, namespace string, daprInstalled bool, deployment *appsv1.Deployment) (*corev1.Service, error) { +// generateService builds the function's Service; existingService is the +// currently-deployed Service on update, nil on create. +func (d *Deployer) generateService(f fn.Function, namespace string, daprInstalled bool, deployment *appsv1.Deployment, existingService *corev1.Service) (*corev1.Service, error) { labels, err := deployer.GenerateCommonLabels(f, d.decorator) if err != nil { return nil, err } annotations := deployer.GenerateCommonAnnotations(f, d.decorator, daprInstalled, KubernetesDeployerName) + // re-apply the hostname annotation, contrary to the rest of annotations + // which are "always regenerate" -- the hostname is cluster-derived, not + // in func.yaml: the router mints it, and only the exposure step (after + // the Route is admitted) can write it, which happens after this + // Service write. + if existingService != nil && existingService.Annotations[RouteHostnameAnnotation] != "" { + annotations[RouteHostnameAnnotation] = existingService.Annotations[RouteHostnameAnnotation] + } service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/k8s/deployer_test.go b/pkg/k8s/deployer_test.go index 020abc2d9d..16b6096559 100644 --- a/pkg/k8s/deployer_test.go +++ b/pkg/k8s/deployer_test.go @@ -6,7 +6,10 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + dynamicfakeclient "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" fn "knative.dev/func/pkg/functions" ) @@ -563,3 +566,84 @@ func Test_ProcessVolumes_ValidPath(t *testing.T) { t.Errorf("expected mount path /etc/secret, got %s", mounts[0].MountPath) } } + +// Test_WithDeployerExposureDisabled: exposure is on by default (raw) and off +// with others +func Test_WithDeployerExposureDisabled(t *testing.T) { + if NewDeployer().exposureDisabled { + t.Error("expected exposure enabled on a default Deployer") + } + if !NewDeployer(WithDeployerExposureDisabled()).exposureDisabled { + t.Error("expected exposure disabled with WithDeployerExposureDisabled") + } +} + +// Test_ResolveExposure_RouteGatedOnOpenShift: functions are exposed by +// default, so an explicit expose:route request hard-errors off OpenShift +// (the user asked for something impossible), and expose:none (explicit +// opt-out) never requires OpenShift or touches the Route API at all, on +// either platform - removeExposure's Get against an empty fake dynamic +// client returns NotFound immediately. The unset/empty value off OpenShift +// also stays cluster-local, but silently (no error): the default degrading +// gracefully rather than failing an ordinary deploy is exactly the point. +// +// The "route on OpenShift" and "empty on OpenShift" cases are NOT exercised +// here: both fall through to ensureExposure, which waits up to 30s +// (hardcoded) for a router to admit the Route - a real wait against a fake +// client with no controller to populate status would either hang the test +// for 30s or require simulating async status writes, disproportionate for +// this table. That deeper path (EnsureRoute, WaitForRouteAdmitted, +// GenerateRoute) is covered directly and fast in route_test.go instead, +// each with its own short timeout. +// +// Note: SetOpenShiftForTest mutates a package-level bool without a mutex - +// this test must not run with t.Parallel() (see openshift.go). +func Test_ResolveExposure_RouteGatedOnOpenShift(t *testing.T) { + d := NewDeployer() + f := fn.Function{Name: "f", Deploy: fn.DeploySpec{Namespace: "ns"}} + ctx := t.Context() + clientset := fake.NewClientset() + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + tests := []struct { + name string + expose string + openShift bool + wantErr bool + wantExpose bool + }{ + {name: "route off OpenShift: hard error", expose: "route", openShift: false, wantErr: true}, + {name: "none off OpenShift: fine", expose: "none", openShift: false}, + {name: "none on OpenShift: fine", expose: "none", openShift: true}, + {name: "empty off OpenShift: fine, cluster-local, no error", expose: "", openShift: false}, + // "empty on OpenShift" is NOT in this table: functions are exposed + // by default now, so unset+OpenShift takes the same real + // Route-creation path as explicit expose:route does - excluded + // here for the same reason "route on OpenShift" already is (see + // the comment above this test). + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cleanup := SetOpenShiftForTest(tt.openShift) + defer cleanup() + + f.Deploy.Expose = tt.expose + url, exposed, err := d.resolveExposure(ctx, f, "ns", clientset, dynClient) + if tt.wantErr { + if err == nil { + t.Fatalf("resolveExposure(%q) on OpenShift=%v: expected an error, got nil", tt.expose, tt.openShift) + } + return + } + if err != nil { + t.Fatalf("resolveExposure(%q) on OpenShift=%v: unexpected error: %v", tt.expose, tt.openShift, err) + } + if exposed != tt.wantExpose { + t.Errorf("resolveExposure(%q) on OpenShift=%v: exposed = %v, want %v", tt.expose, tt.openShift, exposed, tt.wantExpose) + } + if url == "" { + t.Errorf("resolveExposure(%q) on OpenShift=%v: expected a non-empty URL", tt.expose, tt.openShift) + } + }) + } +} diff --git a/pkg/k8s/describer.go b/pkg/k8s/describer.go index 14f8468fdf..4054d135ac 100644 --- a/pkg/k8s/describer.go +++ b/pkg/k8s/describer.go @@ -78,7 +78,19 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In } } - primaryRouteURL := fmt.Sprintf("http://%s.%s.svc", name, namespace) // TODO: get correct scheme? + internalURL := fmt.Sprintf("http://%s.%s.svc", name, namespace) + primaryRouteURL := internalURL + + // External hostname (if exposed) was recorded on the Service by Deploy() + // at exposure time - no extra API call or client needed here. + if hostname, ok := service.Annotations[RouteHostnameAnnotation]; ok && hostname != "" { + primaryRouteURL = fmt.Sprintf("https://%s", hostname) + } + // an exposed function stays reachable in-cluster too + routes := []string{primaryRouteURL} + if primaryRouteURL != internalURL { + routes = append(routes, internalURL) + } // get image image := "" @@ -104,7 +116,7 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In Deployer: KubernetesDeployerName, Labels: deployment.Labels, Route: primaryRouteURL, - Routes: []string{primaryRouteURL}, + Routes: routes, Image: image, Middleware: fn.Middleware{ Version: middlewareVersion, diff --git a/pkg/k8s/lister.go b/pkg/k8s/lister.go index 82e0566f88..9175d029c4 100644 --- a/pkg/k8s/lister.go +++ b/pkg/k8s/lister.go @@ -77,12 +77,17 @@ func (l *Lister) get(ctx context.Context, clientset *kubernetes.Clientset, name, return fn.ListItem{}, fmt.Errorf("could not get service: %w", err) } + url := fmt.Sprintf("http://%s.%s.svc", service.Name, service.Namespace) // TODO: use correct scheme + if hostname, ok := service.Annotations[RouteHostnameAnnotation]; ok && hostname != "" { + url = fmt.Sprintf("https://%s", hostname) + } + runtimeLabel := "" listItem := fn.ListItem{ Name: service.Name, Namespace: service.Namespace, Runtime: runtimeLabel, - URL: fmt.Sprintf("http://%s.%s.svc", service.Name, service.Namespace), // TODO: use correct scheme + URL: url, Ready: string(ready), Deployer: KubernetesDeployerName, } diff --git a/pkg/k8s/route.go b/pkg/k8s/route.go new file mode 100644 index 0000000000..e492dbfd37 --- /dev/null +++ b/pkg/k8s/route.go @@ -0,0 +1,268 @@ +package k8s + +import ( + "context" + "fmt" + "os" + "time" + + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/util/retry" + + "knative.dev/func/pkg/deployer" + fn "knative.dev/func/pkg/functions" +) + +// routeGVR identifies the OpenShift Route resource. No typed client is used +// here: adding github.com/openshift/api as a direct dependency for a +// handful of fields is disproportionate, this project already has a +// precedent for reading Routes through the dynamic client (see +// pkg/pipelines/tekton/pac/pac.go DetectPACOpenShiftRoute), and there is no +// existing github.com/openshift/api requirement anywhere in go.mod to build +// on. Route's structure is also small and stable (a v1, GA API since +// OpenShift 3.x), so hand-built unstructured content carries little +// maintenance risk. +var routeGVR = schema.GroupVersionResource{ + Group: "route.openshift.io", + Version: "v1", + Resource: "routes", +} + +// GenerateRoute builds (but does not create) the OpenShift Route that +// exposes svcName's "http" port. spec.host is left empty so the cluster's +// router mints one (see docs/research citations in the openshift-route-fork +// records) - custom domains are out of scope for this commit. +func GenerateRoute(f fn.Function, svcName string, deployment *appsv1.Deployment, decorator deployer.DeployDecorator, deployerName string) (*unstructured.Unstructured, error) { + labels, err := deployer.GenerateCommonLabels(f, decorator) + if err != nil { + return nil, err + } + annotations := deployer.GenerateCommonAnnotations(f, decorator, false /* dapr n/a for routing */, deployerName) + + route := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": routeGVR.GroupVersion().String(), + "kind": "Route", + "metadata": map[string]any{ + "name": f.Name, + "namespace": deployment.Namespace, + "labels": stringMapToAny(labels), + "annotations": stringMapToAny(annotations), + "ownerReferences": []any{ + map[string]any{ + "apiVersion": appsv1.SchemeGroupVersion.WithKind("Deployment").GroupVersion().String(), + "kind": "Deployment", + "name": deployment.Name, + "uid": string(deployment.UID), + "controller": true, + }, + }, + }, + "spec": map[string]any{ + "to": map[string]any{ + "kind": "Service", + "name": svcName, + }, + "port": map[string]any{ + "targetPort": "http", + }, + // Edge TLS via the router's wildcard cert - zero cert + // management; Redirect upgrades http requests to https. + "tls": map[string]any{ + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect", + }, + }, + }, + } + + return route, nil +} + +// stringMapToAny converts a map[string]string to the map[string]any +// unstructured.Unstructured needs its nested fields to be. +func stringMapToAny(m map[string]string) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// EnsureRoute creates or updates a Route, retrying the whole +// get-mutate-update cycle on a 409 conflict (a controller status write can +// race an update from here). +func EnsureRoute(ctx context.Context, dynClient dynamic.Interface, ns string, route *unstructured.Unstructured) error { + client := dynClient.Resource(routeGVR).Namespace(ns) + name := route.GetName() + + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + existing, getErr := client.Get(ctx, name, metav1.GetOptions{}) + if getErr != nil { + if apierrors.IsNotFound(getErr) { + route.SetResourceVersion("") + _, createErr := client.Create(ctx, route, metav1.CreateOptions{}) + return createErr + } + return getErr + } + route.SetResourceVersion(existing.GetResourceVersion()) + _, updateErr := client.Update(ctx, route, metav1.UpdateOptions{}) + return updateErr + }) + if err != nil { + return fmt.Errorf("failed to ensure Route %q: %w", name, err) + } + return nil +} + +// isManagedRoute reports whether route was created by GenerateRoute() - as +// opposed to a user-authored or third-party Route that happens to share the +// function's name, which must never be deleted out from under the user. +// Both signals are required: a bare boson.dev/function label, or a +// deployer annotation written by some other component, alone does not +// prove func's raw deployer owns the route. +func isManagedRoute(route *unstructured.Unstructured) bool { + return route.GetLabels()["boson.dev/function"] == "true" && + route.GetAnnotations()[deployer.DeployerNameAnnotation] == KubernetesDeployerName +} + +// RemoveManagedRoute deletes the Route named 'name' in 'ns' only if func +// owns it (isManagedRoute()). Returns (removed, error): +// - not found (route absent, or the Route API isn't installed) -> (false, nil) +// - found but not managed -> (false, nil), warning printed, route kept +// - found and managed, deleted -> (true, nil) +func RemoveManagedRoute(ctx context.Context, dynClient dynamic.Interface, ns, name string) (bool, error) { + client := dynClient.Resource(routeGVR).Namespace(ns) + + route, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed to check for existing Route %q: %w", name, err) + } + + if !isManagedRoute(route) { + fmt.Fprintf(os.Stderr, + "⚠️ a Route named %q exists in namespace %q but is not managed by func - leaving it in place\n", + name, ns) + return false, nil + } + + if err := client.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("failed to delete Route %q: %w", name, err) + } + return true, nil +} + +// WaitForRouteAdmitted polls the Route status until any ingress entry (one +// per router/IngressController shard - a cluster can run more than one) +// reports Admitted=True, returning that entry's host. It fails immediately +// (not waiting out the full timeout) only when an ingress entry explicitly +// reports Admitted=False - e.g. a host already claimed by another Route - +// surfacing the condition's reason and message. An entry with no Admitted +// condition yet is polled through to the timeout, fail-open on unknown. +func WaitForRouteAdmitted(ctx context.Context, dynClient dynamic.Interface, ns, name string, timeout time.Duration) (string, error) { + client := dynClient.Resource(routeGVR).Namespace(ns) + + var host string + var lastErr error + pollErr := wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + route, err := client.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + lastErr = fmt.Errorf("failed to get Route %q: %w", name, err) + return false, nil + } + + ingresses, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") + if err != nil || !found { + return false, nil + } + + for _, raw := range ingresses { + ingress, ok := raw.(map[string]any) + if !ok { + continue + } + conditions, found, err := unstructured.NestedSlice(ingress, "conditions") + if err != nil || !found { + continue + } + for _, rawCond := range conditions { + cond, ok := rawCond.(map[string]any) + if !ok || cond["type"] != "Admitted" { + continue + } + status, _ := cond["status"].(string) + switch status { + case "True": + host, _, _ = unstructured.NestedString(ingress, "host") + return true, nil + case "False": + reason, _ := cond["reason"].(string) + message, _ := cond["message"].(string) + lastErr = fmt.Errorf("route %q was rejected by the router: %s: %s", name, reason, message) + return false, lastErr + } + // Unknown or missing status: keep polling. + } + } + + return false, nil + }) + if pollErr != nil { + if lastErr != nil { + return "", lastErr + } + return "", fmt.Errorf("route %q was not admitted by any router within %s: %w", name, timeout, pollErr) + } + return host, nil +} + +// GetAdmittedRouteHost is a single, non-blocking read of a Route's currently +// admitted host, for display paths (describe/list) that must return +// immediately rather than poll like WaitForRouteAdmitted does. Returns +// ("", false, nil) if the Route doesn't exist or has no Admitted=True +// ingress entry yet - both are "no external URL to show", not errors. +func GetAdmittedRouteHost(ctx context.Context, dynClient dynamic.Interface, ns, name string) (string, bool, error) { + route, err := dynClient.Resource(routeGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return "", false, nil + } + return "", false, fmt.Errorf("failed to get Route %q: %w", name, err) + } + + ingresses, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") + if err != nil || !found { + return "", false, nil + } + for _, raw := range ingresses { + ingress, ok := raw.(map[string]any) + if !ok { + continue + } + conditions, found, err := unstructured.NestedSlice(ingress, "conditions") + if err != nil || !found { + continue + } + for _, rawCond := range conditions { + cond, ok := rawCond.(map[string]any) + if !ok || cond["type"] != "Admitted" { + continue + } + if status, _ := cond["status"].(string); status == "True" { + host, _, _ := unstructured.NestedString(ingress, "host") + return host, host != "", nil + } + } + } + return "", false, nil +} diff --git a/pkg/k8s/route_test.go b/pkg/k8s/route_test.go new file mode 100644 index 0000000000..85bb44bda6 --- /dev/null +++ b/pkg/k8s/route_test.go @@ -0,0 +1,290 @@ +package k8s + +import ( + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + + fn "knative.dev/func/pkg/functions" +) + +func newFakeDynamicClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { + return dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{routeGVR: "RouteList"}, + objects..., + ) +} + +func testDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "f", + Namespace: "ns", + UID: types.UID("abc-123"), + }, + } +} + +func Test_GenerateRoute(t *testing.T) { + f := fn.Function{Name: "f", Runtime: "go"} + deployment := testDeployment() + + route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + + if route.GetName() != "f" || route.GetNamespace() != "ns" { + t.Errorf("expected name/namespace f/ns, got %s/%s", route.GetName(), route.GetNamespace()) + } + toName, _, _ := unstructured.NestedString(route.Object, "spec", "to", "name") + if toName != "f" { + t.Errorf("expected spec.to.name %q, got %q", "f", toName) + } + toKind, _, _ := unstructured.NestedString(route.Object, "spec", "to", "kind") + if toKind != "Service" { + t.Errorf("expected spec.to.kind Service, got %q", toKind) + } + targetPort, _, _ := unstructured.NestedString(route.Object, "spec", "port", "targetPort") + if targetPort != "http" { + t.Errorf("expected spec.port.targetPort http, got %q", targetPort) + } + if host, found, _ := unstructured.NestedString(route.Object, "spec", "host"); found && host != "" { + t.Errorf("expected spec.host to be unset (router-minted), got %q", host) + } + termination, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "termination") + if termination != "edge" { + t.Errorf("expected spec.tls.termination edge, got %q", termination) + } + insecurePolicy, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "insecureEdgeTerminationPolicy") + if insecurePolicy != "Redirect" { + t.Errorf("expected spec.tls.insecureEdgeTerminationPolicy Redirect, got %q", insecurePolicy) + } + if !isManagedRoute(route) { + t.Error("expected a freshly generated Route to be self-managed") + } + owners := route.GetOwnerReferences() + if len(owners) != 1 || owners[0].Name != "f" || owners[0].Kind != "Deployment" { + t.Errorf("expected a single Deployment ownerRef named f, got %+v", owners) + } +} + +func Test_EnsureRoute_CreateThenUpdate(t *testing.T) { + ctx := t.Context() + f := fn.Function{Name: "f", Runtime: "go"} + deployment := testDeployment() + client := newFakeDynamicClient() + + route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + if err := EnsureRoute(ctx, client, "ns", route); err != nil { + t.Fatalf("create: %v", err) + } + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Route to exist after create: %v", err) + } + toName, _, _ := unstructured.NestedString(got.Object, "spec", "to", "name") + if toName != "f" { + t.Errorf("expected spec.to.name f, got %q", toName) + } + + // Update path: regenerate (idempotent) and ensure again, no error. + route2, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + if err := EnsureRoute(ctx, client, "ns", route2); err != nil { + t.Fatalf("update: %v", err) + } +} + +func Test_RemoveManagedRoute(t *testing.T) { + ctx := t.Context() + + t.Run("not found: no-op", func(t *testing.T) { + client := newFakeDynamicClient() + removed, err := RemoveManagedRoute(ctx, client, "ns", "f") + if err != nil || removed { + t.Errorf("expected (false, nil), got (%v, %v)", removed, err) + } + }) + + t.Run("managed: deleted", func(t *testing.T) { + f := fn.Function{Name: "f", Runtime: "go"} + deployment := testDeployment() + route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + route.SetNamespace("ns") + client := newFakeDynamicClient(route) + + removed, err := RemoveManagedRoute(ctx, client, "ns", "f") + if err != nil || !removed { + t.Fatalf("expected (true, nil), got (%v, %v)", removed, err) + } + if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err == nil { + t.Error("expected Route to be gone after removal") + } + }) + + t.Run("not managed: kept", func(t *testing.T) { + foreign := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{ + "name": "f", + "namespace": "ns", + }, + }} + client := newFakeDynamicClient(foreign) + + removed, err := RemoveManagedRoute(ctx, client, "ns", "f") + if err != nil || removed { + t.Fatalf("expected (false, nil) for a foreign Route, got (%v, %v)", removed, err) + } + if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err != nil { + t.Error("expected the foreign Route to be left in place") + } + }) +} + +func Test_WaitForRouteAdmitted(t *testing.T) { + ctx := t.Context() + + admittedRoute := func(host string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": host, + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "True"}, + }, + }, + }, + }, + }} + } + + t.Run("admitted: returns host", func(t *testing.T) { + client := newFakeDynamicClient(admittedRoute("f-ns.apps.example.com")) + host, err := WaitForRouteAdmitted(ctx, client, "ns", "f", time.Second) + if err != nil { + t.Fatal(err) + } + if host != "f-ns.apps.example.com" { + t.Errorf("expected host f-ns.apps.example.com, got %q", host) + } + }) + + t.Run("rejected: fails fast with reason", func(t *testing.T) { + rejected := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": "", + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "False", "reason": "HostAlreadyClaimed", "message": "taken"}, + }, + }, + }, + }, + }} + client := newFakeDynamicClient(rejected) + _, err := WaitForRouteAdmitted(ctx, client, "ns", "f", 5*time.Second) + if err == nil { + t.Fatal("expected an error for a rejected Route") + } + }) + + t.Run("never admitted: times out cleanly", func(t *testing.T) { + f := fn.Function{Name: "f", Runtime: "go"} + deployment := testDeployment() + route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + route.SetNamespace("ns") + client := newFakeDynamicClient(route) + + _, err = WaitForRouteAdmitted(ctx, client, "ns", "f", 100*time.Millisecond) + if err == nil { + t.Fatal("expected a timeout error when no router ever admits the route") + } + }) +} + +func Test_GetAdmittedRouteHost(t *testing.T) { + ctx := t.Context() + + admittedRoute := func(host string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": host, + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "True"}, + }, + }, + }, + }, + }} + } + + t.Run("admitted: returns host", func(t *testing.T) { + client := newFakeDynamicClient(admittedRoute("f-ns.apps.example.com")) + host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") + if err != nil { + t.Fatal(err) + } + if !ok || host != "f-ns.apps.example.com" { + t.Errorf("expected (f-ns.apps.example.com, true), got (%q, %v)", host, ok) + } + }) + + t.Run("not found: no error, not found", func(t *testing.T) { + client := newFakeDynamicClient() + host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") + if err != nil || ok || host != "" { + t.Errorf("expected (\"\", false, nil), got (%q, %v, %v)", host, ok, err) + } + }) + + t.Run("not yet admitted: no error, not found", func(t *testing.T) { + f := fn.Function{Name: "f", Runtime: "go"} + deployment := testDeployment() + route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) + if err != nil { + t.Fatal(err) + } + route.SetNamespace("ns") + client := newFakeDynamicClient(route) + + host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") + if err != nil || ok || host != "" { + t.Errorf("expected (\"\", false, nil) for an unadmitted route, got (%q, %v, %v)", host, ok, err) + } + }) +} diff --git a/pkg/keda/deployer.go b/pkg/keda/deployer.go index bb03c35176..7fff29a7a6 100644 --- a/pkg/keda/deployer.go +++ b/pkg/keda/deployer.go @@ -37,6 +37,11 @@ func NewDeployer(opts ...DeployerOpt) *Deployer { Deployer: *k8s.NewDeployer( // init with the kedaDeployerDecorator to have the correct deployer labels&annotations k8s.WithDeployerDecorator(&kedaDeployerDecorator{}), + // keda functions stay behind the interceptor; this deployer + // mints its own Route separately (see route.go) rather than + // letting the embedded raw deployer expose the function's own + // Service directly, which would bypass the interceptor entirely + k8s.WithDeployerExposureDisabled(), ), } diff --git a/pkg/knative/deployer.go b/pkg/knative/deployer.go index 7d47065873..8082864f48 100644 --- a/pkg/knative/deployer.go +++ b/pkg/knative/deployer.go @@ -292,7 +292,7 @@ consider using the --image-pull-secret flag, or setting up pull secrets manually } if d.verbose { - fmt.Printf("Function deployed in namespace %q and exposed at URL:\n%s\n", namespace, route.Status.URL.String()) + fmt.Printf("Function deployed in namespace %q at URL:\n%s\n", namespace, route.Status.URL.String()) } return fn.DeploymentResult{ Status: fn.Deployed, diff --git a/pkg/lister/testing/integration_test_helper.go b/pkg/lister/testing/integration_test_helper.go index d063fb175f..e794d0f452 100644 --- a/pkg/lister/testing/integration_test_helper.go +++ b/pkg/lister/testing/integration_test_helper.go @@ -39,6 +39,10 @@ func TestInt_List(t *testing.T, lister fn.Lister, deployer fn.Deployer, describe Runtime: "go", Namespace: ns, Registry: Registry(), + // Explicit opt-out: keeps this integration deploy cluster-local and + // platform-deterministic under exposed-by-default; ignored entirely + // by the knative deployer. + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) diff --git a/pkg/mock/deployer.go b/pkg/mock/deployer.go index 4398faeeec..74ccd5badc 100644 --- a/pkg/mock/deployer.go +++ b/pkg/mock/deployer.go @@ -39,6 +39,9 @@ func NewDeployer() *Deployer { } else { result.Deployer = f.Deploy.Deployer // redeploy with current } + if err == nil { + result.Status = fn.Deployed + } return }, } diff --git a/pkg/pipelines/tekton/pipelines_provider.go b/pkg/pipelines/tekton/pipelines_provider.go index 186a2dd2cf..4758380e03 100644 --- a/pkg/pipelines/tekton/pipelines_provider.go +++ b/pkg/pipelines/tekton/pipelines_provider.go @@ -275,11 +275,14 @@ func (pp *PipelinesProvider) Run(ctx context.Context, f fn.Function) (string, fn return "", f, fmt.Errorf("problem in retrieving status of deployed function: %v", err) } - if obj.Generation == 1 { - fmt.Fprintf(os.Stderr, "✅ Function deployed in namespace %q and exposed at URL: \n %s\n", obj.Namespace, obj.Route) - } else { - fmt.Fprintf(os.Stderr, "✅ Function updated in namespace %q and exposed at URL: \n %s\n", obj.Namespace, obj.Route) - } + verb := "deployed" + if obj.Generation != 1 { + verb = "updated" + } + // Mirrors the deploy status message in pkg/functions/client.go's Deploy - + // same neutral wording, no exposure claim (duplicated rather than + // exported+imported across packages for one format string). + fmt.Fprintf(os.Stderr, "✅ Function %s in namespace %q at URL: \n %s\n", verb, obj.Namespace, obj.Route) if obj.Namespace != namespace { fmt.Fprintf(os.Stderr, "Warning: Final function namespace %q does not match expected %q", obj.Namespace, namespace) diff --git a/pkg/remover/testing/integration_test_helper.go b/pkg/remover/testing/integration_test_helper.go index 07d9274825..ce0634048c 100644 --- a/pkg/remover/testing/integration_test_helper.go +++ b/pkg/remover/testing/integration_test_helper.go @@ -39,6 +39,7 @@ func TestInt_Remove(t *testing.T, remover fn.Remover, deployer fn.Deployer, desc Runtime: "go", Namespace: ns, Registry: Registry(), + Deploy: fn.DeploySpec{Expose: "none"}, }) if err != nil { t.Fatal(err) diff --git a/schema/func_yaml-schema.json b/schema/func_yaml-schema.json index 044a19b699..79e7af6b3d 100644 --- a/schema/func_yaml-schema.json +++ b/schema/func_yaml-schema.json @@ -130,6 +130,10 @@ "managementDisabled": { "type": "boolean", "description": "ManagementDisabled disables automatic creation/update of a Function CR\nfor operator management after deploy. The zero value (false) means\nthe function is managed by default when the func-operator is installed." + }, + "expose": { + "type": "string", + "description": "Expose controls external access for the raw and keda deployers (the\nknative deployer manages its own exposure and ignores it). Optional.\nValues: \"route\" (create an OpenShift Route; OpenShift clusters only -\na hard error elsewhere), \"none\" (cluster-local only, explicit\nopt-out). Defaults to \"route\" behavior on OpenShift - a deployed\nfunction being externally reachable is the expected outcome - and to\ncluster-local on any other cluster, since a Route is an\nOpenShift-only mechanism and the unset default must not impose a\nplatform requirement." } }, "additionalProperties": false, From 35c4b1ae190518fb1c40fe1d1b8236933b430e3f Mon Sep 17 00:00:00 2001 From: David Fridrich Date: Wed, 29 Jul 2026 18:28:24 +0200 Subject: [PATCH 3/3] feat: expose functions externally via OpenShift Route through a pluggable Exposer --- .github/workflows/functions.yaml | 64 ++ Makefile | 32 +- cmd/client.go | 29 +- cmd/completion_util.go | 2 +- cmd/deploy.go | 68 +- cmd/deploy_test.go | 164 +++- cmd/describe.go | 17 +- cmd/describe_test.go | 29 + cmd/errors.go | 6 +- cmd/func-util/main.go | 20 + docs/reference/func_deploy.md | 9 +- e2e/e2e_expose_test.go | 790 +++++++++++++++++ e2e/e2e_test.go | 64 +- pkg/deployer/common.go | 34 +- pkg/deployer/common_test.go | 30 + pkg/deployer/expose.go | 96 ++ .../testing/integration_test_helper.go | 15 +- .../testing/integration_test_helper.go | 2 +- pkg/functions/client.go | 30 +- pkg/functions/client_test.go | 26 + pkg/functions/errors.go | 2 +- pkg/functions/function.go | 24 +- pkg/functions/function_expose.go | 49 +- pkg/functions/function_expose_unit_test.go | 9 + pkg/k8s/deployer.go | 325 ++++--- pkg/k8s/deployer_test.go | 634 ++++++++++++-- pkg/k8s/describer.go | 3 + pkg/k8s/labels/labels.go | 12 + pkg/k8s/openshift.go | 71 +- pkg/k8s/openshift_unit_test.go | 4 +- pkg/k8s/route.go | 268 ------ pkg/k8s/route_test.go | 290 ------- pkg/k8s/security_context_test.go | 6 +- pkg/keda/deployer.go | 151 +++- pkg/keda/deployer_unit_test.go | 84 ++ pkg/keda/describer.go | 13 +- pkg/keda/exposure.go | 191 ++++ pkg/keda/exposure_test.go | 331 +++++++ pkg/keda/lister.go | 15 +- pkg/keda/remover.go | 96 +- pkg/keda/remover_unit_test.go | 207 +++++ pkg/lister/testing/integration_test_helper.go | 2 +- pkg/mock/deployer.go | 4 + pkg/mock/remover.go | 4 +- pkg/ocproute/route.go | 410 +++++++++ pkg/ocproute/route_test.go | 820 ++++++++++++++++++ pkg/pipelines/tekton/pipelines_provider.go | 10 +- .../testing/integration_test_helper.go | 2 +- schema/func_yaml-schema.json | 16 +- 49 files changed, 4645 insertions(+), 935 deletions(-) create mode 100644 e2e/e2e_expose_test.go create mode 100644 pkg/deployer/common_test.go create mode 100644 pkg/deployer/expose.go delete mode 100644 pkg/k8s/route.go delete mode 100644 pkg/k8s/route_test.go create mode 100644 pkg/keda/deployer_unit_test.go create mode 100644 pkg/keda/exposure.go create mode 100644 pkg/keda/exposure_test.go create mode 100644 pkg/keda/remover_unit_test.go create mode 100644 pkg/ocproute/route.go create mode 100644 pkg/ocproute/route_test.go diff --git a/.github/workflows/functions.yaml b/.github/workflows/functions.yaml index 66f38b7b41..b07c78dc39 100644 --- a/.github/workflows/functions.yaml +++ b/.github/workflows/functions.yaml @@ -250,6 +250,69 @@ jobs: path: ./cluster_log.txt retention-days: 7 + # ---------------- + # E2E EXPOSE TESTS + # ---------------- + # External exposure (--expose). The cluster here is KinD, which has no + # route.openshift.io API, so the tests that assert a Route skip themselves via + # IsOpenShift(), and the ones that assert func REFUSES a Route are the ones + # that run. Pointing the same target at an OpenShift cluster runs the other + # half; CI has no OpenShift cluster, so that half is run manually. + test-e2e-expose: + name: E2E - Expose + needs: precheck + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FUNC_CLUSTER_RETRIES: 5 + FUNC_E2E_CLEAN: false # cluster only used once + FUNC_E2E_VERBOSE: true + # The keda cases needing keda on the cluster are all Route cases, and + # those skip here. The one keda test that does run refuses a too-long + # name before reaching the cluster at all. Flip to "true" if a keda test + # is added that needs the operator; see PR #3914 ('Insufficient cpu') for + # why it is off by default. + FUNC_CLUSTER_KEDA: "false" + steps: + - uses: actions/checkout@v4 + - uses: knative/actions/setup-go@main + - uses: endersonmenezes/free-disk-space@v3 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_swap: true + rm_cmd: "rmz" # Faster than rm + + - name: Install Binaries + run: ./hack/binaries.sh + - name: Allocate Cluster + run: ./hack/cluster.sh + - name: Start Local Registry + run: ./hack/registry.sh + - name: Prepare Images + run: ./hack/images.sh + + - name: Run Expose E2E Tests + run: make test-e2e-expose + + - uses: ./.github/actions/codecov + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: e2e + + # Preserve Cluster Logs + - name: Dump Cluster Logs + if: always() + run: ./hack/dump-logs.sh cluster_log.txt + - name: Archive Cluster Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: cluster-logs-e2e-expose + path: ./cluster_log.txt + retention-days: 7 + # ---------------- # E2E PODMAN TESTS # ---------------- @@ -457,6 +520,7 @@ jobs: - test-integration - test-templates - test-e2e + - test-e2e-expose - test-e2e-podman - test-e2e-runtimes - test-e2e-config-ci diff --git a/Makefile b/Makefile index 5341acdd7e..4f802c2d55 100644 --- a/Makefile +++ b/Makefile @@ -88,20 +88,36 @@ test: generate/zz_filesystem_generated.go ## Run core unit tests go test -race -cover -coverprofile=coverage.txt ./... .PHONY: check -check: check-lint check-goimports check-misspell check-whitespace check-eof ## Check code quality (comprehensive) +check: check-lint check-build-tags check-goimports check-misspell check-whitespace check-eof ## Check code quality (comprehensive) .PHONY: check-lint check-lint: $(BIN_GOLANGCI_LINT) ## Run golangci-lint $(BIN_GOLANGCI_LINT) run --timeout 300s +# cmd/func-util/main.go is behind "exclude_graphdriver_btrfs || !cgo", so the +# default build context compiles only socat.go and every other check in this +# file silently skips it: go vet, golangci-lint and go test all read the +# default context. The sole compile of it anywhere is publish-utils-image in +# functions.yaml, which needs: build and so runs only on a push to main. A type +# error there therefore passes every pull-request check and first fails after +# merge. Vet with the tag so the file is type-checked where the error is cheap. +.PHONY: check-build-tags +check-build-tags: ## Type-check sources the default build context excludes + @echo "Type-checking build-tagged sources..." + go vet -tags exclude_graphdriver_btrfs ./cmd/func-util/ + .PHONY: check-goimports check-goimports: $(BIN_GOIMPORTS) ## Check Go import formatting @echo "Checking Go import formatting..." - @$(LS_SOURCES) | \ + @offenders=$$($(LS_SOURCES) | \ grep '\.go$$' | \ while IFS= read -r file; do [ -f "$$file" ] && echo "$$file"; done | \ - xargs $(BIN_GOIMPORTS) -l | grep . && \ - (echo "Error: Files with incorrect import formatting found. Run 'goimports -w ' to fix."; exit 1) || true + xargs $(BIN_GOIMPORTS) -l); \ + if [ -n "$$offenders" ]; then \ + echo "$$offenders"; \ + echo "Error: Files with incorrect import formatting found. Run 'goimports -w ' to fix."; \ + exit 1; \ + fi .PHONY: check-misspell check-misspell: $(BIN_MISSPELL) ## Check for common misspellings @@ -317,6 +333,14 @@ test-e2e-lifecycle: func-instrumented-bin ## Run lifecycle hook E2E tests (Start go test -tags e2e -timeout 60m ./e2e -v -run TestLifecycle_ go tool covdata textfmt -i=$${FUNC_E2E_GOCOVERDIR:-.coverage} -o coverage.txt +.PHONY: test-e2e-expose +test-e2e-expose: func-instrumented-bin ## Run external exposure E2E tests (--expose) + # Runtime and other options can be configured using the FUNC_E2E_* environment variables. see e2e_test.go + # Assertions about a Route skip unless the target cluster is OpenShift, and the + # assertions that func REFUSES a Route skip unless it is not. See records/test-plan-exposure.md + go test -tags e2e -timeout 30m ./e2e -v -run TestExpose_ + go tool covdata textfmt -i=$${FUNC_E2E_GOCOVERDIR:-.coverage} -o coverage.txt + .PHONY: test-e2e-config-ci test-e2e-config-ci: func-instrumented-bin ## CI tests for generated GitHub Workflows # Runtime and other options can be configured using the FUNC_E2E_* environment variables. see e2e_test.go diff --git a/cmd/client.go b/cmd/client.go index c4acf0802f..a258e7ab5a 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -6,7 +6,9 @@ import ( "os" "github.com/ory/viper" + "knative.dev/func/pkg/deployers" "knative.dev/func/pkg/keda" + "knative.dev/func/pkg/ocproute" "knative.dev/func/cmd/prompt" "knative.dev/func/pkg/buildpacks" @@ -70,7 +72,8 @@ func NewClient(cfg ClientConfig, options ...fn.Option) (*fn.Client, func()) { fn.WithRepositoriesPath(config.RepositoriesPath()), fn.WithScaffolder(buildpacks.NewScaffolder(cfg.Verbose)), fn.WithBuilder(buildpacks.NewBuilder(buildpacks.WithVerbose(cfg.Verbose))), - fn.WithRemovers(knative.NewRemover(cfg.Verbose), k8s.NewRemover(cfg.Verbose), keda.NewRemover(cfg.Verbose)), + fn.WithRemovers(knative.NewRemover(cfg.Verbose), k8s.NewRemover(cfg.Verbose), + keda.NewRemover(cfg.Verbose, keda.WithRemoverExposer(ocproute.New(deployers.Keda)))), fn.WithDescribers( knative.NewDescriber(cfg.Verbose, knative.WithDescriberTransport(t)), k8s.NewDescriber(cfg.Verbose, k8s.WithDescriberTransport(t)), @@ -172,22 +175,30 @@ func newKnativeDeployer(verbose bool) fn.Deployer { return knative.NewDeployer(options...) } +// newK8sDeployer builds the raw deployer. +// +// The Exposer is attached unconditionally, not only when the deploy asks for a +// Route. The record saying whether teardown is owed lives on the cluster, so +// wiring time cannot know. func newK8sDeployer(verbose bool) fn.Deployer { - options := []k8s.DeployerOpt{ + return k8s.NewDeployer( k8s.WithDeployerVerbose(verbose), k8s.WithDeployerDecorator(deployDecorator{}), - } - - return k8s.NewDeployer(options...) + k8s.WithExposer(ocproute.New(deployers.Kubernetes)), + ) } +// newKedaDeployer builds the keda deployer. The Exposer is keda's own, never +// the embedded raw deployer's, so Routes point at the interceptor rather than +// bypassing it. Attached unconditionally for the reason in newK8sDeployer, +// which bites harder here: keda's Route has no owner reference, so a Route +// nothing goes looking for is a Route nothing ever removes. func newKedaDeployer(verbose bool) fn.Deployer { - options := []keda.DeployerOpt{ + return keda.NewDeployer( keda.WithDeployerVerbose(verbose), keda.WithDeployerDecorator(deployDecorator{}), - } - - return keda.NewDeployer(options...) + keda.WithExposer(ocproute.New(deployers.Keda)), + ) } type deployDecorator struct { diff --git a/cmd/completion_util.go b/cmd/completion_util.go index 014f4136b1..80d9f51157 100644 --- a/cmd/completion_util.go +++ b/cmd/completion_util.go @@ -192,7 +192,7 @@ func CompleteDeployerList(cmd *cobra.Command, args []string, complete string) (m } func CompleteExposeList(cmd *cobra.Command, args []string, complete string) (matches []string, d cobra.ShellCompDirective) { - values := []string{"none", "route"} + values := fn.ExposeModes d = cobra.ShellCompDirectiveNoFileComp matches = []string{} diff --git a/cmd/deploy.go b/cmd/deploy.go index 169a1890b9..c882612e38 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -90,6 +90,13 @@ DESCRIPTION selectors. Note that the domain specified must be one of those configured or the flag will be ignored. + With '--expose=route' (raw and keda deployers on OpenShift) the domain + is used verbatim as the Route's hostname. DNS (pointing the name at the + cluster's router) and the TLS certificate (e.g. via cert-manager, whose + injected certificate deploys preserve) are the user's responsibility; + until both exist the name does not resolve or serves the router's + default certificate. + EXAMPLES o Deploy the function @@ -200,9 +207,15 @@ EXAMPLES "Service account to be used in the deployed function ($FUNC_SERVICE_ACCOUNT)") cmd.Flags().String("image-pull-secret", f.Deploy.ImagePullSecret, "Image pull secret to use when the function's image is in a private registry ($FUNC_IMAGE_PULL_SECRET)") - cmd.Flags().String("expose", f.Deploy.Expose, - "External exposure mode: 'route' (create a Route; OpenShift cluster only), "+ - "'none' (cluster-local opt-out). Raw and keda deployers only. ") + cmd.Flags().String("expose", f.Expose, + fmt.Sprintf("External exposure mode: '%s' for an OpenShift Route (OpenShift clusters only), "+ + "'%s' for cluster-local. Default: no exposure. Raw and keda deployers only. ($FUNC_EXPOSE)", + fn.ExposeRoute, fn.ExposeNone)) + cmd.Flags().StringP("namespace", "n", defaultNamespace(f, false), + "Deploy into a specific namespace. Will use the function's current namespace by default if already deployed, and the currently active context if it can be determined. ($FUNC_NAMESPACE)") + cmd.Flags().Bool("management-disabled", f.Deploy.ManagementDisabled, + "Disable operator management of this function ($FUNC_MANAGEMENT_DISABLED)") + // Static Flags: // Options which have static defaults only (not globally configurable nor // persisted with the function) @@ -220,10 +233,6 @@ EXAMPLES cmd.Flags().StringP("token", "", "", "Token to use when pushing to the registry. ($FUNC_TOKEN)") cmd.Flags().BoolP("build-timestamp", "", false, "Use the actual time as the created time for the docker image. This is only useful for buildpacks builder.") - cmd.Flags().Bool("management-disabled", f.Deploy.ManagementDisabled, - "Disable operator management of this function ($FUNC_MANAGEMENT_DISABLED)") - cmd.Flags().StringP("namespace", "n", defaultNamespace(f, false), - "Deploy into a specific namespace. Will use the function's current namespace by default if already deployed, and the currently active context if it can be determined. ($FUNC_NAMESPACE)") // Oft-shared flags: addConfirmFlag(cmd, cfg.Confirm) @@ -308,6 +317,23 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) (err error) { return } + // A Route is an OpenShift-only resource, not compatible with knative deployer. + // Dont error here, knative + expose=route means expose key is ignored and + // we print warning in warnExposeIgnore() + if f.Expose == fn.ExposeRoute && f.Deployer != deployers.Knative { + ok, probeErr := k8s.DetectOpenShift() + if probeErr != nil { + return fmt.Errorf("--expose=route requires an OpenShift cluster, and this one "+ + "could not be reached to check: %w. Fix the connection, or use --expose=none "+ + "to deploy cluster-local", probeErr) + } + if !ok { + return fmt.Errorf("--expose=route requires an OpenShift cluster: " + + "route.openshift.io Routes are an OpenShift-specific resource. " + + "Use --expose=none to deploy cluster-local") + } + } + changingNamespace := func(f fn.Function) bool { // We're changing namespace if: return f.Deploy.Namespace != "" && // it's already deployed @@ -329,8 +355,7 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) (err error) { // Informative non-error messages regarding the final deployment request printDeployMessages(cmd.OutOrStdout(), f) - // Get options based on the value of the config such as concrete impls - // of builders and pushers based on the value of the --builder flag + // create client with options from cfg clientOptions, err := cfg.clientOptions() if err != nil { return @@ -349,11 +374,21 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) (err error) { var url string // Invoke a remote build/push/deploy pipeline // Returned is the function with fields like Registry, f.Deploy.Image & - // f.Deploy.Namespace populated. + // f.Deploy.Namespace, f.Deploy.Expose populated. if url, f, err = client.RunPipeline(cmd.Context(), f); err != nil { return wrapDeploymentError(err) } fmt.Fprintf(cmd.OutOrStdout(), "Function Deployed at %v\n", url) + + // If it was intended [ActiveExpose(f.Expose)] to expose AND + // pipeline recorded no exposure[f.Deploy.Expose == ""] AND + // deployer is a relevant to 'Expose' func.yaml key => we are dealing + // with older (non-expose compatible) func-util img - warn the user + if fn.ActiveExpose(f.Expose) && f.Deploy.Expose == "" && + f.Deploy.Deployer != deployers.Knative { + fmt.Fprintf(cmd.OutOrStderr(), "Warning: expose %q was requested but the cluster's "+ + "func-util image applied no external exposure; the function is running cluster-local\n", f.Expose) + } } else { var buildOptions []fn.BuildOption if buildOptions, err = cfg.buildOptions(); err != nil { @@ -581,9 +616,9 @@ type deployConfig struct { // ManagementDisabled disables automatic Function CR sync after deploy. ManagementDisabled bool - // Expose controls external access - how/if the function should be - // exposed externally. Defaults to exposed on OpenShift, cluster-local - // elsewhere; "none" opts out explicitly. + // Expose is the intended external exposure mode from --expose / FUNC_EXPOSE + // (maps to Function.Expose). Defaults to cluster-local; "none" is explicit; + // "route" is OpenShift-only. Expose string } @@ -645,7 +680,7 @@ func (c deployConfig) Configure(f fn.Function) (fn.Function, error) { f.Deploy.ImagePullSecret = c.ImagePullSecret f.Deployer = c.Deployer f.Deploy.ManagementDisabled = c.ManagementDisabled - f.Deploy.Expose = c.Expose + f.Expose = c.Expose f.Local.Remote = c.Remote // PVCSize @@ -821,7 +856,8 @@ func (c deployConfig) Validate(cmd *cobra.Command) (err error) { return } -// clientOptions returns client options specific to deploy, including the appropriate deployer +// clientOptions returns client options specific to deploy, including the +// appropriate deployer func (c deployConfig) clientOptions() ([]fn.Option, error) { // Start with build config options o, err := c.buildConfig.clientOptions() @@ -939,7 +975,7 @@ func isDigested(v string) (validDigest bool, err error) { func warnExposeIgnore(w io.Writer, expose, deployer string) { if expose != "" && deployer != k8s.KubernetesDeployerName && deployer != keda.KedaDeployerName { - fmt.Fprintf(w, "warning: deploy.expose %q is ignored - only the raw and keda deployers "+ + fmt.Fprintf(w, "warning: expose %q is ignored - only the raw and keda deployers "+ "support external exposure via this field.\n", expose) } } diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go index f7bdac3f96..00ad2d15d4 100644 --- a/cmd/deploy_test.go +++ b/cmd/deploy_test.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "context" "errors" "fmt" @@ -1348,7 +1349,7 @@ func TestDeploy_BasicRedeployPipelinesCorrectNamespace(t *testing.T) { func TestDeploy_NamespaceChangePreservesExternalRegistry(t *testing.T) { root := FromTempDirectory(t) - cleanup := k8s.SetOpenShiftForTest(true) + cleanup := k8s.SetOpenShiftForTest(true, nil) defer cleanup() // Create a function deployed to "ns1" with an external registry @@ -1383,7 +1384,7 @@ func TestDeploy_NamespaceChangePreservesExternalRegistry(t *testing.T) { func TestDeploy_NamespaceChangeUpdatesInternalRegistry(t *testing.T) { root := FromTempDirectory(t) - cleanup := k8s.SetOpenShiftForTest(true) + cleanup := k8s.SetOpenShiftForTest(true, nil) defer cleanup() // Create a function deployed to "ns1" using the internal registry @@ -2775,12 +2776,12 @@ func TestDeploy_ExposeEmptyVsUnset(t *testing.T) { return f } - t.Run(`--expose="" clears a previously-persisted "none"`, func(t *testing.T) { + t.Run(`--expose="" clears a previously-persisted "none" intent`, func(t *testing.T) { root := newFn(t) deploy(t, "--deployer", "raw", "--expose", "none") - if f := loadFn(t, root); f.Deploy.Expose != "none" { - t.Fatalf("setup: expected expose 'none' to be persisted, got %q", f.Deploy.Expose) + if f := loadFn(t, root); f.Expose != "none" { + t.Fatalf("setup: expected intent expose 'none' to be persisted, got %q", f.Expose) } deploy(t, "--deployer", "raw", "--expose=") @@ -2795,26 +2796,34 @@ func TestDeploy_ExposeEmptyVsUnset(t *testing.T) { } }) - t.Run("plain deploy without the flag still works and leaves expose unpersisted", func(t *testing.T) { + t.Run("plain deploy without the flag leaves intent and status empty", func(t *testing.T) { root := newFn(t) deploy(t, "--deployer", "raw") - if f := loadFn(t, root); f.Deploy.Expose != "" { - t.Errorf("expected expose to remain unpersisted (empty), got %q", f.Deploy.Expose) + f := loadFn(t, root) + if f.Expose != "" { + t.Errorf("expected intent expose empty, got %q", f.Expose) + } + if f.Deploy.Expose != "" { + t.Errorf("expected status expose empty, got %q", f.Deploy.Expose) } }) - t.Run("persisted none + no flag round-trips untouched", func(t *testing.T) { + t.Run("persisted none intent + no flag round-trips", func(t *testing.T) { root := newFn(t) deploy(t, "--deployer", "raw", "--expose", "none") - if f := loadFn(t, root); f.Deploy.Expose != "none" { - t.Fatalf("expected expose 'none' to be persisted, got %q", f.Deploy.Expose) + if f := loadFn(t, root); f.Expose != "none" { + t.Fatalf("expected intent expose 'none', got %q", f.Expose) + } + // status is observed applied mode; "none"/empty both mean cluster-local + if f := loadFn(t, root); f.Deploy.Expose != "" { + t.Fatalf("expected status expose empty for cluster-local, got %q", f.Deploy.Expose) } - // redeploy without changing the flag should keep it as is + // redeploy without the flag should keep intent via flag default deploy(t, "--deployer", "raw") - if f := loadFn(t, root); f.Deploy.Expose != "none" { - t.Errorf("expected persisted 'none' to round-trip untouched, got %q", f.Deploy.Expose) + if f := loadFn(t, root); f.Expose != "none" { + t.Errorf("expected intent 'none' to round-trip, got %q", f.Expose) } }) } @@ -2838,10 +2847,13 @@ func TestDeploy_ExposeInvalidValueError(t *testing.T) { } } -// TestDeploy_ExposeRoutePersists ensures "route" round-trips through -// --expose into f.Deploy.Expose end-to-end. +// TestDeploy_ExposeRoutePersists ensures "route" round-trips as intent +// (Function.Expose) and observed status (Deploy.Expose) end-to-end. func TestDeploy_ExposeRoutePersists(t *testing.T) { root := FromTempDirectory(t) + // CLI gates route on OpenShift; tests run without a cluster. + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { t.Fatal(err) @@ -2861,8 +2873,11 @@ func TestDeploy_ExposeRoutePersists(t *testing.T) { if err != nil { t.Fatal(err) } + if f.Expose != "route" { + t.Fatalf("expected intent expose 'route', got %q", f.Expose) + } if f.Deploy.Expose != "route" { - t.Fatalf("expected expose 'route' to be persisted, got %q", f.Deploy.Expose) + t.Fatalf("expected status expose 'route', got %q", f.Deploy.Expose) } } @@ -2885,12 +2900,12 @@ func TestDeploy_ExposeIgnoredByDeployerNote(t *testing.T) { { name: "knative+route: warns, proceeds", args: []string{"--deployer", "knative", "--expose", "route"}, - wantWarning: `deploy.expose "route" is ignored - only the raw and keda deployers support external exposure via this field.`, + wantWarning: `expose "route" is ignored - only the raw and keda deployers support external exposure via this field.`, }, { name: "knative+none: warns, proceeds", args: []string{"--deployer", "knative", "--expose", "none"}, - wantWarning: `deploy.expose "none" is ignored - only the raw and keda deployers support external exposure via this field.`, + wantWarning: `expose "none" is ignored - only the raw and keda deployers support external exposure via this field.`, }, { name: "keda+route: silent, keda supports expose too", @@ -2905,6 +2920,9 @@ func TestDeploy_ExposeIgnoredByDeployerNote(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { root := FromTempDirectory(t) + // route cases need OpenShift gate open; none/empty do not care. + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { t.Fatal(err) } @@ -2929,7 +2947,7 @@ func TestDeploy_ExposeIgnoredByDeployerNote(t *testing.T) { } if tt.wantWarning == "" { - if strings.Contains(stderr.String(), "deploy.expose") { + if strings.Contains(stderr.String(), "expose") && strings.Contains(stderr.String(), "ignored") { t.Errorf("expected no warning on stderr, got:\n%s", stderr.String()) } return @@ -2940,3 +2958,109 @@ func TestDeploy_ExposeIgnoredByDeployerNote(t *testing.T) { }) } } + +// TestDeploy_RemoteExposeRecordsObservation: after a remote deploy the +// recorded exposure is what the pipeline's describer observed on the cluster, +// not just passed along from f.Expose. A pipeline whose func-util predates +// expose leaves no record, and that mismatch is warned about rather than +// papered over with a record of a Route that does not exist. +func TestDeploy_RemoteExposeRecordsObservation(t *testing.T) { + tests := []struct { + name string + // observed is what the pipeline run leaves in Deploy.Expose, standing + // in for what the on-cluster deployer recorded on the Service. + observed string + wantRecord string + wantWarning bool + }{ + {name: "pipeline honoured the intent", observed: fn.ExposeRoute, wantRecord: fn.ExposeRoute}, + {name: "stale func-util ignored the intent", observed: "", wantRecord: "", wantWarning: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + root := FromTempDirectory(t) + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() + + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + + pipeliner := mock.NewPipelinesProvider() + // Wrap the stock RunFn: run it, then stamp Deploy.Expose with what + // the describer would have read off the cluster. base snapshots the + // stock func value; stamping after the call mirrors the real provider, + // which records exposure only after the pipeline finishes. + base := pipeliner.RunFn + pipeliner.RunFn = func(f fn.Function) (string, fn.Function, error) { + // add exposure tracking to the base RunFn + url, f, err := base(f) + f.Deploy.Expose = tt.observed + return url, f, err + } + + cmd := NewDeployCmd(NewTestClient( + fn.WithPipelinesProvider(pipeliner), + fn.WithRegistry(TestRegistry), + )) + var out bytes.Buffer + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{"--remote", + "--git-url=https://example.com/user/repo", + "--deployer=raw", "--expose=route"}) + if err := cmd.Execute(); err != nil { + t.Fatal(err) + } + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Deploy.Expose != tt.wantRecord { + t.Errorf("Deploy.Expose = %q, want %q", f.Deploy.Expose, tt.wantRecord) + } + warned := strings.Contains(out.String(), "applied no external exposure") + if warned != tt.wantWarning { + t.Errorf("warning present = %v, want %v; output:\n%s", warned, tt.wantWarning, out.String()) + } + }) + } +} + +// TestDeploy_ExposeRouteUnreachableClusterIsNotAPlatformClaim: when the +// OpenShift probe gets no answer the deploy is still refused, since a Route +// on a cluster that may not serve Routes is what the gate prevents. But the +// error must name the connection as the cause. "Not OpenShift" is a claim +// about the cluster, and an unanswered probe cannot support it. +func TestDeploy_ExposeRouteUnreachableClusterIsNotAPlatformClaim(t *testing.T) { + root := FromTempDirectory(t) + + cleanup := k8s.SetOpenShiftForTest(false, errors.New("dial tcp 127.0.0.1:6443: connect: connection refused")) + defer cleanup() + + if _, err := fn.New().Init(fn.Function{Runtime: "go", Root: root}); err != nil { + t.Fatal(err) + } + + cmd := NewDeployCmd(NewTestClient( + fn.WithBuilder(mock.NewBuilder()), + fn.WithDeployer(mock.NewDeployer()), + fn.WithRegistry(TestRegistry), + )) + cmd.SetArgs([]string{"--deployer", "raw", "--expose=route"}) + var out strings.Builder + cmd.SetOut(&out) + cmd.SetErr(&out) + + err := cmd.Execute() + if err == nil { + t.Fatal("expected the deploy to be refused when the platform could not be determined") + } + if strings.Contains(err.Error(), "requires an OpenShift cluster: ") { + t.Errorf("the refusal asserts the cluster is not OpenShift, which was never established:\n%v", err) + } + if !strings.Contains(err.Error(), "could not be reached") { + t.Errorf("expected the refusal to name the real reason, got:\n%v", err) + } +} diff --git a/cmd/describe.go b/cmd/describe.go index f8beb99af2..129c561923 100644 --- a/cmd/describe.go +++ b/cmd/describe.go @@ -11,6 +11,7 @@ import ( "gopkg.in/yaml.v2" "knative.dev/func/pkg/config" + "knative.dev/func/pkg/deployers" fn "knative.dev/func/pkg/functions" ) @@ -135,6 +136,18 @@ func newDescribeConfig(cmd *cobra.Command, args []string) (cfg describeConfig, e type info fn.Instance +// routeMarker labels the route at idx " (exposed)" or " (cluster-local)". +// Raw and keda list the external route first; knative gets no label. +func (i info) routeMarker(idx int) string { + if i.Deployer != deployers.Kubernetes && i.Deployer != deployers.Keda { + return "" + } + if i.Expose == fn.ExposeRoute && idx == 0 { + return " (exposed)" + } + return " (cluster-local)" +} + func (i info) Human(w io.Writer) error { fmt.Fprintln(w, "Function name:") fmt.Fprintf(w, " %v\n", i.Name) @@ -144,8 +157,8 @@ func (i info) Human(w io.Writer) error { fmt.Fprintf(w, " %v\n", i.Namespace) fmt.Fprintln(w, "Routes:") - for _, route := range i.Routes { - fmt.Fprintf(w, " %v\n", route) + for idx, route := range i.Routes { + fmt.Fprintf(w, " %v%v\n", route, i.routeMarker(idx)) } fmt.Fprintln(w, "Function is ready:") diff --git a/cmd/describe_test.go b/cmd/describe_test.go index 10b1206c2b..73c53636ae 100644 --- a/cmd/describe_test.go +++ b/cmd/describe_test.go @@ -6,11 +6,40 @@ import ( "strings" "testing" + "knative.dev/func/pkg/deployers" fn "knative.dev/func/pkg/functions" "knative.dev/func/pkg/mock" . "knative.dev/func/pkg/testing" ) +// Test_routeMarker pins the human-facing route labels. Only the raw and keda +// deployers get markers; their describers list the external route first when +// one exists. Knative manages its own exposure and gets no label. +func Test_routeMarker(t *testing.T) { + tests := []struct { + name string + deployer string + expose string + idx int + want string + }{ + {"knative is never marked", deployers.Knative, "", 0, ""}, + {"raw exposed: external route first", deployers.Kubernetes, fn.ExposeRoute, 0, " (exposed)"}, + {"raw exposed: internal route second", deployers.Kubernetes, fn.ExposeRoute, 1, " (cluster-local)"}, + {"raw cluster-local", deployers.Kubernetes, "", 0, " (cluster-local)"}, + {"keda exposed: external route first", deployers.Keda, fn.ExposeRoute, 0, " (exposed)"}, + {"keda cluster-local", deployers.Keda, "", 0, " (cluster-local)"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := info{Deployer: tt.deployer, Expose: tt.expose} + if got := i.routeMarker(tt.idx); got != tt.want { + t.Errorf("routeMarker(%d) = %q, want %q", tt.idx, got, tt.want) + } + }) + } +} + // TestDescribe_Default ensures that running describe when there is no // function in the given directory fails correctly. func TestDescribe_Default(t *testing.T) { diff --git a/cmd/errors.go b/cmd/errors.go index f99ed9db3b..97042ff5a3 100644 --- a/cmd/errors.go +++ b/cmd/errors.go @@ -233,10 +233,10 @@ func (e *ErrInvalidExpose) Error() string { Try this: func deploy --expose=route Create an OpenShift Route (OpenShift clusters only) - func deploy --expose=none Cluster-local opt-out, no external exposure + func deploy --expose=none Cluster-local, no external exposure -deploy.expose takes effect with the raw and keda deployers only (--deployer=raw or --deployer=keda), -which expose by default when the platform and deployer support it. +deploy.expose takes effect with the raw and keda deployers only (--deployer=raw or --deployer=keda). +Functions are cluster-local by default; external exposure is opt-in. For more options, run 'func deploy --help'`, e.Err) } diff --git a/cmd/func-util/main.go b/cmd/func-util/main.go index 7a9dcaf48c..41c01e4124 100644 --- a/cmd/func-util/main.go +++ b/cmd/func-util/main.go @@ -18,10 +18,12 @@ import ( "k8s.io/klog/v2" "knative.dev/func/pkg/buildpacks" + "knative.dev/func/pkg/deployers" fn "knative.dev/func/pkg/functions" "knative.dev/func/pkg/k8s" "knative.dev/func/pkg/keda" "knative.dev/func/pkg/knative" + "knative.dev/func/pkg/ocproute" "knative.dev/func/pkg/s2i" "knative.dev/func/pkg/scaffolding" "knative.dev/func/pkg/tar" @@ -173,16 +175,34 @@ func deploy(ctx context.Context) error { d = k8s.NewDeployer( k8s.WithDeployerDecorator(deployDecorator{}), k8s.WithDeployerVerbose(true), + k8s.WithExposer(ocproute.New(deployers.Kubernetes)), ) case keda.KedaDeployerName: d = keda.NewDeployer( keda.WithDeployerDecorator(deployDecorator{}), keda.WithDeployerVerbose(true), + keda.WithExposer(ocproute.New(deployers.Keda)), ) default: return fmt.Errorf("unknown deployer: %s", deployer) } + // Refuse an impossible request rather than deploying and reporting a + // success that carries no external address; this catches intent carried + // on-cluster in func.yaml. Same rule and same split as the CLI gate; only + // the wording differs (the user is holding func.yaml here, not a flag). + if f.Expose == fn.ExposeRoute && deployer != knative.KnativeDeployerName { + ok, probeErr := k8s.DetectOpenShift() + if probeErr != nil { + return fmt.Errorf("expose is %q but this cluster could not be asked whether it "+ + "serves route.openshift.io: %w", f.Expose, probeErr) + } + if !ok { + return fmt.Errorf("expose is %q but this is not an OpenShift cluster: "+ + "route.openshift.io Routes are an OpenShift-specific resource", f.Expose) + } + } + client := fn.New(fn.WithDeployer(d)) res, err := client.Deploy(ctx, f, fn.WithDeploySkipBuildCheck(true)) if err != nil { diff --git a/docs/reference/func_deploy.md b/docs/reference/func_deploy.md index 9ecd104022..6fdb998af7 100644 --- a/docs/reference/func_deploy.md +++ b/docs/reference/func_deploy.md @@ -66,6 +66,13 @@ DESCRIPTION selectors. Note that the domain specified must be one of those configured or the flag will be ignored. + With '--expose=route' (raw and keda deployers on OpenShift) the domain + is used verbatim as the Route's hostname. DNS (pointing the name at the + cluster's router) and the TLS certificate (e.g. via cert-manager, whose + injected certificate deploys preserve) are the user's responsibility; + until both exist the name does not resolve or serves the router's + default certificate. + EXAMPLES o Deploy the function @@ -122,7 +129,7 @@ func deploy --deployer string Type of deployment to use: 'knative' for Knative Service, 'raw' for Kubernetes Deployment, or 'keda' for Deployment with a KEDA HTTP scaler ($FUNC_DEPLOYER) (default "knative") --domain string Domain to use for the function's route. Cluster must be configured with domain matching for the given domain (ignored if unrecognized) ($FUNC_DOMAIN) -e, --env stringArray Environment variable to set in the form NAME=VALUE. You may provide this flag multiple times for setting multiple environment variables. To unset, specify the environment variable name followed by a "-" (e.g., NAME-). - --expose string External exposure mode: 'route' (create a Route; OpenShift clusters only), 'none' (cluster-local opt-out). Raw and keda deployers only. Defaults to exposed on OpenShift, cluster-local elsewhere. An explicitly empty value (--expose="") clears the persisted deploy.expose key and returns to the default. ($FUNC_EXPOSE) + --expose string External exposure mode: 'route' for an OpenShift Route (OpenShift clusters only), 'none' for cluster-local. Default: no exposure. Raw and keda deployers only. ($FUNC_EXPOSE) -t, --git-branch string Git revision (branch) to be used when deploying via the Git repository ($FUNC_GIT_BRANCH) -d, --git-dir string Directory in the Git repository containing the function (default is the root) ($FUNC_GIT_DIR) -g, --git-url string Repository url containing the function to build ($FUNC_GIT_URL) diff --git a/e2e/e2e_expose_test.go b/e2e/e2e_expose_test.go new file mode 100644 index 0000000000..b0c7215d0f --- /dev/null +++ b/e2e/e2e_expose_test.go @@ -0,0 +1,790 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/http" + "os" + "slices" + "strings" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8slabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + + fn "knative.dev/func/pkg/functions" + "knative.dev/func/pkg/k8s" + fnlabels "knative.dev/func/pkg/k8s/labels" + "knative.dev/func/pkg/keda" +) + +// --------------------------------------------------------------------------- +// EXPOSE TESTS +// External exposure of a deployed function: --expose, its persistence, and +// the platform gate. +// +// Split by what the cluster can answer, and routed by IsOpenShift() as the +// first statement of each test. Anything asserting a Route needs the +// route.openshift.io API and so runs on OpenShift and skips elsewhere. +// Anything asserting func REFUSES a Route is only meaningful where that API is +// absent, so it runs on KinD and skips on OpenShift. Neither environment runs +// the whole file, by design: point the suite at each in turn and every test +// runs somewhere. +// +// The object graph behind an exposure - the Route's namespace, target, owner +// references, and its host's registration with the keda interceptor - is +// asserted in the pkg/ocproute and pkg/keda integration tests, which gate the +// same way. What is here is the CLI contract: what the user asked for, what +// they were told, and what func.yaml records afterwards. +// --------------------------------------------------------------------------- + +// requiresOpenShift skips a test whose assertions need a real Route. +func requiresOpenShift(t *testing.T) { + t.Helper() + // The gate must answer for the suite's cluster, not the ambient one, and + // IsOpenShift caches its first answer for the whole binary. setupEnv sets + // this same value later; the probe needs it first. + os.Setenv("KUBECONFIG", Kubeconfig) + if !k8s.IsOpenShift() { + t.Skip("not an OpenShift cluster: route.openshift.io is unavailable, " + + "so there is no Route to assert on") + } +} + +// requiresNotOpenShift skips a test that asserts func declines to do something +// no non-OpenShift cluster can do. On OpenShift the refusal correctly does not +// happen, so the test has nothing to say there. +func requiresNotOpenShift(t *testing.T) { + t.Helper() + os.Setenv("KUBECONFIG", Kubeconfig) // see requiresOpenShift + if k8s.IsOpenShift() { + t.Skip("this is an OpenShift cluster, where a Route is possible, " + + "so there is no refusal to assert on") + } +} + +// TestExpose_RouteRequiresOpenShift ensures asking for a Route on a cluster +// that has no Route API is refused, and refused BEFORE anything is created. +// Deploying and then failing would leave a running function the user believes +// is externally reachable and a func.yaml that never recorded the attempt. +// +// func deploy --expose=route +func TestExpose_RouteRequiresOpenShift(t *testing.T) { + requiresNotOpenShift(t) + + name := "func-e2e-test-expose-gate" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + + out, err := newCmdOutput(t, "deploy", "--deployer=raw", "--expose=route").CombinedOutput() + if err == nil { + t.Fatal("expected --expose=route to be refused on a cluster with no Route API") + } + if !strings.Contains(string(out), "route.openshift.io") { + t.Errorf("expected the error to name the missing API, got:\n%s", out) + } + + // Nothing should have been recorded, because nothing was deployed. + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Deploy.Namespace != "" { + t.Errorf("expected no recorded deployment after a refused deploy, got namespace %q", f.Deploy.Namespace) + } +} + +// TestExpose_RemoteRouteRequiresOpenShift ensures the platform gate refuses +// --expose=route on a remote deploy too, and before any pipeline work: the +// refusal needs no Tekton on the cluster and leaves nothing behind. +// +// func deploy --remote --expose=route +func TestExpose_RemoteRouteRequiresOpenShift(t *testing.T) { + requiresNotOpenShift(t) + + name := "func-e2e-test-expose-remote-gate" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + + out, err := newCmdOutput(t, "deploy", "--remote", "--deployer=raw", "--expose=route", + "--registry="+Registry).CombinedOutput() + if err == nil { + t.Fatal("expected a remote --expose=route to be refused on a cluster with no Route API") + } + if !strings.Contains(string(out), "route.openshift.io") { + t.Errorf("expected the error to name the missing API, got:\n%s", out) + } + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Deploy.Namespace != "" { + t.Errorf("expected no recorded deployment after a refused remote deploy, got namespace %q", f.Deploy.Namespace) + } +} + +// TestExpose_ClusterLocalByDefault ensures a function deployed without the +// flag is cluster-local, and stays that way in the record. Knative functions +// get an external URL by default, so this is the default most likely to +// surprise a deployer-switching user: raw exposes only when asked. +// +// func deploy +func TestExpose_ClusterLocalByDefault(t *testing.T) { + name := "func-e2e-test-expose-default" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--deployer=raw").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Expose != "" { + t.Errorf("expected no exposure intent recorded, got %q", f.Expose) + } + if f.Deploy.Expose != "" { + t.Errorf("expected no exposure applied, got %q", f.Deploy.Expose) + } +} + +// TestExpose_KedaRejectsLongName ensures a function name that is legal on its +// own but too long once keda's bridge suffix is added is refused up front, +// rather than by an opaque API rejection after the Deployment already exists. +// +// func deploy --deployer=keda +func TestExpose_KedaRejectsLongName(t *testing.T) { + // 45 characters: legal as a function name (DNS-1035 allows 63), one over + // what keda's "-interceptor-bridge" suffix leaves room for. + name := "func-e2e-test-expose-name-far-too-long-for-ke" + if len(name) != 45 { + t.Fatalf("test setup: expected a 45 character name, got %d", len(name)) + } + fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + + out, err := newCmdOutput(t, "deploy", "--deployer=keda").CombinedOutput() + if err == nil { + t.Fatal("expected a name too long for keda's bridge Service to be refused") + } + if !strings.Contains(string(out), "too long") { + t.Errorf("expected the error to explain the length limit, got:\n%s", out) + } +} + +// TestExpose_Route walks the raw exposure lifecycle in the order a user +// would: deploy cluster-local, decide later to expose, opt out again, +// re-expose, delete. Each leg asserts the Route and the Service records that +// deploy leaves behind. The last leg asserts delete takes the Route with it, +// which for the raw deployer happens through its owner reference rather than +// by remover code, so the disappearance is polled, not read once. +// +// func deploy ; --expose=route ; --expose=none ; --expose=route ; func delete +func TestExpose_Route(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-test-expose-route" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + + // Cluster-local first: no intent, nothing applied, no Route. + if err := newCmd(t, "deploy", "--deployer=raw").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + ns := f.Deploy.Namespace + if f.Deploy.Expose != "" { + t.Errorf("expected no exposure applied on a flagless deploy, got %q", f.Deploy.Expose) + } + if n := routeCount(t, ns, name, ns); n != 0 { + t.Fatalf("expected no Route for a cluster-local function, found %d in %q", n, ns) + } + + // Exposing an already-running function: the Route appears and the Service + // records both halves of the exposure. The raw deployer's Route sits + // beside its function. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + if f, err = fn.NewFunction(root); err != nil { + t.Fatal(err) + } + if f.Expose != fn.ExposeRoute { + t.Errorf("expected intent %q, got %q", fn.ExposeRoute, f.Expose) + } + if f.Deploy.Expose != fn.ExposeRoute { + t.Errorf("expected applied exposure %q, got %q", fn.ExposeRoute, f.Deploy.Expose) + } + ann := serviceAnnotations(t, ns, name) + if ann[k8s.RouteHostnameAnnotation] == "" { + t.Error("expected the Service to record the exposed hostname") + } + if got := ann[k8s.RouteNamespaceAnnotation]; got != ns { + t.Errorf("expected the Route recorded in the function's namespace %q, got %q", ns, got) + } + if n := routeCount(t, ns, name, ns); n != 1 { + t.Fatalf("expected 1 Route in %q, found %d", ns, n) + } + + // Turning it off must take the Route away, clear the records, and leave + // the function running. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=none").Run(); err != nil { + t.Fatal(err) + } + if f, err = fn.NewFunction(root); err != nil { + t.Fatal(err) + } + if f.Deploy.Expose != "" { + t.Errorf("expected applied exposure cleared after opting out, got %q", f.Deploy.Expose) + } + if n := routeCount(t, ns, name, ns); n != 0 { + t.Errorf("expected the Route removed on opt-out, found %d in %q", n, ns) + } + ann = serviceAnnotations(t, ns, name) // the Service surviving is itself the liveness assert + if v := ann[k8s.RouteHostnameAnnotation]; v != "" { + t.Errorf("expected the hostname record cleared on opt-out, got %q", v) + } + if v := ann[k8s.RouteNamespaceAnnotation]; v != "" { + t.Errorf("expected the namespace record cleared on opt-out, got %q", v) + } + + // On again, so delete below removes an exposed function. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + if n := routeCount(t, ns, name, ns); n != 1 { + t.Fatalf("expected the Route back after re-exposing, found %d in %q", n, ns) + } + + // Delete while exposed: the Route goes by owner reference, so garbage + // collection is given a deadline rather than one look. + if err := newCmd(t, "delete").Run(); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(60 * time.Second) + for { + if n := routeCount(t, ns, name, ns); n == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("expected garbage collection to remove the Route after delete, still present in %q", ns) + } + time.Sleep(3 * time.Second) + } +} + +// TestExpose_KedaRoute ensures a keda function can be exposed and unexposed +// through the CLI. Keda's Route is the one nothing garbage collects: a Route +// left behind by an opt-out survives forever. +// +// func deploy --deployer=keda --expose=route +func TestExpose_KedaRoute(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-test-expose-keda" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + if f.Deploy.Deployer != "keda" { + t.Fatalf("expected the keda deployer to be recorded, got %q", f.Deploy.Deployer) + } + if f.Deploy.Expose != fn.ExposeRoute { + t.Errorf("expected applied exposure %q, got %q", fn.ExposeRoute, f.Deploy.Expose) + } + + // An exposed keda function must lead with its external URL. The bridge + // addresses stay listed, but they are cluster-local and answer nothing + // from outside. + out, err := newCmdOutput(t, "describe", "-o=plain").CombinedOutput() + if err != nil { + t.Fatalf("describe failed: %v\n%s", err, out) + } + if !strings.Contains(string(out), "https://") { + t.Errorf("expected describe to report an https URL for an exposed function, got:\n%s", out) + } + + // Opting out must take the Route away. Nothing else ever will. + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=none").Run(); err != nil { + t.Fatal(err) + } + if f, err = fn.NewFunction(root); err != nil { + t.Fatal(err) + } + if f.Deploy.Expose != "" { + t.Errorf("expected applied exposure cleared after opting out, got %q", f.Deploy.Expose) + } +} + +// routeCount reports how many Routes in ns carry this function's identity +// labels. The lookup mirrors the one func delete uses, so these tests fail if +// the identity rules change. +func routeCount(t *testing.T, ns, fnName, fnNamespace string) int { + t.Helper() + client, err := k8s.NewDynamicClient() + if err != nil { + t.Fatal(err) + } + gvr := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"} + sel := k8slabels.SelectorFromSet(k8slabels.Set{ + fnlabels.FunctionKey: "true", + fnlabels.FunctionNameKey: fnName, + fnlabels.FunctionNamespaceKey: fnNamespace, + }).String() + list, err := client.Resource(gvr).Namespace(ns).List(context.Background(), metav1.ListOptions{LabelSelector: sel}) + if err != nil { + t.Fatal(err) + } + return len(list.Items) +} + +// serviceAnnotations returns the function Service's annotations, where deploy +// records the exposed hostname and the Route's namespace. +func serviceAnnotations(t *testing.T, ns, name string) map[string]string { + t.Helper() + clientset, err := k8s.NewKubernetesClientset() + if err != nil { + t.Fatal(err) + } + svc, err := clientset.CoreV1().Services(ns).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return svc.Annotations +} + +// TestExpose_KedaToggle ensures exposure turns off and on again in place. +// Off must remove the Route and both Service records while the function keeps +// running; on must bring them back. Keda's Route is the one nothing garbage +// collects, so off doing its half is what keeps an opt-out from leaking. +// +// func deploy --deployer=keda --expose=route ; --expose=none ; --expose=route +func TestExpose_KedaToggle(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-expose-keda-toggle" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + ns := f.Deploy.Namespace + + ann := serviceAnnotations(t, ns, name) + recordedNS := ann[k8s.RouteNamespaceAnnotation] + if recordedNS == "" { + t.Fatal("expected the Service to record the Route's namespace") + } + if ann[k8s.RouteHostnameAnnotation] == "" { + t.Error("expected the Service to record the exposed hostname") + } + if n := routeCount(t, recordedNS, name, ns); n != 1 { + t.Fatalf("expected 1 Route in %q, found %d", recordedNS, n) + } + + // Off: the Route and both records go; the function does not. + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=none").Run(); err != nil { + t.Fatal(err) + } + ann = serviceAnnotations(t, ns, name) // the Service surviving is itself the liveness assert + if v := ann[k8s.RouteNamespaceAnnotation]; v != "" { + t.Errorf("expected the namespace record cleared on opt-out, got %q", v) + } + if v := ann[k8s.RouteHostnameAnnotation]; v != "" { + t.Errorf("expected the hostname record cleared on opt-out, got %q", v) + } + if n := routeCount(t, recordedNS, name, ns); n != 0 { + t.Errorf("expected the Route removed on opt-out, found %d in %q", n, recordedNS) + } + + // On again: both come back. + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + if n := routeCount(t, recordedNS, name, ns); n != 1 { + t.Errorf("expected the Route back after re-exposing, found %d in %q", n, recordedNS) + } + if serviceAnnotations(t, ns, name)[k8s.RouteNamespaceAnnotation] == "" { + t.Error("expected the namespace record back after re-exposing") + } +} + +// TestExpose_KedaDeleteCleansRoute ensures func delete removes an exposed keda +// function's Route. The Route lives in the interceptor's namespace with no +// owner, so delete is the only thing that ever removes it; one left behind is +// an orphan forever. +// +// func deploy --deployer=keda --expose=route ; func delete +func TestExpose_KedaDeleteCleansRoute(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-expose-keda-delete" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + ns := f.Deploy.Namespace + + recordedNS := serviceAnnotations(t, ns, name)[k8s.RouteNamespaceAnnotation] + if recordedNS == "" { + t.Fatal("expected the Service to record the Route's namespace") + } + if n := routeCount(t, recordedNS, name, ns); n != 1 { + t.Fatalf("expected 1 Route before delete, found %d in %q", n, recordedNS) + } + + if err := newCmd(t, "delete").Run(); err != nil { + t.Fatal(err) + } + + if n := routeCount(t, recordedNS, name, ns); n != 0 { + t.Errorf("expected delete to remove the Route, found %d left in %q", n, recordedNS) + } + clientset, err := k8s.NewKubernetesClientset() + if err != nil { + t.Fatal(err) + } + if _, err := clientset.CoreV1().Services(ns).Get(context.Background(), name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("expected the function's Service gone after delete, got %v", err) + } +} + +// TestExpose_KedaRouteDomain ensures a custom domain rides keda's exposure: +// the Route in the interceptor's namespace carries the domain as its host, +// the Service records it as the exposed hostname, and the HTTPScaledObject +// registers it, since the interceptor 404s any Host header its +// HTTPScaledObject does not list. TLS and traffic for a custom domain are +// TestExpose_RouteDomainTLS's assert; the Route machinery is shared. +// +// func deploy --deployer=keda --expose=route --domain= +func TestExpose_KedaRouteDomain(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-expose-keda-domain" + root := fromCleanEnv(t, name) + const domain = "func-e2e-expose-keda-domain.test" + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--deployer=keda", "--expose=route", "--domain="+domain).Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + ns := f.Deploy.Namespace + + ann := serviceAnnotations(t, ns, name) + if got := ann[k8s.RouteHostnameAnnotation]; got != domain { + t.Errorf("expected the custom domain %q recorded as the exposed hostname, got %q", domain, got) + } + recordedNS := ann[k8s.RouteNamespaceAnnotation] + if recordedNS == "" { + t.Fatal("expected the Service to record the Route's namespace") + } + route := routeFor(t, recordedNS, name, ns) + if host, _, _ := unstructured.NestedString(route.Object, "spec", "host"); host != domain { + t.Errorf("expected spec.host %q, got %q", domain, host) + } + + hsoClient, err := keda.NewHTTPScaledObjectClientset() + if err != nil { + t.Fatal(err) + } + hso, err := hsoClient.HttpV1alpha1().HTTPScaledObjects(ns).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(hso.Spec.Hosts, domain) { + t.Errorf("expected the HTTPScaledObject to register the domain %q, hosts: %v", domain, hso.Spec.Hosts) + } +} + +// routeFor returns the one Route carrying this function's identity labels. +func routeFor(t *testing.T, ns, fnName, fnNamespace string) *unstructured.Unstructured { + t.Helper() + client, err := k8s.NewDynamicClient() + if err != nil { + t.Fatal(err) + } + gvr := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"} + sel := k8slabels.SelectorFromSet(k8slabels.Set{ + fnlabels.FunctionKey: "true", + fnlabels.FunctionNameKey: fnName, + fnlabels.FunctionNamespaceKey: fnNamespace, + }).String() + list, err := client.Resource(gvr).Namespace(ns).List(context.Background(), metav1.ListOptions{LabelSelector: sel}) + if err != nil { + t.Fatal(err) + } + if len(list.Items) != 1 { + t.Fatalf("expected exactly 1 Route for %s in %s, found %d", fnName, ns, len(list.Items)) + } + return &list.Items[0] +} + +// selfSignedCert returns a PEM cert and key for domain plus a pool trusting +// them: the test's stand-in for what cert-manager would issue. +func selfSignedCert(t *testing.T, domain string) (certPEM, keyPEM string, pool *x509.CertPool) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: domain}, + DNSNames: []string{domain}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + certPEM = string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + keyPEM = string(pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})) + pool = x509.NewCertPool() + pool.AppendCertsFromPEM([]byte(certPEM)) + return certPEM, keyPEM, pool +} + +// TestExpose_RouteDomainTLS proves the custom-domain chain end to end with +// neither cert-manager nor DNS: the test plays the certificate controller by +// injecting a self-signed cert into the Route, a redeploy proves func carries +// the injection over, and an HTTPS request dialed straight at the router +// (DNS bypassed) must be served with exactly that certificate. +// +// func deploy --deployer=raw --expose=route --domain= +func TestExpose_RouteDomainTLS(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-expose-domain" + root := fromCleanEnv(t, name) + const domain = "func-e2e-expose-domain.test" + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + // First without a domain: the minted host is how the router is found, + // since the custom name deliberately has no DNS. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + ns := f.Deploy.Namespace + minted := serviceAnnotations(t, ns, name)[k8s.RouteHostnameAnnotation] + if minted == "" { + t.Fatal("expected a minted hostname to locate the router with") + } + routerAddrs, err := net.LookupHost(minted) + if err != nil || len(routerAddrs) == 0 { + t.Fatalf("could not resolve the router via %q: %v", minted, err) + } + + // The domain lands verbatim on the Route (a recreate: in-place host + // updates are permission-gated) and in the record. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=route", "--domain="+domain).Run(); err != nil { + t.Fatal(err) + } + if got := serviceAnnotations(t, ns, name)[k8s.RouteHostnameAnnotation]; got != domain { + t.Fatalf("expected the custom domain %q recorded, got %q", domain, got) + } + route := routeFor(t, ns, name, ns) + if host, _, _ := unstructured.NestedString(route.Object, "spec", "host"); host != domain { + t.Fatalf("expected spec.host %q, got %q", domain, host) + } + + // Play the certificate controller: inject a self-signed cert for the + // domain, exactly as cert-manager's openshift-routes plugin would. + certPEM, keyPEM, pool := selfSignedCert(t, domain) + client, err := k8s.NewDynamicClient() + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(route.Object, certPEM, "spec", "tls", "certificate"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(route.Object, keyPEM, "spec", "tls", "key"); err != nil { + t.Fatal(err) + } + gvr := schema.GroupVersionResource{Group: "route.openshift.io", Version: "v1", Resource: "routes"} + if _, err := client.Resource(gvr).Namespace(ns).Update(context.Background(), route, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + // A redeploy must not wipe the injected material. + if err := newCmd(t, "deploy", "--deployer=raw", "--expose=route", "--domain="+domain).Run(); err != nil { + t.Fatal(err) + } + route = routeFor(t, ns, name, ns) + if cert, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "certificate"); cert != certPEM { + t.Fatal("expected the injected certificate to survive a redeploy") + } + + // HTTPS through the router, DNS bypassed, trust anchored only at the + // injected cert: a 200 here proves routing AND that the router serves + // the user's certificate for the custom domain. + dial := func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, net.JoinHostPort(routerAddrs[0], "443")) + } + httpClient := &http.Client{ + Transport: &http.Transport{DialContext: dial, TLSClientConfig: &tls.Config{RootCAs: pool}}, + Timeout: 10 * time.Second, + } + deadline := time.Now().Add(90 * time.Second) + for { + resp, getErr := httpClient.Get("https://" + domain + "/") + if getErr == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + break + } + getErr = fmt.Errorf("status %d", resp.StatusCode) + } + if time.Now().After(deadline) { + t.Fatalf("the router never served the custom domain with the injected cert: %v", getErr) + } + time.Sleep(3 * time.Second) + } +} + +// TestExpose_RemoteRoute ensures the exposure chain holds when the deploy +// runs in-cluster: intent travels in func.yaml, the pipeline's func-util +// creates the Route, and the CLI records what the pipeline's describer read +// back rather than what was asked for. Presumes Tekton on the cluster, like +// every remote test, and a func-util image built from this source +// (make FUNC_UTILS_IMG=): a published image that predates expose +// deploys cluster-local, which is exactly the drift this test catches. +// +// func deploy --remote --deployer=raw --expose=route +func TestExpose_RemoteRoute(t *testing.T) { + requiresOpenShift(t) + + name := "func-e2e-expose-remote" + root := fromCleanEnv(t, name) + + if err := newCmd(t, "init", "-l=go").Run(); err != nil { + t.Fatal(err) + } + if err := newCmd(t, "deploy", "--remote", "--builder=pack", "--registry="+Registry, + "--deployer=raw", "--expose=route").Run(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = newCmd(t, "delete").Run() }) + + f, err := fn.NewFunction(root) + if err != nil { + t.Fatal(err) + } + // Recorded from the cluster by the pipeline describer; empty here means + // the pipeline ran a func-util that ignored the intent. + if f.Deploy.Expose != fn.ExposeRoute { + t.Fatalf("expected applied exposure %q read back from the cluster, got %q", fn.ExposeRoute, f.Deploy.Expose) + } + ns := f.Deploy.Namespace + ann := serviceAnnotations(t, ns, name) + if ann[k8s.RouteHostnameAnnotation] == "" { + t.Error("expected the Service to record the exposed hostname") + } + if got := ann[k8s.RouteNamespaceAnnotation]; got != ns { + t.Errorf("expected the Route recorded in the function's namespace %q, got %q", ns, got) + } + if n := routeCount(t, ns, name, ns); n != 1 { + t.Fatalf("expected 1 Route in %q, found %d", ns, n) + } +} + +// Not covered here, deliberately: +// +// - Route object shape, admission, and the refusal to adopt a hand-authored +// Route. Those assert on cluster objects rather than on the CLI contract, +// so they belong in pkg/ocproute and pkg/keda integration tests, which +// gate on IsOpenShift() the same way. +// - Anything involving an account without rights in the interceptor's +// namespace. Every permission-shaped defect in this feature was found by +// hand because no rig here builds a restricted Role; exercising one needs +// a cluster with a deliberately limited account. diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index eff66cd455..2039c6cc61 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -174,6 +174,12 @@ var ( // Can be set with FUNC_E2E_NAMESPACE Namespace string + // namespaceExplicit records whether FUNC_E2E_NAMESPACE was set, as opposed + // to Namespace holding its default. setupEnv forces the CLI's namespace + // only when it was, so an unset run keeps deploying wherever the + // kubeconfig's current context points. + namespaceExplicit bool + // Plugin indicates func is being run as a plugin within Bin, and // the value of this argument is the subcommand. For example, when // running e2e tests as a plugin to `kn`, Bin will be /path/to/kn and @@ -337,7 +343,7 @@ func readEnvs() { // Final = current ENV, deprecated ENV, default // BrokerHost - the hostname of the Knative broker ingress - BrokerHost = getEnv("FUNC_E2E_BROKER_HOST", "", DefaultBrokerHost) + BrokerHost, _ = getEnv("FUNC_E2E_BROKER_HOST", "", DefaultBrokerHost) // Clean up deployed functions before starting next test Clean = getEnvBool("FUNC_E2E_CLEAN", "", DefaultClean) @@ -346,10 +352,10 @@ func readEnvs() { CleanImages = getEnvBool("FUNC_E2E_CLEAN_IMAGES", "", DefaultCleanImages) // DockerHost - the DOCKER_HOST to use for container operations (not including podman-specific tests) - DockerHost = getEnv("FUNC_E2E_DOCKER_HOST", "", "") + DockerHost, _ = getEnv("FUNC_E2E_DOCKER_HOST", "", "") // Domain - the DNS domain suffix for function URLs - Domain = getEnv("FUNC_E2E_DOMAIN", "", DefaultDomain) + Domain, _ = getEnv("FUNC_E2E_DOMAIN", "", DefaultDomain) // Gocoverdir - the coverage directory to use while testing the go binary. Gocoverdir = getEnvPath("FUNC_E2E_GOCOVERDIR", "", DefaultGocoverdir) @@ -373,12 +379,15 @@ func readEnvs() { // Templates MatrixTemplates = getEnvList("FUNC_E2E_MATRIX_TEMPLATES", "", toCSV(MatrixTemplates)) - // Namespace - the Kubernetes namespace where functions will be deployed - Namespace = getEnv("FUNC_E2E_NAMESPACE", "", DefaultNamespace) + // Namespace - the Kubernetes namespace where functions will be deployed. + // namespaceExplicit records whether FUNC_E2E_NAMESPACE was set, as + // opposed to Namespace holding its default. Only an explicitly set + // namespace is forced on the CLI; see setupEnv. + Namespace, namespaceExplicit = getEnv("FUNC_E2E_NAMESPACE", "", DefaultNamespace) // Plugin - if set, func is a plugin and Bin is the one plugging. The value // is the name of the subcommand. - Plugin = getEnv("FUNC_E2E_PLUGIN", "E2E_USE_KN_FUNC", "") + Plugin, _ = getEnv("FUNC_E2E_PLUGIN", "E2E_USE_KN_FUNC", "") // Plugin Backwards compatibility: // If set to "true", the default value is "func" because the deprecated // value was literal string "true". @@ -395,7 +404,7 @@ func readEnvs() { // PodmanHost - the DOCKER_HOST to use specifically during Podman tests // If FUNC_E2E_PODMAN is enabled but FUNC_E2E_PODMAN_HOST is not set, // try to auto-detect the Podman socket path - PodmanHost = getEnv("FUNC_E2E_PODMAN_HOST", "", "") + PodmanHost, _ = getEnv("FUNC_E2E_PODMAN_HOST", "", "") if Podman && PodmanHost == "" { PodmanHost = detectPodmanSocket() if PodmanHost != "" { @@ -405,10 +414,10 @@ func readEnvs() { // Registry - the registry URL including any account/repository at that // registry. Example: docker.io/alice. Default is the local registry. - Registry = getEnv("FUNC_E2E_REGISTRY", "E2E_REGISTRY_URL", DefaultRegistry) + Registry, _ = getEnv("FUNC_E2E_REGISTRY", "E2E_REGISTRY_URL", DefaultRegistry) // ClusterRegistry - the cluster-internal registry URL for in-cluster builds - ClusterRegistry = getEnv("FUNC_E2E_CLUSTER_REGISTRY", "", DefaultClusterRegistry) + ClusterRegistry, _ = getEnv("FUNC_E2E_CLUSTER_REGISTRY", "", DefaultClusterRegistry) // Verbose env as a truthy boolean Verbose = getEnvBool("FUNC_E2E_VERBOSE", "", DefaultVerbose) @@ -495,6 +504,15 @@ func setupEnv(t *testing.T) { // global config, or already defaulted by the user via environment variable. os.Setenv("FUNC_REGISTRY", Registry) + // The CLI reads FUNC_NAMESPACE through viper's "func" env prefix, so this + // is what makes FUNC_E2E_NAMESPACE reach the deploy. Set only when + // FUNC_E2E_NAMESPACE was set explicitly: without it the CLI falls back + // to the kubeconfig's current context, which is the long-standing + // behaviour and what CI relies on. + if namespaceExplicit { + os.Setenv("FUNC_NAMESPACE", Namespace) + } + // When using the default registry (registry.localtest.me), mark it as // insecure since it serves plain HTTP. if strings.Contains(Registry, "registry.localtest.me") { @@ -600,6 +618,16 @@ func newCmd(t *testing.T, args ...string) *exec.Cmd { return cmd } +// newCmdOutput is newCmd for tests that read the output: CombinedOutput +// refuses a command whose streams are already set, so leave them unset. +func newCmdOutput(t *testing.T, args ...string) *exec.Cmd { + t.Helper() + cmd := newCmd(t, args...) + cmd.Stdout = nil + cmd.Stderr = nil + return cmd +} + type waitOption func(*waitCfg) type waitCfg struct { @@ -1164,7 +1192,7 @@ func detectPodmanSocket() string { // getEnvPath converts the value returned from getEnv to an absolute path. // See getEnv docs for details. func getEnvPath(env, deprecated, dflt string) (val string) { - val = getEnv(env, deprecated, dflt) + val, _ = getEnv(env, deprecated, dflt) if !filepath.IsAbs(val) { // convert to abs var err error if val, err = filepath.Abs(val); err != nil { @@ -1176,13 +1204,15 @@ func getEnvPath(env, deprecated, dflt string) (val string) { // getEnvPath converts the value returned from getEnv into a string slice. func getEnvList(env, deprecated, dflt string) (vals []string) { - return fromCSV(getEnv(env, deprecated, dflt)) + val, _ := getEnv(env, deprecated, dflt) + return fromCSV(val) } // getEnvBool converts the value returned from getEnv into a boolean. func getEnvBool(env, deprecated string, dfltBool bool) bool { dflt := fmt.Sprintf("%t", dfltBool) - val, err := strconv.ParseBool(getEnv(env, deprecated, dflt)) + raw, _ := getEnv(env, deprecated, dflt) + val, err := strconv.ParseBool(raw) if err != nil { panic(fmt.Sprintf("value for %v %v expected to be boolean. %v", env, deprecated, err)) } @@ -1192,17 +1222,19 @@ func getEnvBool(env, deprecated string, dfltBool bool) bool { // getEnv gets the value of the given environment variable, or the default. // If the optional deprecated environment variable name is passed, it will be used // as a fallback with a warning about its deprecation status being printed. -// The final value will be converted to an absolute path. -func getEnv(env, deprecated, dflt string) (val string) { +// set reports whether either variable carried a value, letting a caller +// distinguish an explicitly set variable from the default filling in. +func getEnv(env, deprecated, dflt string) (val string, set bool) { // First check deprecated if provided if deprecated != "" { - if val = os.Getenv(deprecated); val != "" { + if v := os.Getenv(deprecated); v != "" { fmt.Fprintf(os.Stderr, "warning: the env var %v is deprecated and support will be removed in a future release. please use %v.", deprecated, env) + val, set = v, true } } // Current env takes precedence if v := os.Getenv(env); v != "" { - val = v + val, set = v, true } // Default if val == "" { diff --git a/pkg/deployer/common.go b/pkg/deployer/common.go index 34a3b995aa..e550775072 100644 --- a/pkg/deployer/common.go +++ b/pkg/deployer/common.go @@ -1,12 +1,20 @@ package deployer import ( + "maps" + fn "knative.dev/func/pkg/functions" ) const ( DeployerNameAnnotation = "function.knative.dev/deployer" + // DomainLabel records the custom domain a function was deployed with. + // On a Route it is the domain spec.host was built from, which ensure() + // compares to detect a domain change (host updates are permission-gated, + // so a change means recreating the Route). + DomainLabel = "func.domain" + // Dapr constants DaprEnabled = "true" DaprMetricsPort = "9092" @@ -32,7 +40,7 @@ func GenerateCommonLabels(f fn.Function, decorator DeployDecorator) (map[string] ll["function.knative.dev/runtime"] = f.Runtime if f.Domain != "" { - ll["func.domain"] = f.Domain + ll[DomainLabel] = f.Domain } if decorator != nil { @@ -42,6 +50,19 @@ func GenerateCommonLabels(f fn.Function, decorator DeployDecorator) (map[string] return ll, nil } +// SelectorLabels returns the subset of ll usable as a pod selector. +// +// Deployment.spec.selector is immutable, so a selector may only carry values +// fixed for the lifetime of the function. The domain is not one: it changes +// whenever the user redeploys with a different --domain, and including it +// makes the API server reject that update outright. It stays on the object's +// metadata and on the Route, which is where it is read from. +func SelectorLabels(ll map[string]string) map[string]string { + sl := maps.Clone(ll) + delete(sl, DomainLabel) + return sl +} + // GenerateCommonAnnotations creates annotations common to both Knative and K8s deployments func GenerateCommonAnnotations(f fn.Function, decorator DeployDecorator, daprInstalled bool, deployerName string) map[string]string { aa := make(map[string]string) @@ -53,15 +74,18 @@ func GenerateCommonAnnotations(f fn.Function, decorator DeployDecorator, daprIns } } - if len(deployerName) > 0 { - aa[DeployerNameAnnotation] = deployerName - } - // Add user-defined annotations for k, v := range f.Deploy.Annotations { aa[k] = v } + // The ownership stamp is asserted after user annotations, mirroring + // GenerateCommonLabels' order: it decides which component manages the + // object, so a stray copy of it in func.yaml must not change that. + if len(deployerName) > 0 { + aa[DeployerNameAnnotation] = deployerName + } + // Apply decorator if decorator != nil { aa = decorator.UpdateAnnotations(f, aa) diff --git a/pkg/deployer/common_test.go b/pkg/deployer/common_test.go new file mode 100644 index 0000000000..faf7400a6c --- /dev/null +++ b/pkg/deployer/common_test.go @@ -0,0 +1,30 @@ +package deployer + +import ( + "testing" + + fn "knative.dev/func/pkg/functions" +) + +// Test_GenerateCommonAnnotations_StampNotUserAssignable: the deployer stamp +// decides which component manages an object (Route ownership, describer and +// remover routing), so a same-key annotation in func.yaml must lose to it. +// Deployed objects carry the stamp visibly, and copying `kubectl get -o +// yaml` output into func.yaml's annotations is an ordinary accident. +func Test_GenerateCommonAnnotations_StampNotUserAssignable(t *testing.T) { + f := fn.Function{ + Deploy: fn.DeploySpec{Annotations: map[string]string{ + DeployerNameAnnotation: "banana", + "my-annotation": "kept", + }}, + } + + aa := GenerateCommonAnnotations(f, nil, false, "kubernetes") + + if got := aa[DeployerNameAnnotation]; got != "kubernetes" { + t.Errorf("stamp = %q, want %q: the ownership stamp must not be user-assignable", got, "kubernetes") + } + if got := aa["my-annotation"]; got != "kept" { + t.Errorf("user annotation = %q, want %q to survive", got, "kept") + } +} diff --git a/pkg/deployer/expose.go b/pkg/deployer/expose.go new file mode 100644 index 0000000000..7acc4a3601 --- /dev/null +++ b/pkg/deployer/expose.go @@ -0,0 +1,96 @@ +package deployer + +import ( + "context" + "errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + + fn "knative.dev/func/pkg/functions" +) + +// ErrExposureNotVisible means the Exposer could not tell whether an exposing +// object exists, not that none does. Denial is the usual cause: an account with +// no rights in the object's namespace gets the same answer either way. +// +// The two need opposite handling. A caller removing what the user asked to +// remove must not report success on this; a caller reconciling a function that +// never mentioned exposure must not fail on it. +var ErrExposureNotVisible = errors.New("cannot determine whether an exposing object exists") + +// ExposureRef identifies one function's exposing object without describing how +// to build it. Teardown needs only this, and finds the object by the labels +// these fields become: func looks for the object it labelled, not for whatever +// sits at a name it would have chosen. +type ExposureRef struct { + // Both are needed: where one namespace collects every function's objects, + // the name alone cannot separate two functions of the same name. + FunctionName string + FunctionNamespace string + + // Namespace holds the exposing object, not always the function's own: + // keda targets the interceptor Service in the operator's namespace, and an + // exposing object can only target a Service beside it. + Namespace string +} + +// Exposure describes the external address wanted for one function: which +// Service to send traffic to, and what to name the object that does it. +type Exposure struct { + Function fn.Function + + // Where the function is deployed, which is not always where its exposing + // object goes. See ExposureRef. + FunctionNamespace string + + // Name of the exposing object, not always the function's. Where one + // namespace collects every function's objects it must carry the function's + // namespace too, or same-named functions collide. Creation uses it; lookup + // goes by label. + Name string + + // Namespace holds the exposing object. See ExposureRef. + Namespace string + + // TargetService is the Service to send traffic to, TargetPort the port name + // on it. The raw deployer targets the function's own Service on "http"; + // keda targets the interceptor on "proxy". + TargetService string + TargetPort string + + // Owner is deleted together with the exposing object. Nil where the two + // cannot be linked, since Kubernetes rejects an owner reference across + // namespaces, which obliges the caller to Unexpose by hand. + Owner *metav1.OwnerReference + + Decorator DeployDecorator +} + +// Ref is what identifies this Exposure's object on the cluster, as opposed to +// what describes how to build it. +func (e Exposure) Ref() ExposureRef { + return ExposureRef{ + FunctionName: e.Function.Name, + FunctionNamespace: e.FunctionNamespace, + Namespace: e.Namespace, + } +} + +// Exposer gives a function an address reachable from outside the cluster, one +// implementation per mechanism. A deployer with no Exposer leaves its functions +// cluster-local, which is the default. +// +// The dynamic client is a parameter, not a field: constructing an Exposer must +// not need a kubeconfig, because every command builds the deployer, including +// those that never reach a cluster. +type Exposer interface { + // Expose creates or updates the exposing object and returns the hostname + // a router admitted it at, waiting for that admission. + Expose(ctx context.Context, client dynamic.Interface, e Exposure) (host string, err error) + + // Unexpose removes the object belonging to ref's function. It takes a + // ref, not a name, so it removes the object func labelled and leaves + // anything else alone. Finding nothing to remove is success. + Unexpose(ctx context.Context, client dynamic.Interface, ref ExposureRef) error +} diff --git a/pkg/deployer/testing/integration_test_helper.go b/pkg/deployer/testing/integration_test_helper.go index 9a4aaff3c3..429d68f1bf 100644 --- a/pkg/deployer/testing/integration_test_helper.go +++ b/pkg/deployer/testing/integration_test_helper.go @@ -65,7 +65,7 @@ func TestInt_Deploy(t *testing.T, deployer fn.Deployer, remover fn.Remover, desc // Explicit opt-out: keeps this integration deploy cluster-local and // platform-deterministic under exposed-by-default; ignored entirely // by the knative deployer. - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -169,7 +169,7 @@ func TestInt_Metadata(t *testing.T, deployer fn.Deployer, remover fn.Remover, de Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -325,7 +325,7 @@ func TestInt_Events(t *testing.T, deployer fn.Deployer, remover fn.Remover, desc Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -409,7 +409,7 @@ func TestInt_Scale(t *testing.T, deployer fn.Deployer, remover fn.Remover, descr Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -525,7 +525,7 @@ func TestInt_EnvsUpdate(t *testing.T, deployer fn.Deployer, remover fn.Remover, Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -887,6 +887,7 @@ func TestInt_FullPath(t *testing.T, deployer fn.Deployer, remover fn.Remover, li t.Error("environment variable was not set from config-map") } + // Removal by name, as the CLI does with --name: no local record. if err = remover.Remove(ctx, functionName, namespace); err != nil { t.Fatal(err) } @@ -936,7 +937,7 @@ func TestInt_ResourceValidationOnFirstDeploy(t *testing.T, deployer fn.Deployer, Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) @@ -1244,7 +1245,7 @@ func TestInt_OperatorSync(t *testing.T, deployer fn.Deployer, remover fn.Remover Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) diff --git a/pkg/describer/testing/integration_test_helper.go b/pkg/describer/testing/integration_test_helper.go index 5d57cbdc0b..7ba4d782f6 100644 --- a/pkg/describer/testing/integration_test_helper.go +++ b/pkg/describer/testing/integration_test_helper.go @@ -38,7 +38,7 @@ func TestInt_Describe(t *testing.T, describer fn.Describer, deployer fn.Deployer Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) diff --git a/pkg/functions/client.go b/pkg/functions/client.go index da54dc9a9f..1bed9b39e9 100644 --- a/pkg/functions/client.go +++ b/pkg/functions/client.go @@ -123,6 +123,7 @@ type DeploymentResult struct { URL string Namespace string Deployer string + Expose string } // Status of the function from the DeploymentResult @@ -145,9 +146,10 @@ type Runner interface { // Remover of deployed services. type Remover interface { - // Remove the function from remote. - // It should only return nil, when the Function was removed. - // In case the remover is not responsible for a Function, it should return a ErrNotHandled error. + // Remove the named function from the cluster. Returns nil only when the + // function was actually removed. A remover that does not recognize the + // function as its own returns ErrNotHandled so the next remover in the + // client's list can try. Remove(ctx context.Context, name string, namespace string) error } @@ -194,6 +196,7 @@ type Instance struct { Image string `json:"image" yaml:"image"` Namespace string `json:"namespace" yaml:"namespace"` Deployer string `json:"deployer" yaml:"deployer"` + Expose string `json:"expose,omitempty" yaml:"expose,omitempty"` Subscriptions []Subscription `json:"subscriptions" yaml:"subscriptions"` Labels map[string]string `json:"labels" yaml:"labels" xml:"-"` Middleware Middleware `json:"middleware,omitempty" yaml:"middleware,omitempty"` @@ -863,6 +866,13 @@ func (c *Client) Deploy(ctx context.Context, f Function, oo ...DeployOption) (Fu return f, ErrNameRequired } + // Checked here rather than per-deployer so every deployer rejects the same + // set: a mode this build does not recognize can neither be applied nor + // torn down. + if err := ValidateExpose(f.Expose); err != nil { + return f, err + } + // Deployer switch gate - changing deployers when function is currently // deployed would leave stranded resources on cluster. We error clearly // and expect the user to undeploy first, which removes the resources @@ -914,6 +924,15 @@ func (c *Client) Deploy(ctx context.Context, f Function, oo ...DeployOption) (Fu // Update the function to reflect the new deployed state of the Function f.Deploy.Namespace = result.Namespace f.Deploy.Deployer = result.Deployer + f.Deploy.Expose = result.Expose + + // A deployer with no support for the requested mechanism reports applying + // none. The function is running but unreachable from outside the cluster, + // which is otherwise indistinguishable from a successful exposure. + if ActiveExpose(f.Expose) && result.Expose == "" { + fmt.Fprintf(os.Stderr, "Warning: expose %q was requested but the %q deployer applied no external exposure\n", + f.Expose, result.Deployer) + } switch result.Status { case Deployed: @@ -1243,10 +1262,11 @@ func (c *Client) Remove(ctx context.Context, name, namespace string, f Function, if combinedErr == nil { // Function was undeployed successfully. The user's INTENT (the top-level - // Function.Deployer and Function.Namespace) is untouched and is what a - // subsequent deploy reuses. + // Function.Deployer, Function.Expose, Function.Namespace) is untouched + // and is what a subsequent deploy reuses. f.Deploy.Namespace = "" f.Deploy.Deployer = "" + f.Deploy.Expose = "" } return f, combinedErr } diff --git a/pkg/functions/client_test.go b/pkg/functions/client_test.go index f509c2ecdc..028a27c7c3 100644 --- a/pkg/functions/client_test.go +++ b/pkg/functions/client_test.go @@ -1576,6 +1576,32 @@ func TestClient_Pipelines_Deploy_Namespace(t *testing.T) { } } +// TestClient_Deploy_InvalidExposeErrors ensures Deploy rejects an exposure +// mode this build does not recognize, before any deployer runs. The check +// lives here rather than per-deployer, so this is the only place that proves +// every deployer rejects the same set. +func TestClient_Deploy_InvalidExposeErrors(t *testing.T) { + root, rm := Mktemp(t) + defer rm() + + client := fn.New(fn.WithRegistry(TestRegistry), fn.WithDeployer(mock.NewDeployer())) + + f, err := client.Init(fn.Function{Runtime: TestRuntime, Root: root}) + if err != nil { + t.Fatal(err) + } + f.Expose = "fake-exposer" // not a mode this build knows + + // The built check fires first and is not what this pins. + _, err = client.Deploy(t.Context(), f, fn.WithDeploySkipBuildCheck(true)) + if err == nil { + t.Fatal("expected an unrecognized exposure mode to be refused") + } + if !errors.Is(err, fn.ErrInvalidExpose) { + t.Fatalf("expected ErrInvalidExpose, got %v", err) + } +} + // TestClient_Deploy_UnbuiltErrors ensures that a call to deploy a function // which was not fully created (ie. was only initialized, not actually built // or deployed) yields the expected error. diff --git a/pkg/functions/errors.go b/pkg/functions/errors.go index b52cc3d137..2431eec39f 100644 --- a/pkg/functions/errors.go +++ b/pkg/functions/errors.go @@ -10,7 +10,7 @@ import ( var ( ErrEnvironmentNotFound = errors.New("environment not found") ErrFunctionNotFound = errors.New("function not found") - ErrInvalidExpose = errors.New("invalid deploy.expose value") + ErrInvalidExpose = errors.New("invalid expose value") ErrMismatchedName = errors.New("name passed the function source") ErrNameRequired = errors.New("name required") ErrNamespaceRequired = errors.New("namespace required") diff --git a/pkg/functions/function.go b/pkg/functions/function.go index f9273760b2..029d333799 100644 --- a/pkg/functions/function.go +++ b/pkg/functions/function.go @@ -106,6 +106,14 @@ type Function struct { // in .Deploy.Deployer, which is cleared on undeploy. Deployer string `yaml:"deployer,omitempty" jsonschema:"enum=knative,enum=raw,enum=keda"` + // Expose is the requested (intended) external exposure mode for the raw + // and keda deployers (knative manages its own networking and ignores it). + // Values: "route" (OpenShift Route; OpenShift only), "none" (cluster-local). + // Empty means cluster-local. Persists across undeploy like Deployer. + // The mode CURRENTLY applied on the cluster is recorded separately in + // .Deploy.Expose, which is cleared on undeploy. + Expose string `yaml:"expose,omitempty" jsonschema:"enum=route,enum=none,enum="` + // Created time is the moment that creation was successfully completed // according to the client which is in charge of what constitutes being // fully "Created" (aka initialized) @@ -278,16 +286,11 @@ type DeploySpec struct { // the function is managed by default when the func-operator is installed. ManagementDisabled bool `yaml:"managementDisabled,omitempty"` - // Expose controls external access for the raw and keda deployers (the - // knative deployer manages its own exposure and ignores it). Optional. - // Values: "route" (create an OpenShift Route; OpenShift clusters only - - // a hard error elsewhere), "none" (cluster-local only, explicit - // opt-out). Defaults to "route" behavior on OpenShift - a deployed - // function being externally reachable is the expected outcome - and to - // cluster-local on any other cluster, since a Route is an - // OpenShift-only mechanism and the unset default must not impose a - // platform requirement. - Expose string `yaml:"expose,omitempty"` + // Expose records the external exposure mode CURRENTLY applied on the + // cluster for raw/keda (observed state). Written after successful deploy, + // cleared on undeploy alongside Namespace and Deployer. Empty means + // cluster-local (or never exposed). User intent lives on Function.Expose. + Expose string `yaml:"expose,omitempty" jsonschema:"enum=route,enum=none,enum="` } // HealthEndpoints specify the liveness and readiness endpoints for a Runtime @@ -417,6 +420,7 @@ func (f Function) Validate() error { ValidateLabels(f.Deploy.Labels), validateGit(f.Build.Git), validateKafka(f.Run.Kafka, f.Invoke, f.Runtime), + validateExpose(f.Deploy.Expose, f.Expose), } var b strings.Builder diff --git a/pkg/functions/function_expose.go b/pkg/functions/function_expose.go index 104bddcb74..50c006f6d4 100644 --- a/pkg/functions/function_expose.go +++ b/pkg/functions/function_expose.go @@ -1,21 +1,50 @@ +/* +This file is for exposure stuff -> about functions being externally exposed ( +outside of cluster) and what constants does functions project define with +validation and helper functions +*/ package functions import ( "fmt" + "slices" ) -// ValidateExpose reports whether expose is a valid deploy.expose value: "" -// (default - exposed via an OpenShift Route, since a deployed function -// being reachable is the expected outcome; cluster-local on non-OpenShift -// clusters, since a Route is an OpenShift-only mechanism), "none" -// (cluster-local, explicit opt-out), or "route" (explicit request for an -// OpenShift Route). There is no ref suffix: an OpenShift Route has no -// concept of "which ingress controller to attach to" - the cluster's -// IngressController picks the router, and the Route object doesn't -// reference one. Any other value is rejected. +const ( + // ExposeNone intends to deploy cluster-local (no exposure) + ExposeNone = "none" + // ExposeRoute is an OpenShift Route, an OpenShift-only resource + ExposeRoute = "route" +) + +// ExposeModes are the mechanisms accepted in addition to "". Adding one here +// extends validation and shell completion together +var ExposeModes = []string{ExposeNone, ExposeRoute} + +// ValidateExpose reports whether expose is a valid exposure mode: "" or +// ExposeNone for cluster-local, or one of ExposeModes. Applies to both intent +// (Function.Expose) and observed status (DeploySpec.Expose). Rejects anything +// else. func ValidateExpose(expose string) error { - if expose == "" || expose == "none" || expose == "route" { + if expose == "" || slices.Contains(ExposeModes, expose) { return nil } return fmt.Errorf("%w: %q", ErrInvalidExpose, expose) } + +// ActiveExpose reports whether mode names an external mechanism, as opposed to +// cluster-local ("" or ExposeNone). +func ActiveExpose(mode string) bool { + return mode != "" && mode != ExposeNone +} + +// wrapper for validating exposure for f.Validate() +func validateExpose(vals ...string) (errs []string) { + for _, v := range vals { + err := ValidateExpose(v) + if err != nil { + errs = append(errs, err.Error()) + } + } + return +} diff --git a/pkg/functions/function_expose_unit_test.go b/pkg/functions/function_expose_unit_test.go index 0af532c640..031674eb95 100644 --- a/pkg/functions/function_expose_unit_test.go +++ b/pkg/functions/function_expose_unit_test.go @@ -28,3 +28,12 @@ func Test_ValidateExpose(t *testing.T) { }) } } + +func Test_ActiveExpose(t *testing.T) { + if ActiveExpose("") || ActiveExpose("none") { + t.Error("empty and none must not be active") + } + if !ActiveExpose("route") { + t.Error("route must be active") + } +} diff --git a/pkg/k8s/deployer.go b/pkg/k8s/deployer.go index 231cd44a68..bf464ea9ad 100644 --- a/pkg/k8s/deployer.go +++ b/pkg/k8s/deployer.go @@ -45,6 +45,14 @@ const ( // without re-deriving or re-querying the Route. RouteHostnameAnnotation = "function.knative.dev/route-hostname" + // RouteNamespaceAnnotation records where the exposing Route was created, + // written by the code that created it; removal reads it. It exists because + // keda's Route lives with the interceptor, whose namespace depends on how + // keda was installed. + // + // Written and cleared together with RouteHostnameAnnotation. + RouteNamespaceAnnotation = "function.knative.dev/route-namespace" + // managedByAnnotation identifies triggers managed by this deployer managedByAnnotation = "func.knative.dev/managed-by" managedByValue = "func-raw-deployer" @@ -56,10 +64,7 @@ type Deployer struct { verbose bool decorator deployer.DeployDecorator - // exposureDisabled marks a Deployer embedded by another deployer (keda) - // whose functions must stay cluster-local: a Route pointed at the - // raw ClusterIP Service would bypass keda's scale-to-zero interceptor. - exposureDisabled bool + exposer deployer.Exposer } func NewDeployer(opts ...DeployerOpt) *Deployer { @@ -70,19 +75,15 @@ func NewDeployer(opts ...DeployerOpt) *Deployer { return d } -func WithDeployerVerbose(verbose bool) DeployerOpt { +func WithExposer(exposer deployer.Exposer) DeployerOpt { return func(d *Deployer) { - d.verbose = verbose + d.exposer = exposer } } -// WithDeployerExposureDisabled turns off this Deployer's own OpenShift -// Route exposure; for deployers that embed this Deployer but manage -// exposure themselves (eg. keda, whose functions stay behind its own -// interceptor and mint their own Route separately). -func WithDeployerExposureDisabled() DeployerOpt { +func WithDeployerVerbose(verbose bool) DeployerOpt { return func(d *Deployer) { - d.exposureDisabled = true + d.verbose = verbose } } @@ -126,12 +127,9 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu // effective namespace is not logic for the deployer implementation, which // should have a minimum of logic. In this case limited to "new ns or // existing namespace? - namespace := f.Namespace - if namespace == "" { - namespace = f.Deploy.Namespace - } - if namespace == "" { - return fn.DeploymentResult{}, fmt.Errorf("deployer requires either a target namespace or that the function be already deployed") + namespace, err := DeployNamespace(f) + if err != nil { + return fn.DeploymentResult{}, err } // Choosing an image to deploy: @@ -173,6 +171,7 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu existingDeployment, err := deploymentClient.Get(ctx, f.Name, metav1.GetOptions{}) var status fn.Status + var svc *corev1.Service if err == nil { // Update the existing function referencedSecrets := sets.New[string]() @@ -196,7 +195,7 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu existingService = nil } - svc, err := d.generateService(f, namespace, daprInstalled, existingDeployment, existingService) + svc, err = d.generateService(f, namespace, daprInstalled, existingDeployment, existingService) if err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to generate service resources: %w", err) } @@ -204,19 +203,24 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu // Preserve resource version for update deployment.ResourceVersion = existingDeployment.ResourceVersion + if err := preserveDeploymentSelector(existingDeployment, deployment, f.Name); err != nil { + return fn.DeploymentResult{}, err + } + if _, err = deploymentClient.Update(ctx, deployment, metav1.UpdateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to update deployment: %w", err) } - // update/create service + // update/create service; keep the returned object, its UID owns the + // exposure and trigger satellites below if svcGetErr == nil { svc.ResourceVersion = existingService.ResourceVersion - if _, err = serviceClient.Update(ctx, svc, metav1.UpdateOptions{}); err != nil { + if svc, err = serviceClient.Update(ctx, svc, metav1.UpdateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to update service: %w", err) } } else { // Confirmed IsNotFound above the generateService() - if _, err = serviceClient.Create(ctx, svc, metav1.CreateOptions{}); err != nil { + if svc, err = serviceClient.Create(ctx, svc, metav1.CreateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to create service: %w", err) } } @@ -248,12 +252,12 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("failed to create deployment: %w", err) } - svc, err := d.generateService(f, namespace, daprInstalled, deployment, nil) + svc, err = d.generateService(f, namespace, daprInstalled, deployment, nil) if err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to generate service resources: %w", err) } - if _, err = serviceClient.Create(ctx, svc, metav1.CreateOptions{}); err != nil { + if svc, err = serviceClient.Create(ctx, svc, metav1.CreateOptions{}); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to create service: %w", err) } @@ -267,8 +271,9 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("deployment did not become ready: %w", err) } - // External exposure via an OpenShift Route; see resolveExposure(). - url, _, err := d.resolveExposure(ctx, f, namespace, clientset, dynClient) + // Reconcile external exposure after Service/Deployment exists on cluster + // (backend + owner reference for the exposing object). + url, appliedExpose, err := d.resolveExposure(ctx, f, namespace, svc, clientset, dynClient) if err != nil { return fn.DeploymentResult{}, err } @@ -287,150 +292,176 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu URL: url, Namespace: namespace, Deployer: KubernetesDeployerName, + Expose: appliedExpose, }, nil } -// resolveExposure keeps a raw-deployer function's external exposure (an -// OpenShift Route) in sync with what's currently wanted. This is symmetric -// for both directions: create/update the Route when exposure is wanted, -// remove it when it isn't - so toggling expose:route on and off across -// redeploys just works. -// Removal is unconditional whenever exposure isn't currently wanted, so a -// stale Route from a prior raw deploy never survives a raw -> keda deployer -// switch (the only cross-deployer path that still runs this code, since -// keda embeds this deployer with exposure disabled). -// Functions are exposed BY DEFAULT: a deployed function being reachable is -// the expected outcome, matching what a plain "func deploy" already implies -// for every other deployer, so the unset value behaves the same as -// explicit expose:route, not like expose:none. This is only meaningful on -// OpenShift, since a Route is an OpenShift-only mechanism: IsOpenShift() -// keeps plain-Kubernetes deploys safe without requiring any flag - an -// explicit expose:route request off OpenShift is still a hard error (the -// user asked for something impossible), but the unset default just quietly -// degrades to cluster-local there rather than failing an ordinary deploy. -func (d *Deployer) resolveExposure(ctx context.Context, f fn.Function, namespace string, clientset kubernetes.Interface, dynClient dynamic.Interface) (string, bool, error) { - defaultURL := fmt.Sprintf("http://%s.%s.svc", f.Name, namespace) - - if err := fn.ValidateExpose(f.Deploy.Expose); err != nil { - return "", false, err - } - tech := f.Deploy.Expose - - if tech == "route" && !IsOpenShift() { - return "", false, fmt.Errorf( - "expose:route requires an OpenShift cluster: route.openshift.io Routes are an " + - "OpenShift-specific resource, and this does not appear to be an OpenShift cluster") +// preserveDeploymentSelector keeps the live Deployment's selector on an +// update: the selector is immutable, and older funcs pinned their whole +// label map there. When the new pod template no longer satisfies a pinned +// label it refuses; only recreation can change an immutable field. +func preserveDeploymentSelector(existing, desired *appsv1.Deployment, fnName string) error { + if existing == nil || existing.Spec.Selector == nil { + return nil } - - if d.exposureDisabled { - if err := d.removeExposure(ctx, clientset, dynClient, namespace, f.Name, false); err != nil { - return "", false, err + desired.Spec.Selector = existing.Spec.Selector.DeepCopy() + for k, v := range existing.Spec.Selector.MatchLabels { + if desired.Spec.Template.Labels[k] != v { + return fmt.Errorf( + "function %q cannot be updated in place: its Deployment was created by an older func whose selector pins %s=%q, and this deploy no longer carries that label. A pinned label can only change by recreation: run 'func delete' and deploy again", + fnName, k, v) } - return defaultURL, false, nil } + return nil +} - wantsRoute := tech == "route" || (tech == "" && IsOpenShift()) - if !wantsRoute { - // expose:none (explicit opt-out): enforce=true, a hard error if - // removal fails to verify/clear. Unset on a non-OpenShift cluster - // (default gracefully declined, not requested): enforce=false, - // since nothing was actually asked for here. - if err := d.removeExposure(ctx, clientset, dynClient, namespace, f.Name, tech == "none"); err != nil { - return "", false, err - } - return defaultURL, false, nil +// DeployNamespace is where a function will be deployed: the requested +// namespace, or the one it is already deployed in. The wider arbitration +// between kube context, flags, environment and global defaults is settled +// earlier, before a Function reaches a deployer. +// +// Exported so pkg/keda can apply this exact rule ahead of its embedded raw +// deploy; a copy could drift. +func DeployNamespace(f fn.Function) (string, error) { + if f.Namespace != "" { + return f.Namespace, nil } - - url, err := d.ensureExposure(ctx, f, namespace, clientset, dynClient) - if err != nil { - return "", false, fmt.Errorf("external exposure failed: %w", err) + if f.Deploy.Namespace != "" { + return f.Deploy.Namespace, nil } - return url, true, nil + return "", fmt.Errorf("deployer requires either a target namespace or that the function be already deployed") } -// removeExposure deletes the managed Route (never a user-authored route -// sharing the function's name) and clears the recorded exposure state. -// Missing Route API support needs no special-casing: the GET reports -// NotFound either way, meaning nothing to remove. +// resolveExposure reconciles external exposure: it creates or updates the +// Route when f.Expose asks for one, removes it when only the Service's +// record says one exists, and records the Route's namespace and hostname as +// annotations on the function's Service. // -// enforce selects the failure posture: -// - true (unset or expose:none): failing to verify/remove is a hard error; -// - false (deployer switched away from raw): an RBAC 403 on the route -// GET/DELETE prints a warning and the deploy continues, since keda -// users without Route permissions must stay green. -func (d *Deployer) removeExposure(ctx context.Context, clientset kubernetes.Interface, dynClient dynamic.Interface, namespace, name string, enforce bool) error { - if _, err := RemoveManagedRoute(ctx, dynClient, namespace, name); err != nil { - if !enforce && errors.IsForbidden(err) { - fmt.Fprintf(os.Stderr, "⚠️ cannot remove Route %q (forbidden) - leaving it in place\n", name) - } else { - return fmt.Errorf("failed to remove Route: %w", err) - } +// A nil exposer means cluster-local: create nothing, remove nothing, leave +// the record alone, return the Service's own URL. This is how keda uses the +// embedded raw deployer - Deployment and Service only; its exposure is its +// own. +func (d *Deployer) resolveExposure(ctx context.Context, f fn.Function, + namespace string, svc *corev1.Service, clientset kubernetes.Interface, + dynClient dynamic.Interface) (url string, appliedExpose string, err error) { + + defaultURL := fmt.Sprintf("http://%s.%s.svc", f.Name, namespace) + + // do nothing with nil exposer - cluster-local exposure + if d.exposer == nil { + return defaultURL, "", nil } - if err := writeRouteHostnameAnnotation(ctx, clientset, namespace, name, ""); err != nil { - if !enforce { - fmt.Fprintf(os.Stderr, "⚠️ failed to clear exposure state: %v\n", err) - return nil + // Route ns from function svc annotation + recordedNS := svc.Annotations[RouteNamespaceAnnotation] + + switch { + case fn.ActiveExpose(f.Expose): // want a Route: create or update it + url, err = d.ensureExposure(ctx, f, namespace, svc, clientset, dynClient) + if err != nil { + return "", "", fmt.Errorf("external exposure failed: %w", err) } - return fmt.Errorf("failed to clear exposure state: %w", err) - } - return nil -} + return url, f.Expose, nil + + // since we have the svc fetched, use its annotation - cluster-side info + case recordedNS != "": // f.Deploy.Expose != "" + ref := deployer.ExposureRef{ + FunctionName: f.Name, + FunctionNamespace: namespace, + Namespace: recordedNS, + } + if err := d.exposer.Unexpose(ctx, dynClient, ref); err != nil { + return "", "", fmt.Errorf("failed to remove external exposure: %w", err) + } + if err := SetRouteHostname(ctx, clientset, namespace, f.Name, "", ""); err != nil { + return "", "", err + } + return defaultURL, "", nil -// ensureExposure creates or updates the Route exposing f, waits for it to -// be admitted by a router, and records the minted hostname. -func (d *Deployer) ensureExposure(ctx context.Context, f fn.Function, namespace string, clientset kubernetes.Interface, dynClient dynamic.Interface) (string, error) { - deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, f.Name, metav1.GetOptions{}) - if err != nil { - return "", fmt.Errorf("failed to get deployment for owner reference: %w", err) + default: // nothing wanted, nothing recorded: Route API untouched + return defaultURL, "", nil } +} - route, err := GenerateRoute(f, f.Name, deployment, d.decorator, KubernetesDeployerName) - if err != nil { - return "", fmt.Errorf("failed to generate Route: %w", err) +// ensureExposure builds an Exposure for the function's own Service, calls +// the configured Exposer, and records the admitted hostname on svc. +func (d *Deployer) ensureExposure(ctx context.Context, f fn.Function, namespace string, svc *corev1.Service, clientset kubernetes.Interface, dynClient dynamic.Interface) (string, error) { + controller := true + e := deployer.Exposure{ + Function: f, + // The raw deployer's Route sits beside its function, so these two + // are the same namespace. They diverge for keda. + FunctionNamespace: namespace, + Name: f.Name, + Namespace: namespace, + TargetService: f.Name, + TargetPort: "http", + Owner: &metav1.OwnerReference{ + APIVersion: "v1", + Kind: "Service", + Name: svc.Name, + UID: svc.UID, + Controller: &controller, + }, + Decorator: d.decorator, } - fmt.Fprintf(os.Stderr, "🌐 Exposing function externally -> %s\n", f.Name) - - if err := EnsureRoute(ctx, dynClient, namespace, route); err != nil { - return "", err + if d.verbose { + fmt.Fprintf(os.Stderr, "🌐 Exposing function externally -> %s\n", f.Name) } - // Wait for a router to accept the route - enforced, never downgraded to a warning. - host, err := WaitForRouteAdmitted(ctx, dynClient, namespace, f.Name, 30*time.Second) + host, err := d.exposer.Expose(ctx, dynClient, e) if err != nil { - return "", fmt.Errorf("route was not admitted: %w", err) + return "", err } - if err := writeRouteHostnameAnnotation(ctx, clientset, namespace, f.Name, host); err != nil { - return "", err + // The raw deployer's Route sits in the function's own namespace, so the + // recorded location is that namespace. Unchanged behaviour: its Route is + // garbage collected through an owner reference. + if err := SetRouteHostname(ctx, clientset, namespace, f.Name, host, namespace); err != nil { + // The Route exists but nothing records it, and the no-record paths + // deliberately never look for one. Take it back down rather than + // leave a live, unrecorded exposure. A kill between the two calls + // still orphans; delete's garbage collection covers that. + if rbErr := d.exposer.Unexpose(ctx, dynClient, e.Ref()); rbErr != nil { + return "", fmt.Errorf("recording the exposure failed: %w; rolling the Route back failed too: %v", err, rbErr) + } + return "", fmt.Errorf("recording the exposure failed, the Route was rolled back: %w", err) } - // The Route redirects http to https (see GenerateRoute's tls stanza). + // ocproute uses edge TLS with redirect; other exposers may differ later. return fmt.Sprintf("https://%s", host), nil } -// writeRouteHostnameAnnotation records (hostname != "") or clears -// (hostname == "") the exposed hostname on the function's Service: no-op -// when already current, retried on write conflicts. A missing Service is -// tolerated only when clearing; recording against one that doesn't exist -// is a real error. -func writeRouteHostnameAnnotation(ctx context.Context, clientset kubernetes.Interface, namespace, name, hostname string) error { +// SetRouteHostname updates the Route record on the function's Service. +// A non-empty hostname writes the hostname and routeNamespace annotations; an +// empty hostname removes them. Does nothing when the annotations are already +// current, retries on write conflicts. A missing Service is fine when +// clearing and an error when recording. +// +// Exported so keda can publish its Route's hostname here too, where Describe +// and List read it (pkg/keda/deployer.go). +func SetRouteHostname(ctx context.Context, clientset kubernetes.Interface, namespace, name, hostname, routeNamespace string) error { err := retry.RetryOnConflict(retry.DefaultRetry, func() error { svc, err := clientset.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } - if svc.Annotations[RouteHostnameAnnotation] == hostname { + if svc.Annotations[RouteHostnameAnnotation] == hostname && + svc.Annotations[RouteNamespaceAnnotation] == routeNamespace { return nil } + // Both annotations together, always: a half record would leave + // teardown or describe reading incomplete information. if hostname == "" { delete(svc.Annotations, RouteHostnameAnnotation) + delete(svc.Annotations, RouteNamespaceAnnotation) } else { if svc.Annotations == nil { svc.Annotations = map[string]string{} } svc.Annotations[RouteHostnameAnnotation] = hostname + svc.Annotations[RouteNamespaceAnnotation] = routeNamespace } _, err = clientset.CoreV1().Services(namespace).Update(ctx, svc, metav1.UpdateOptions{}) return err @@ -512,6 +543,7 @@ func syncTriggers(ctx context.Context, f fn.Function, namespace string, eventing return fmt.Errorf("failed to get service: %w", err) } + // gauron99: take the name and UID from the service - no need to fetch depl here? deployment, err := clientset.AppsV1().Deployments(namespace).Get(ctx, f.Name, metav1.GetOptions{}) if err != nil { return fmt.Errorf("failed to get deployment: %w", err) @@ -660,7 +692,7 @@ func (d *Deployer) generateDeployment(f fn.Function, namespace string, daprInsta Spec: appsv1.DeploymentSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: labels, + MatchLabels: deployer.SelectorLabels(labels), }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -689,15 +721,28 @@ func (d *Deployer) generateService(f fn.Function, namespace string, daprInstalle } annotations := deployer.GenerateCommonAnnotations(f, d.decorator, daprInstalled, KubernetesDeployerName) - // re-apply the hostname annotation, contrary to the rest of annotations - // which are "always regenerate" -- the hostname is cluster-derived, not - // in func.yaml: the router mints it, and only the exposure step (after - // the Route is admitted) can write it, which happens after this - // Service write. + // Unlike the annotations above, which always regenerate, the exposure + // record is re-applied: it is cluster-derived, written only once the Route + // is admitted, and this Update replaces the whole annotation map. Carry + // both halves over or the write drops the record. if existingService != nil && existingService.Annotations[RouteHostnameAnnotation] != "" { annotations[RouteHostnameAnnotation] = existingService.Annotations[RouteHostnameAnnotation] + // No key means no record; never write it empty. + if recordedNS := existingService.Annotations[RouteNamespaceAnnotation]; recordedNS != "" { + annotations[RouteNamespaceAnnotation] = recordedNS + } } + // Built by hand rather than with metav1.NewControllerRef, which also sets + // BlockOwnerDeletion. That flag takes effect only during foreground + // cascading deletion, which nothing here requests; deletion defaults to + // background, where it does nothing. Setting it does, however, make the + // OwnerReferencesPermissionEnforcement admission plugin require update on + // the owner's finalizers subresource - a grant neither a plain func user + // nor the Tekton pipeline ServiceAccount holds by default, so the Service + // create is rejected outright. OpenShift enables that plugin; KinD does + // not, so the failure never appears in upstream CI. + controller := true service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: f.Name, @@ -705,12 +750,22 @@ func (d *Deployer) generateService(f fn.Function, namespace string, daprInstalle Labels: labels, Annotations: annotations, OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(deployment, appsv1.SchemeGroupVersion.WithKind("Deployment")), + { + APIVersion: appsv1.SchemeGroupVersion.String(), + Kind: "Deployment", + Name: deployment.Name, + UID: deployment.UID, + Controller: &controller, + }, }, }, Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: labels, + Type: corev1.ServiceTypeClusterIP, + // Same selector as the Deployment: a Service selecting on the + // domain would stop matching the running pods the moment the + // domain changed, blacking the function out until the rollout + // caught up. + Selector: deployer.SelectorLabels(labels), Ports: []corev1.ServicePort{ { Name: "http", diff --git a/pkg/k8s/deployer_test.go b/pkg/k8s/deployer_test.go index 16b6096559..7e6ef61804 100644 --- a/pkg/k8s/deployer_test.go +++ b/pkg/k8s/deployer_test.go @@ -1,15 +1,24 @@ package k8s import ( + "context" + "errors" + "fmt" + "maps" "os" "strings" "testing" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/dynamic" dynamicfakeclient "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "knative.dev/func/pkg/deployer" fn "knative.dev/func/pkg/functions" ) @@ -567,83 +576,602 @@ func Test_ProcessVolumes_ValidPath(t *testing.T) { } } -// Test_WithDeployerExposureDisabled: exposure is on by default (raw) and off -// with others -func Test_WithDeployerExposureDisabled(t *testing.T) { - if NewDeployer().exposureDisabled { - t.Error("expected exposure enabled on a default Deployer") +// Test_ResolveExposure_NoExposer: a Deployer with no Exposer performs no +// exposure at all. Every valid intent, active or not, leaves the function +// cluster-local and reports applying nothing; only an unrecognized value is +// an error, since that cannot be applied or torn down by anyone. This is the +// state keda's embedded raw Deployer runs in on every deploy, so it must stay +// quiet - keda exposes its own Route afterwards (pkg/keda/exposure.go). +func Test_ResolveExposure_NoExposer(t *testing.T) { + d := NewDeployer() + f := fn.Function{Name: "f", Deploy: fn.DeploySpec{Namespace: "ns"}} + ctx := t.Context() + clientset := fake.NewClientset() + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + tests := []struct { + name string + expose string + }{ + {name: "empty: cluster-local", expose: ""}, + {name: "none: cluster-local", expose: "none"}, + {name: "route without exposer: cluster-local, no error", expose: "route"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f.Expose = tt.expose + url, applied, err := d.resolveExposure(ctx, f, "ns", testService(nil), clientset, dynClient) + if err != nil { + t.Fatalf("resolveExposure(%q): unexpected error: %v", tt.expose, err) + } + if applied != "" { + t.Errorf("resolveExposure(%q): applied = %q, want empty", tt.expose, applied) + } + if url == "" { + t.Errorf("resolveExposure(%q): expected a non-empty cluster-local URL", tt.expose) + } + }) + } +} + +// stubExposer stands in for a real exposure mechanism, recording what it was +// asked to do so a test can assert on it without a cluster. +type stubExposer struct { + host string + unexposeErr error + exposed []deployer.Exposure + unexposed []string +} + +func (s *stubExposer) Expose(_ context.Context, _ dynamic.Interface, e deployer.Exposure) (string, error) { + s.exposed = append(s.exposed, e) + return s.host, nil +} + +func (s *stubExposer) Unexpose(_ context.Context, _ dynamic.Interface, ref deployer.ExposureRef) error { + s.unexposed = append(s.unexposed, ref.Namespace+"/"+ref.FunctionNamespace+"/"+ref.FunctionName) + return s.unexposeErr +} + +func testService(annotations map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "f", Namespace: "ns", Annotations: annotations, + }, } - if !NewDeployer(WithDeployerExposureDisabled()).exposureDisabled { - t.Error("expected exposure disabled with WithDeployerExposureDisabled") +} + +// Test_ResolveExposure_NoExposerLeavesHostnameAlone: a Deployer with no +// Exposer must leave the exposure record untouched, cluster-local intent +// included. Keda's embedded raw Deployer runs in exactly this state on every +// keda deploy, while keda's own exposing object and the record keda wrote +// for it are in place; clearing it here would blank the hostname Describe +// and List read back and lose the Route's location teardown reads. +func Test_ResolveExposure_NoExposerLeavesHostnameAlone(t *testing.T) { + const ( + host = "f-ns.apps.example.com" + routeNS = "openshift-keda" + ) + ctx := t.Context() + svc := testService(map[string]string{ + RouteHostnameAnnotation: host, + RouteNamespaceAnnotation: routeNS, + }) + clientset := fake.NewClientset(svc) + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + d := NewDeployer() + f := fn.Function{Name: "f", Expose: fn.ExposeNone, Deploy: fn.DeploySpec{Namespace: "ns"}} + + if _, applied, err := d.resolveExposure(ctx, f, "ns", svc, clientset, dynClient); err != nil { + t.Fatal(err) + } else if applied != "" { + t.Errorf("applied = %q, want empty", applied) + } + + svc, err := clientset.CoreV1().Services("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if got := svc.Annotations[RouteHostnameAnnotation]; got != host { + t.Errorf("expected hostname %q to survive, got %q", host, got) + } + if got := svc.Annotations[RouteNamespaceAnnotation]; got != routeNS { + t.Errorf("expected route namespace %q to survive, got %q", routeNS, got) } } -// Test_ResolveExposure_RouteGatedOnOpenShift: functions are exposed by -// default, so an explicit expose:route request hard-errors off OpenShift -// (the user asked for something impossible), and expose:none (explicit -// opt-out) never requires OpenShift or touches the Route API at all, on -// either platform - removeExposure's Get against an empty fake dynamic -// client returns NotFound immediately. The unset/empty value off OpenShift -// also stays cluster-local, but silently (no error): the default degrading -// gracefully rather than failing an ordinary deploy is exactly the point. +// Test_generateService_CarriesExposureRecordAcrossRedeploy pins that a +// redeploy does not erase the exposure record. // -// The "route on OpenShift" and "empty on OpenShift" cases are NOT exercised -// here: both fall through to ensureExposure, which waits up to 30s -// (hardcoded) for a router to admit the Route - a real wait against a fake -// client with no controller to populate status would either hang the test -// for 30s or require simulating async status writes, disproportionate for -// this table. That deeper path (EnsureRoute, WaitForRouteAdmitted, -// GenerateRoute) is covered directly and fast in route_test.go instead, -// each with its own short timeout. +// Every other annotation here is regenerated from func.yaml, and the update +// replaces the Service's whole annotation map, so anything cluster-derived +// survives only by being copied off the live Service. The record is +// cluster-derived: the hostname is the one the router admitted, whether +// minted by it or given with --domain, and the exposing step chooses the +// namespace for Route. Both are known only after this write. // -// Note: SetOpenShiftForTest mutates a package-level bool without a mutex - -// this test must not run with t.Parallel() (see openshift.go). -func Test_ResolveExposure_RouteGatedOnOpenShift(t *testing.T) { +// The hostname is what Describe and List report; the location is what keda's +// delete and its unexpose toggle use to find the Route. Carrying only the +// hostname would leave a function advertising a URL whose Route has no +// recorded home, putting the delete back to guessing at a namespace. +func Test_generateService_CarriesExposureRecordAcrossRedeploy(t *testing.T) { + const ( + host = "f-ns.apps.example.com" + routeNS = "openshift-keda" + ) d := NewDeployer() + // A function already deployed to "ns". Each subtest regenerates its + // Service as a redeploy would, against a different live Service. f := fn.Function{Name: "f", Deploy: fn.DeploySpec{Namespace: "ns"}} - ctx := t.Context() - clientset := fake.NewClientset() + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "f", Namespace: "ns"}} + + generate := func(t *testing.T, existing *corev1.Service) *corev1.Service { + t.Helper() + svc, err := d.generateService(f, "ns", false, deployment, existing) + if err != nil { + t.Fatal(err) + } + return svc + } + + // An exposed function: the live Service records both the hostname and + // where the Route is. A redeploy must keep both. + t.Run("hostname and route namespace both survive", func(t *testing.T) { + svc := generate(t, testService(map[string]string{ + RouteHostnameAnnotation: host, + RouteNamespaceAnnotation: routeNS, + })) + if got := svc.Annotations[RouteHostnameAnnotation]; got != host { + t.Errorf("hostname = %q, want %q", got, host) + } + if got := svc.Annotations[RouteNamespaceAnnotation]; got != routeNS { + t.Errorf("route namespace = %q, want %q", got, routeNS) + } + }) + + // Never exposed: no record appears, and a create (no live Service at + // all) is the same case. The copy is conditional on the key being + // present, so an unexposed Service does not grow empty annotations. + t.Run("nothing recorded leaves both off", func(t *testing.T) { + for name, existing := range map[string]*corev1.Service{ + "live Service with no record": testService(nil), + "create, no live Service": nil, + } { + t.Run(name, func(t *testing.T) { + svc := generate(t, existing) + if got, ok := svc.Annotations[RouteHostnameAnnotation]; ok { + t.Errorf("expected no hostname annotation, got %q", got) + } + if got, ok := svc.Annotations[RouteNamespaceAnnotation]; ok { + t.Errorf("expected no route-namespace annotation, got %q", got) + } + }) + } + }) +} + +// Test_ResolveExposure_WithExposer covers both directions of the reconcile a +// wired Exposer performs: active intent creates and records the hostname, +// cluster-local intent removes and clears it. The removal half is what makes +// toggling --expose=route off across redeploys work. +func Test_ResolveExposure_WithExposer(t *testing.T) { + t.Run("active intent exposes and records the host", func(t *testing.T) { + const host = "f-ns.apps.example.com" + ctx := t.Context() + // No Deployment staged: exposure must not depend on one existing. + clientset := fake.NewClientset(testService(nil)) + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + exposer := &stubExposer{host: host} + d := NewDeployer(WithExposer(exposer)) + f := fn.Function{Name: "f", Expose: fn.ExposeRoute, Deploy: fn.DeploySpec{Namespace: "ns"}} + + svc := testService(nil) + svc.UID = "uid-svc" + url, applied, err := d.resolveExposure(ctx, f, "ns", svc, clientset, dynClient) + if err != nil { + t.Fatal(err) + } + if applied != fn.ExposeRoute { + t.Errorf("applied = %q, want %q", applied, fn.ExposeRoute) + } + if url != "https://"+host { + t.Errorf("url = %q, want %q", url, "https://"+host) + } + if len(exposer.exposed) != 1 { + t.Fatalf("expected exactly one Expose call, got %d", len(exposer.exposed)) + } + // The raw deployer exposes the function's own Service, which also owns + // the Route so it never outlives its traffic target. + e := exposer.exposed[0] + if e.Name != "f" || e.Namespace != "ns" || e.TargetService != "f" || e.TargetPort != "http" { + t.Errorf("unexpected Exposure target: %+v", e) + } + if e.Owner == nil || e.Owner.Kind != "Service" || e.Owner.Name != "f" || e.Owner.UID != "uid-svc" { + t.Errorf("expected the Service as owner, got %+v", e.Owner) + } + + updated, err := clientset.CoreV1().Services("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if got := updated.Annotations[RouteHostnameAnnotation]; got != host { + t.Errorf("expected hostname %q recorded on the Service, got %q", host, got) + } + // The record is written both-or-neither; the raw deployer's Route + // sits in the function's own namespace. + if got := updated.Annotations[RouteNamespaceAnnotation]; got != "ns" { + t.Errorf("expected route namespace %q recorded on the Service, got %q", "ns", got) + } + }) + + t.Run("cluster-local intent with a record unexposes and clears it", func(t *testing.T) { + ctx := t.Context() + svc := testService(map[string]string{ + RouteHostnameAnnotation: "stale.apps.example.com", + RouteNamespaceAnnotation: "ns", + }) + clientset := fake.NewClientset(svc) + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + exposer := &stubExposer{} + d := NewDeployer(WithExposer(exposer)) + f := fn.Function{Name: "f", Expose: fn.ExposeNone, Deploy: fn.DeploySpec{Namespace: "ns"}} + + _, applied, err := d.resolveExposure(ctx, f, "ns", svc, clientset, dynClient) + if err != nil { + t.Fatal(err) + } + if applied != "" { + t.Errorf("applied = %q, want empty", applied) + } + // The raw deployer's Route sits beside its function, so the recorded + // namespace and the function's namespace are the same. + if len(exposer.unexposed) != 1 || exposer.unexposed[0] != "ns/ns/f" { + t.Errorf("expected one Unexpose of ns/ns/f, got %v", exposer.unexposed) + } + + updated, err := clientset.CoreV1().Services("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if got, ok := updated.Annotations[RouteHostnameAnnotation]; ok { + t.Errorf("expected the stale hostname cleared, still %q", got) + } + if got, ok := updated.Annotations[RouteNamespaceAnnotation]; ok { + t.Errorf("expected the recorded namespace cleared, still %q", got) + } + }) +} + +// Test_ResolveExposure_RecordFailureRollsBack: the Route is created before +// the record is written, and the no-record paths never look for one. A +// recording failure must therefore take the just-created Route back down; +// otherwise a live, unrecorded exposure survives that --expose=none cannot +// remove. +func Test_ResolveExposure_RecordFailureRollsBack(t *testing.T) { + const host = "f-ns.apps.example.com" + newClientset := func() *fake.Clientset { + clientset := fake.NewClientset(testService(nil)) + clientset.PrependReactor("update", "services", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("boom") + }) + return clientset + } + f := fn.Function{Name: "f", Expose: fn.ExposeRoute, Deploy: fn.DeploySpec{Namespace: "ns"}} dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + t.Run("rollback happens and both facts are reported", func(t *testing.T) { + exposer := &stubExposer{host: host} + d := NewDeployer(WithExposer(exposer)) + + _, _, err := d.resolveExposure(t.Context(), f, "ns", testService(nil), newClientset(), dynClient) + if err == nil { + t.Fatal("expected the deploy to fail when the record cannot be written") + } + if len(exposer.unexposed) != 1 || exposer.unexposed[0] != "ns/ns/f" { + t.Errorf("expected exactly one rollback Unexpose of ns/ns/f, got %v", exposer.unexposed) + } + if !strings.Contains(err.Error(), "rolled back") { + t.Errorf("expected the error to say the Route was rolled back, got: %v", err) + } + }) + + t.Run("a failed rollback reports both failures", func(t *testing.T) { + exposer := &stubExposer{host: host, unexposeErr: fmt.Errorf("also boom")} + d := NewDeployer(WithExposer(exposer)) + + _, _, err := d.resolveExposure(t.Context(), f, "ns", testService(nil), newClientset(), dynClient) + if err == nil { + t.Fatal("expected the deploy to fail") + } + if !strings.Contains(err.Error(), "recording the exposure failed") || + !strings.Contains(err.Error(), "rolling the Route back failed") { + t.Errorf("expected both failures reported, got: %v", err) + } + }) +} + +// Test_ResolveExposure_RecordOrSilence: teardown acts on the record the Service +// carries, and on nothing else. +// +// No record means this deployer created no Route, so the Route API is not +// reached at all and the intent behind the opt-out makes no difference. A +// record means a Route was made and is owed removal, so any failure to remove +// it is fatal: reporting cluster-local while an address nothing owns keeps +// serving is the outcome this refuses. +func Test_ResolveExposure_RecordOrSilence(t *testing.T) { + // unexposeErr on a no-record case is staged to fail loudly if reached. + notCalled := errors.New("must not be called") + denied := fmt.Errorf("%w: forbidden", deployer.ErrExposureNotVisible) + + tests := []struct { + name string + expose string + record bool + unexposeErr error + wantErr bool + wantCalls int + }{ + {name: "no record, unset intent: silent", expose: "", unexposeErr: notCalled}, + {name: "no record, explicit none: silent", expose: fn.ExposeNone, unexposeErr: notCalled}, + {name: "record, denied removal: fatal", expose: fn.ExposeNone, record: true, + unexposeErr: denied, wantErr: true, wantCalls: 1}, + {name: "record, any other failure: fatal", expose: fn.ExposeNone, record: true, + unexposeErr: errors.New("boom"), wantErr: true, wantCalls: 1}, + {name: "record, removal succeeds: proceeds", expose: fn.ExposeNone, record: true, wantCalls: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + var ann map[string]string + if tt.record { + ann = map[string]string{ + RouteHostnameAnnotation: "f-ns.apps.example.com", + RouteNamespaceAnnotation: "ns", + } + } + svc := testService(ann) + clientset := fake.NewClientset(svc) + dynClient := dynamicfakeclient.NewSimpleDynamicClient(runtime.NewScheme()) + + exposer := &stubExposer{unexposeErr: tt.unexposeErr} + d := NewDeployer(WithExposer(exposer)) + f := fn.Function{Name: "f", Expose: tt.expose, Deploy: fn.DeploySpec{Namespace: "ns"}} + + _, applied, err := d.resolveExposure(ctx, f, "ns", svc, clientset, dynClient) + switch { + case tt.wantErr && err == nil: + t.Fatal("expected a removal failure to be fatal where a record exists") + case !tt.wantErr && err != nil: + t.Fatalf("expected the deploy to proceed, got %v", err) + case !tt.wantErr && applied != "": + t.Errorf("applied = %q, want empty", applied) + } + if len(exposer.unexposed) != tt.wantCalls { + t.Errorf("Unexpose calls = %d, want %d", len(exposer.unexposed), tt.wantCalls) + } + }) + } +} + +// Test_DeployNamespace pins the rule two packages now share. pkg/keda checks +// the name its Route would take BEFORE the embedded raw deploy runs, so it +// needs this answer early; a copy of the rule there could drift and validate a +// name this deployer would not use. +func Test_DeployNamespace(t *testing.T) { tests := []struct { - name string - expose string - openShift bool - wantErr bool - wantExpose bool + name string + requested string + deployed string + want string + wantErr bool }{ - {name: "route off OpenShift: hard error", expose: "route", openShift: false, wantErr: true}, - {name: "none off OpenShift: fine", expose: "none", openShift: false}, - {name: "none on OpenShift: fine", expose: "none", openShift: true}, - {name: "empty off OpenShift: fine, cluster-local, no error", expose: "", openShift: false}, - // "empty on OpenShift" is NOT in this table: functions are exposed - // by default now, so unset+OpenShift takes the same real - // Route-creation path as explicit expose:route does - excluded - // here for the same reason "route on OpenShift" already is (see - // the comment above this test). + { + name: "requested wins", requested: "want-this", deployed: "already-here", + want: "want-this", + }, + { + // A redeploy with no --namespace stays where it is. + name: "falls back to where it is deployed", requested: "", deployed: "already-here", + want: "already-here", + }, + { + name: "requested only", requested: "want-this", deployed: "", + want: "want-this", + }, + { + // Neither known: the caller has to be told, not given a guess. + name: "neither: an error, never a default", requested: "", deployed: "", + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - cleanup := SetOpenShiftForTest(tt.openShift) - defer cleanup() + f := fn.Function{Name: "f", Namespace: tt.requested} + f.Deploy.Namespace = tt.deployed - f.Deploy.Expose = tt.expose - url, exposed, err := d.resolveExposure(ctx, f, "ns", clientset, dynClient) + got, err := DeployNamespace(f) if tt.wantErr { if err == nil { - t.Fatalf("resolveExposure(%q) on OpenShift=%v: expected an error, got nil", tt.expose, tt.openShift) + t.Fatalf("expected an error, got namespace %q", got) } return } if err != nil { - t.Fatalf("resolveExposure(%q) on OpenShift=%v: unexpected error: %v", tt.expose, tt.openShift, err) - } - if exposed != tt.wantExpose { - t.Errorf("resolveExposure(%q) on OpenShift=%v: exposed = %v, want %v", tt.expose, tt.openShift, exposed, tt.wantExpose) + t.Fatal(err) } - if url == "" { - t.Errorf("resolveExposure(%q) on OpenShift=%v: expected a non-empty URL", tt.expose, tt.openShift) + if got != tt.want { + t.Errorf("DeployNamespace() = %q, want %q", got, tt.want) } }) } } + +// Test_DomainStaysOutOfSelectors guards func.domain's exclusion from both +// selector fields. Deployment.spec.selector is immutable, so a domain there +// makes any redeploy with a changed --domain rejected outright; a domain in +// the Service selector stops matching running pods the moment it changes. +// The label stays everywhere else, since the Route reads it back to detect a +// domain change. Only a cluster enforces any of this, so this test is the +// one check that runs everywhere. +func Test_DomainStaysOutOfSelectors(t *testing.T) { + d := NewDeployer() + f := fn.Function{ + Name: "f", + Domain: "f.example.test", + Deploy: fn.DeploySpec{Image: "registry.example.com/f:latest"}, + } + rs, rcm, rpvc := sets.New[string](), sets.New[string](), sets.New[string]() + deployment, err := d.generateDeployment(f, "ns", false, &rs, &rcm, &rpvc) + if err != nil { + t.Fatal(err) + } + if _, ok := deployment.Spec.Selector.MatchLabels[deployer.DomainLabel]; ok { + t.Error("expected the immutable Deployment selector to exclude the domain label") + } + if got := deployment.Spec.Template.Labels[deployer.DomainLabel]; got != f.Domain { + t.Errorf("expected the pod template labels to carry the domain, got %q", got) + } + if got := deployment.Labels[deployer.DomainLabel]; got != f.Domain { + t.Errorf("expected the Deployment labels to carry the domain, got %q", got) + } + // The API server requires the selector to be satisfied by the template + // labels; assert the subset relation holds after the filtering. + for k, v := range deployment.Spec.Selector.MatchLabels { + if deployment.Spec.Template.Labels[k] != v { + t.Errorf("selector entry %s=%s not satisfied by template labels", k, v) + } + } + + svc, err := d.generateService(f, "ns", false, deployment, nil) + if err != nil { + t.Fatal(err) + } + if _, ok := svc.Spec.Selector[deployer.DomainLabel]; ok { + t.Error("expected the Service selector to exclude the domain label") + } + if got := svc.Labels[deployer.DomainLabel]; got != f.Domain { + t.Errorf("expected the Service labels to carry the domain, got %q", got) + } +} + +// Test_generateService_OwnerReferenceOmitsBlockOwnerDeletion guards the +// hand-built owner reference. metav1.NewControllerRef would also set +// BlockOwnerDeletion. Under OpenShift's OwnerReferencesPermissionEnforcement +// admission plugin, writing that flag requires update on the owner's +// finalizers subresource; ordinary users and the pipeline ServiceAccount +// hold no such grant, so the Service create fails outright. KinD does not +// run the plugin, so CI cannot catch a regression; this test can. +func Test_generateService_OwnerReferenceOmitsBlockOwnerDeletion(t *testing.T) { + d := NewDeployer() + f := fn.Function{Name: "f", Deploy: fn.DeploySpec{Image: "registry.example.com/f:latest"}} + deployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "f", Namespace: "ns", UID: "uid-1"}} + + svc, err := d.generateService(f, "ns", false, deployment, nil) + if err != nil { + t.Fatal(err) + } + if len(svc.OwnerReferences) != 1 { + t.Fatalf("expected exactly 1 owner reference, got %d", len(svc.OwnerReferences)) + } + ref := svc.OwnerReferences[0] + if ref.Kind != "Deployment" || ref.Name != "f" || ref.UID != "uid-1" { + t.Errorf("expected the Deployment as owner, got %+v", ref) + } + if ref.Controller == nil || !*ref.Controller { + t.Error("expected Controller set on the owner reference") + } + if ref.BlockOwnerDeletion != nil { + t.Error("expected BlockOwnerDeletion left unset: setting it makes OpenShift's " + + "OwnerReferencesPermissionEnforcement reject the Service create") + } +} + +// Older funcs pinned the whole label map in the immutable selector; keep the +// live selector on update and refuse when a pinned label changes. +func Test_preserveDeploymentSelector(t *testing.T) { + stable := map[string]string{ + "boson.dev/function": "true", + "function.knative.dev/name": "f", + } + withDomain := func(domain string) map[string]string { + m := map[string]string{} + maps.Copy(m, stable) + if domain != "" { + m[deployer.DomainLabel] = domain + } + return m + } + newDesired := func(templateLabels map[string]string) *appsv1.Deployment { + return &appsv1.Deployment{Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: stable}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: templateLabels}, + }, + }} + } + legacy := &appsv1.Deployment{Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: withDomain("old.example.com")}, + }} + + t.Run("unchanged domain keeps the live selector", func(t *testing.T) { + desired := newDesired(withDomain("old.example.com")) + if err := preserveDeploymentSelector(legacy, desired, "f"); err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + if got := desired.Spec.Selector.MatchLabels[deployer.DomainLabel]; got != "old.example.com" { + t.Errorf("expected the live selector preserved, pinned domain = %q", got) + } + }) + + t.Run("changed domain refuses with recreation instructions", func(t *testing.T) { + desired := newDesired(withDomain("new.example.com")) + err := preserveDeploymentSelector(legacy, desired, "f") + if err == nil { + t.Fatal("expected a refusal for a changed pinned domain") + } + if !strings.Contains(err.Error(), "func delete") { + t.Errorf("expected the refusal to point at recreation, got: %v", err) + } + }) + + t.Run("removed domain refuses too", func(t *testing.T) { + if err := preserveDeploymentSelector(legacy, newDesired(withDomain("")), "f"); err == nil { + t.Fatal("expected a refusal for a removed pinned domain") + } + }) + + t.Run("pinned user label refuses without domain wording", func(t *testing.T) { + pinned := withDomain("") + pinned["team"] = "a" + legacyUser := &appsv1.Deployment{Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: pinned}, + }} + err := preserveDeploymentSelector(legacyUser, newDesired(stable), "f") + if err == nil { + t.Fatal("expected a refusal for a dropped pinned user label") + } + if !strings.Contains(err.Error(), "team") { + t.Errorf("expected the pinned key named, got: %v", err) + } + if strings.Contains(err.Error(), "domain") { + t.Errorf("expected no domain-specific advice for a user label, got: %v", err) + } + }) + + t.Run("current-era selector accepts a new domain without pinning it", func(t *testing.T) { + current := &appsv1.Deployment{Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: stable}, + }} + desired := newDesired(withDomain("new.example.com")) + if err := preserveDeploymentSelector(current, desired, "f"); err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + if _, pinned := desired.Spec.Selector.MatchLabels[deployer.DomainLabel]; pinned { + t.Error("expected the domain to stay out of the selector") + } + }) +} diff --git a/pkg/k8s/describer.go b/pkg/k8s/describer.go index 4054d135ac..d7a904e218 100644 --- a/pkg/k8s/describer.go +++ b/pkg/k8s/describer.go @@ -83,8 +83,10 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In // External hostname (if exposed) was recorded on the Service by Deploy() // at exposure time - no extra API call or client needed here. + expose := "" if hostname, ok := service.Annotations[RouteHostnameAnnotation]; ok && hostname != "" { primaryRouteURL = fmt.Sprintf("https://%s", hostname) + expose = fn.ExposeRoute } // an exposed function stays reachable in-cluster too routes := []string{primaryRouteURL} @@ -114,6 +116,7 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In Name: name, Namespace: namespace, Deployer: KubernetesDeployerName, + Expose: expose, Labels: deployment.Labels, Route: primaryRouteURL, Routes: routes, diff --git a/pkg/k8s/labels/labels.go b/pkg/k8s/labels/labels.go index 1da9a797fc..6e7ea4cd66 100644 --- a/pkg/k8s/labels/labels.go +++ b/pkg/k8s/labels/labels.go @@ -3,4 +3,16 @@ package labels const ( FunctionRuntimeKey = "function.knative.dev/runtime" FunctionNameKey = "function.knative.dev/name" + + // FunctionNamespaceKey records the namespace the function itself lives + // in. It is redundant on an object sitting beside the function, and it is + // the only thing that tells two functions apart on an object that does + // not: keda's Routes all share the interceptor's namespace, so a function + // called "x" in two namespaces yields two Routes there whose name label + // is identical. Name plus namespace selects exactly one cluster-wide. + FunctionNamespaceKey = "function.knative.dev/namespace" + + // FunctionKey marks an object as created by func at all. Long-standing; + // named here so a selector can be built without a literal. + FunctionKey = "boson.dev/function" ) diff --git a/pkg/k8s/openshift.go b/pkg/k8s/openshift.go index b537ca47fc..f47258618c 100644 --- a/pkg/k8s/openshift.go +++ b/pkg/k8s/openshift.go @@ -10,6 +10,7 @@ import ( "time" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/rand" @@ -130,40 +131,60 @@ func GetOpenShiftDockerCredentialLoaders() []creds.CredentialsCallback { } -var isOpenShift bool -var checkOpenShiftOnce sync.Once +// openShiftRouteGroupVersion is the API group whose presence identifies an +// OpenShift cluster. Routes are OpenShift-specific, and discovery should answer +// it even under restrictive RBAC, unlike listing namespaces or services. +const openShiftRouteGroupVersion = "route.openshift.io/v1" -// SetOpenShiftForTest overrides OpenShift detection for testing. -// Returns a cleanup function that restores the previous state. -func SetOpenShiftForTest(val bool) func() { - checkOpenShiftOnce.Do(func() {}) // ensure real detection won't run - prev := isOpenShift - isOpenShift = val - return func() { isOpenShift = prev } -} +var ( + detectOnce sync.Once + isOpenShift bool + detectErr error +) -func IsOpenShift() bool { - checkOpenShiftOnce.Do(func() { - isOpenShift = false +// DetectOpenShift reports whether the cluster serves the OpenShift Route API. +// A non-nil error means the cluster could not be asked and the bool is +// meaningless. Probes once per process, answers from cache after. +func DetectOpenShift() (bool, error) { + detectOnce.Do(func() { client, err := NewKubernetesClientset() if err != nil { + detectErr = err return } - - // Detect OpenShift by checking for OpenShift-specific API groups - // This is reliable and works even with restrictive RBAC, unlike checking - // for namespaces/services which can produce false positives when forbidden - discoveryClient := client.Discovery() - - // Check for route.openshift.io API group (Routes are OpenShift-specific) - _, err = discoveryClient.ServerResourcesForGroupVersion("route.openshift.io/v1") - if err == nil { - // API group exists - this is OpenShift + _, err = client.Discovery().ServerResourcesForGroupVersion(openShiftRouteGroupVersion) + switch { + case err == nil: isOpenShift = true + case apierrors.IsNotFound(err): + // The cluster answered: it does not serve this API. + default: + detectErr = err } - // If NotFound or any other error, this is most likely not OpenShift }) - return isOpenShift + return isOpenShift, detectErr +} + +// IsOpenShift is a convenient wrapper for getting simple yes/no for openshift +// cluster. The inner function should run in the cmd layer once to resolve the +// detectOnce.Do(), any call after is cached so we dont have to call API all the +// time. +// +// note: gauron99: this might change after restructuring to kubeconfig resolution +// at the start of program instead of adhoc API calls of kube client throughout +// the codebase +func IsOpenShift() bool { + ok, _ := DetectOpenShift() + return ok +} + +// SetOpenShiftForTest seeds the detection cache; err simulates a cluster that +// could not be asked. Returns a cleanup restoring the previous state. +func SetOpenShiftForTest(val bool, err error) func() { + detectOnce.Do(func() {}) // ensure real detection won't run + prevB, prevE := isOpenShift, detectErr + isOpenShift, detectErr = val, err + return func() { isOpenShift, detectErr = prevB, prevE } } const ( diff --git a/pkg/k8s/openshift_unit_test.go b/pkg/k8s/openshift_unit_test.go index fdae044d3b..7571bd03de 100644 --- a/pkg/k8s/openshift_unit_test.go +++ b/pkg/k8s/openshift_unit_test.go @@ -1,6 +1,8 @@ package k8s -import "testing" +import ( + "testing" +) func TestIsOpenShiftInternalRegistry(t *testing.T) { tests := []struct { diff --git a/pkg/k8s/route.go b/pkg/k8s/route.go deleted file mode 100644 index e492dbfd37..0000000000 --- a/pkg/k8s/route.go +++ /dev/null @@ -1,268 +0,0 @@ -package k8s - -import ( - "context" - "fmt" - "os" - "time" - - appsv1 "k8s.io/api/apps/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/util/retry" - - "knative.dev/func/pkg/deployer" - fn "knative.dev/func/pkg/functions" -) - -// routeGVR identifies the OpenShift Route resource. No typed client is used -// here: adding github.com/openshift/api as a direct dependency for a -// handful of fields is disproportionate, this project already has a -// precedent for reading Routes through the dynamic client (see -// pkg/pipelines/tekton/pac/pac.go DetectPACOpenShiftRoute), and there is no -// existing github.com/openshift/api requirement anywhere in go.mod to build -// on. Route's structure is also small and stable (a v1, GA API since -// OpenShift 3.x), so hand-built unstructured content carries little -// maintenance risk. -var routeGVR = schema.GroupVersionResource{ - Group: "route.openshift.io", - Version: "v1", - Resource: "routes", -} - -// GenerateRoute builds (but does not create) the OpenShift Route that -// exposes svcName's "http" port. spec.host is left empty so the cluster's -// router mints one (see docs/research citations in the openshift-route-fork -// records) - custom domains are out of scope for this commit. -func GenerateRoute(f fn.Function, svcName string, deployment *appsv1.Deployment, decorator deployer.DeployDecorator, deployerName string) (*unstructured.Unstructured, error) { - labels, err := deployer.GenerateCommonLabels(f, decorator) - if err != nil { - return nil, err - } - annotations := deployer.GenerateCommonAnnotations(f, decorator, false /* dapr n/a for routing */, deployerName) - - route := &unstructured.Unstructured{ - Object: map[string]any{ - "apiVersion": routeGVR.GroupVersion().String(), - "kind": "Route", - "metadata": map[string]any{ - "name": f.Name, - "namespace": deployment.Namespace, - "labels": stringMapToAny(labels), - "annotations": stringMapToAny(annotations), - "ownerReferences": []any{ - map[string]any{ - "apiVersion": appsv1.SchemeGroupVersion.WithKind("Deployment").GroupVersion().String(), - "kind": "Deployment", - "name": deployment.Name, - "uid": string(deployment.UID), - "controller": true, - }, - }, - }, - "spec": map[string]any{ - "to": map[string]any{ - "kind": "Service", - "name": svcName, - }, - "port": map[string]any{ - "targetPort": "http", - }, - // Edge TLS via the router's wildcard cert - zero cert - // management; Redirect upgrades http requests to https. - "tls": map[string]any{ - "termination": "edge", - "insecureEdgeTerminationPolicy": "Redirect", - }, - }, - }, - } - - return route, nil -} - -// stringMapToAny converts a map[string]string to the map[string]any -// unstructured.Unstructured needs its nested fields to be. -func stringMapToAny(m map[string]string) map[string]any { - out := make(map[string]any, len(m)) - for k, v := range m { - out[k] = v - } - return out -} - -// EnsureRoute creates or updates a Route, retrying the whole -// get-mutate-update cycle on a 409 conflict (a controller status write can -// race an update from here). -func EnsureRoute(ctx context.Context, dynClient dynamic.Interface, ns string, route *unstructured.Unstructured) error { - client := dynClient.Resource(routeGVR).Namespace(ns) - name := route.GetName() - - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - existing, getErr := client.Get(ctx, name, metav1.GetOptions{}) - if getErr != nil { - if apierrors.IsNotFound(getErr) { - route.SetResourceVersion("") - _, createErr := client.Create(ctx, route, metav1.CreateOptions{}) - return createErr - } - return getErr - } - route.SetResourceVersion(existing.GetResourceVersion()) - _, updateErr := client.Update(ctx, route, metav1.UpdateOptions{}) - return updateErr - }) - if err != nil { - return fmt.Errorf("failed to ensure Route %q: %w", name, err) - } - return nil -} - -// isManagedRoute reports whether route was created by GenerateRoute() - as -// opposed to a user-authored or third-party Route that happens to share the -// function's name, which must never be deleted out from under the user. -// Both signals are required: a bare boson.dev/function label, or a -// deployer annotation written by some other component, alone does not -// prove func's raw deployer owns the route. -func isManagedRoute(route *unstructured.Unstructured) bool { - return route.GetLabels()["boson.dev/function"] == "true" && - route.GetAnnotations()[deployer.DeployerNameAnnotation] == KubernetesDeployerName -} - -// RemoveManagedRoute deletes the Route named 'name' in 'ns' only if func -// owns it (isManagedRoute()). Returns (removed, error): -// - not found (route absent, or the Route API isn't installed) -> (false, nil) -// - found but not managed -> (false, nil), warning printed, route kept -// - found and managed, deleted -> (true, nil) -func RemoveManagedRoute(ctx context.Context, dynClient dynamic.Interface, ns, name string) (bool, error) { - client := dynClient.Resource(routeGVR).Namespace(ns) - - route, err := client.Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("failed to check for existing Route %q: %w", name, err) - } - - if !isManagedRoute(route) { - fmt.Fprintf(os.Stderr, - "⚠️ a Route named %q exists in namespace %q but is not managed by func - leaving it in place\n", - name, ns) - return false, nil - } - - if err := client.Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { - return false, fmt.Errorf("failed to delete Route %q: %w", name, err) - } - return true, nil -} - -// WaitForRouteAdmitted polls the Route status until any ingress entry (one -// per router/IngressController shard - a cluster can run more than one) -// reports Admitted=True, returning that entry's host. It fails immediately -// (not waiting out the full timeout) only when an ingress entry explicitly -// reports Admitted=False - e.g. a host already claimed by another Route - -// surfacing the condition's reason and message. An entry with no Admitted -// condition yet is polled through to the timeout, fail-open on unknown. -func WaitForRouteAdmitted(ctx context.Context, dynClient dynamic.Interface, ns, name string, timeout time.Duration) (string, error) { - client := dynClient.Resource(routeGVR).Namespace(ns) - - var host string - var lastErr error - pollErr := wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { - route, err := client.Get(ctx, name, metav1.GetOptions{}) - if err != nil { - lastErr = fmt.Errorf("failed to get Route %q: %w", name, err) - return false, nil - } - - ingresses, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") - if err != nil || !found { - return false, nil - } - - for _, raw := range ingresses { - ingress, ok := raw.(map[string]any) - if !ok { - continue - } - conditions, found, err := unstructured.NestedSlice(ingress, "conditions") - if err != nil || !found { - continue - } - for _, rawCond := range conditions { - cond, ok := rawCond.(map[string]any) - if !ok || cond["type"] != "Admitted" { - continue - } - status, _ := cond["status"].(string) - switch status { - case "True": - host, _, _ = unstructured.NestedString(ingress, "host") - return true, nil - case "False": - reason, _ := cond["reason"].(string) - message, _ := cond["message"].(string) - lastErr = fmt.Errorf("route %q was rejected by the router: %s: %s", name, reason, message) - return false, lastErr - } - // Unknown or missing status: keep polling. - } - } - - return false, nil - }) - if pollErr != nil { - if lastErr != nil { - return "", lastErr - } - return "", fmt.Errorf("route %q was not admitted by any router within %s: %w", name, timeout, pollErr) - } - return host, nil -} - -// GetAdmittedRouteHost is a single, non-blocking read of a Route's currently -// admitted host, for display paths (describe/list) that must return -// immediately rather than poll like WaitForRouteAdmitted does. Returns -// ("", false, nil) if the Route doesn't exist or has no Admitted=True -// ingress entry yet - both are "no external URL to show", not errors. -func GetAdmittedRouteHost(ctx context.Context, dynClient dynamic.Interface, ns, name string) (string, bool, error) { - route, err := dynClient.Resource(routeGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apierrors.IsNotFound(err) { - return "", false, nil - } - return "", false, fmt.Errorf("failed to get Route %q: %w", name, err) - } - - ingresses, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") - if err != nil || !found { - return "", false, nil - } - for _, raw := range ingresses { - ingress, ok := raw.(map[string]any) - if !ok { - continue - } - conditions, found, err := unstructured.NestedSlice(ingress, "conditions") - if err != nil || !found { - continue - } - for _, rawCond := range conditions { - cond, ok := rawCond.(map[string]any) - if !ok || cond["type"] != "Admitted" { - continue - } - if status, _ := cond["status"].(string); status == "True" { - host, _, _ := unstructured.NestedString(ingress, "host") - return host, host != "", nil - } - } - } - return "", false, nil -} diff --git a/pkg/k8s/route_test.go b/pkg/k8s/route_test.go deleted file mode 100644 index 85bb44bda6..0000000000 --- a/pkg/k8s/route_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package k8s - -import ( - "testing" - "time" - - appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - dynamicfake "k8s.io/client-go/dynamic/fake" - - fn "knative.dev/func/pkg/functions" -) - -func newFakeDynamicClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { - return dynamicfake.NewSimpleDynamicClientWithCustomListKinds( - runtime.NewScheme(), - map[schema.GroupVersionResource]string{routeGVR: "RouteList"}, - objects..., - ) -} - -func testDeployment() *appsv1.Deployment { - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "f", - Namespace: "ns", - UID: types.UID("abc-123"), - }, - } -} - -func Test_GenerateRoute(t *testing.T) { - f := fn.Function{Name: "f", Runtime: "go"} - deployment := testDeployment() - - route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - - if route.GetName() != "f" || route.GetNamespace() != "ns" { - t.Errorf("expected name/namespace f/ns, got %s/%s", route.GetName(), route.GetNamespace()) - } - toName, _, _ := unstructured.NestedString(route.Object, "spec", "to", "name") - if toName != "f" { - t.Errorf("expected spec.to.name %q, got %q", "f", toName) - } - toKind, _, _ := unstructured.NestedString(route.Object, "spec", "to", "kind") - if toKind != "Service" { - t.Errorf("expected spec.to.kind Service, got %q", toKind) - } - targetPort, _, _ := unstructured.NestedString(route.Object, "spec", "port", "targetPort") - if targetPort != "http" { - t.Errorf("expected spec.port.targetPort http, got %q", targetPort) - } - if host, found, _ := unstructured.NestedString(route.Object, "spec", "host"); found && host != "" { - t.Errorf("expected spec.host to be unset (router-minted), got %q", host) - } - termination, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "termination") - if termination != "edge" { - t.Errorf("expected spec.tls.termination edge, got %q", termination) - } - insecurePolicy, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "insecureEdgeTerminationPolicy") - if insecurePolicy != "Redirect" { - t.Errorf("expected spec.tls.insecureEdgeTerminationPolicy Redirect, got %q", insecurePolicy) - } - if !isManagedRoute(route) { - t.Error("expected a freshly generated Route to be self-managed") - } - owners := route.GetOwnerReferences() - if len(owners) != 1 || owners[0].Name != "f" || owners[0].Kind != "Deployment" { - t.Errorf("expected a single Deployment ownerRef named f, got %+v", owners) - } -} - -func Test_EnsureRoute_CreateThenUpdate(t *testing.T) { - ctx := t.Context() - f := fn.Function{Name: "f", Runtime: "go"} - deployment := testDeployment() - client := newFakeDynamicClient() - - route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - if err := EnsureRoute(ctx, client, "ns", route); err != nil { - t.Fatalf("create: %v", err) - } - got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) - if err != nil { - t.Fatalf("expected Route to exist after create: %v", err) - } - toName, _, _ := unstructured.NestedString(got.Object, "spec", "to", "name") - if toName != "f" { - t.Errorf("expected spec.to.name f, got %q", toName) - } - - // Update path: regenerate (idempotent) and ensure again, no error. - route2, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - if err := EnsureRoute(ctx, client, "ns", route2); err != nil { - t.Fatalf("update: %v", err) - } -} - -func Test_RemoveManagedRoute(t *testing.T) { - ctx := t.Context() - - t.Run("not found: no-op", func(t *testing.T) { - client := newFakeDynamicClient() - removed, err := RemoveManagedRoute(ctx, client, "ns", "f") - if err != nil || removed { - t.Errorf("expected (false, nil), got (%v, %v)", removed, err) - } - }) - - t.Run("managed: deleted", func(t *testing.T) { - f := fn.Function{Name: "f", Runtime: "go"} - deployment := testDeployment() - route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - route.SetNamespace("ns") - client := newFakeDynamicClient(route) - - removed, err := RemoveManagedRoute(ctx, client, "ns", "f") - if err != nil || !removed { - t.Fatalf("expected (true, nil), got (%v, %v)", removed, err) - } - if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err == nil { - t.Error("expected Route to be gone after removal") - } - }) - - t.Run("not managed: kept", func(t *testing.T) { - foreign := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "route.openshift.io/v1", - "kind": "Route", - "metadata": map[string]any{ - "name": "f", - "namespace": "ns", - }, - }} - client := newFakeDynamicClient(foreign) - - removed, err := RemoveManagedRoute(ctx, client, "ns", "f") - if err != nil || removed { - t.Fatalf("expected (false, nil) for a foreign Route, got (%v, %v)", removed, err) - } - if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err != nil { - t.Error("expected the foreign Route to be left in place") - } - }) -} - -func Test_WaitForRouteAdmitted(t *testing.T) { - ctx := t.Context() - - admittedRoute := func(host string) *unstructured.Unstructured { - return &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "route.openshift.io/v1", - "kind": "Route", - "metadata": map[string]any{"name": "f", "namespace": "ns"}, - "status": map[string]any{ - "ingress": []any{ - map[string]any{ - "host": host, - "conditions": []any{ - map[string]any{"type": "Admitted", "status": "True"}, - }, - }, - }, - }, - }} - } - - t.Run("admitted: returns host", func(t *testing.T) { - client := newFakeDynamicClient(admittedRoute("f-ns.apps.example.com")) - host, err := WaitForRouteAdmitted(ctx, client, "ns", "f", time.Second) - if err != nil { - t.Fatal(err) - } - if host != "f-ns.apps.example.com" { - t.Errorf("expected host f-ns.apps.example.com, got %q", host) - } - }) - - t.Run("rejected: fails fast with reason", func(t *testing.T) { - rejected := &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "route.openshift.io/v1", - "kind": "Route", - "metadata": map[string]any{"name": "f", "namespace": "ns"}, - "status": map[string]any{ - "ingress": []any{ - map[string]any{ - "host": "", - "conditions": []any{ - map[string]any{"type": "Admitted", "status": "False", "reason": "HostAlreadyClaimed", "message": "taken"}, - }, - }, - }, - }, - }} - client := newFakeDynamicClient(rejected) - _, err := WaitForRouteAdmitted(ctx, client, "ns", "f", 5*time.Second) - if err == nil { - t.Fatal("expected an error for a rejected Route") - } - }) - - t.Run("never admitted: times out cleanly", func(t *testing.T) { - f := fn.Function{Name: "f", Runtime: "go"} - deployment := testDeployment() - route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - route.SetNamespace("ns") - client := newFakeDynamicClient(route) - - _, err = WaitForRouteAdmitted(ctx, client, "ns", "f", 100*time.Millisecond) - if err == nil { - t.Fatal("expected a timeout error when no router ever admits the route") - } - }) -} - -func Test_GetAdmittedRouteHost(t *testing.T) { - ctx := t.Context() - - admittedRoute := func(host string) *unstructured.Unstructured { - return &unstructured.Unstructured{Object: map[string]any{ - "apiVersion": "route.openshift.io/v1", - "kind": "Route", - "metadata": map[string]any{"name": "f", "namespace": "ns"}, - "status": map[string]any{ - "ingress": []any{ - map[string]any{ - "host": host, - "conditions": []any{ - map[string]any{"type": "Admitted", "status": "True"}, - }, - }, - }, - }, - }} - } - - t.Run("admitted: returns host", func(t *testing.T) { - client := newFakeDynamicClient(admittedRoute("f-ns.apps.example.com")) - host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") - if err != nil { - t.Fatal(err) - } - if !ok || host != "f-ns.apps.example.com" { - t.Errorf("expected (f-ns.apps.example.com, true), got (%q, %v)", host, ok) - } - }) - - t.Run("not found: no error, not found", func(t *testing.T) { - client := newFakeDynamicClient() - host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") - if err != nil || ok || host != "" { - t.Errorf("expected (\"\", false, nil), got (%q, %v, %v)", host, ok, err) - } - }) - - t.Run("not yet admitted: no error, not found", func(t *testing.T) { - f := fn.Function{Name: "f", Runtime: "go"} - deployment := testDeployment() - route, err := GenerateRoute(f, "f", deployment, nil, KubernetesDeployerName) - if err != nil { - t.Fatal(err) - } - route.SetNamespace("ns") - client := newFakeDynamicClient(route) - - host, ok, err := GetAdmittedRouteHost(ctx, client, "ns", "f") - if err != nil || ok || host != "" { - t.Errorf("expected (\"\", false, nil) for an unadmitted route, got (%q, %v, %v)", host, ok, err) - } - }) -} diff --git a/pkg/k8s/security_context_test.go b/pkg/k8s/security_context_test.go index 246171cd09..9c096c6ea3 100644 --- a/pkg/k8s/security_context_test.go +++ b/pkg/k8s/security_context_test.go @@ -11,7 +11,7 @@ import ( // See openshift.go:SetOpenShiftForTest. func TestDefaultPodSecurityContext_NonOpenShift(t *testing.T) { - cleanup := SetOpenShiftForTest(false) + cleanup := SetOpenShiftForTest(false, nil) defer cleanup() sc := defaultPodSecurityContext() @@ -36,7 +36,7 @@ func TestDefaultPodSecurityContext_NonOpenShift(t *testing.T) { } func TestDefaultPodSecurityContext_OpenShift(t *testing.T) { - cleanup := SetOpenShiftForTest(true) + cleanup := SetOpenShiftForTest(true, nil) defer cleanup() sc := defaultPodSecurityContext() @@ -107,7 +107,7 @@ func TestRestrictedProfileCompliance(t *testing.T) { name = "openshift" } t.Run(name, func(t *testing.T) { - cleanup := SetOpenShiftForTest(openshift) + cleanup := SetOpenShiftForTest(openshift, nil) defer cleanup() pod := defaultPodSecurityContext() diff --git a/pkg/keda/deployer.go b/pkg/keda/deployer.go index 7fff29a7a6..3bc83c0780 100644 --- a/pkg/keda/deployer.go +++ b/pkg/keda/deployer.go @@ -11,6 +11,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" "knative.dev/func/pkg/deployer" @@ -30,6 +31,7 @@ type Deployer struct { verbose bool decorator deployer.DeployDecorator + exposer deployer.Exposer } func NewDeployer(opts ...DeployerOpt) *Deployer { @@ -37,11 +39,6 @@ func NewDeployer(opts ...DeployerOpt) *Deployer { Deployer: *k8s.NewDeployer( // init with the kedaDeployerDecorator to have the correct deployer labels&annotations k8s.WithDeployerDecorator(&kedaDeployerDecorator{}), - // keda functions stay behind the interceptor; this deployer - // mints its own Route separately (see route.go) rather than - // letting the embedded raw deployer expose the function's own - // Service directly, which would bypass the interceptor entirely - k8s.WithDeployerExposureDisabled(), ), } @@ -58,6 +55,12 @@ func WithDeployerVerbose(verbose bool) DeployerOpt { } } +func WithExposer(exposer deployer.Exposer) DeployerOpt { + return func(d *Deployer) { + d.exposer = exposer + } +} + func WithDeployerDecorator(decorator deployer.DeployDecorator) DeployerOpt { // use the custom keda decorator, which wraps the given decorator, // but with the keda specific annotations @@ -97,6 +100,45 @@ func (k *kedaDeployerDecorator) UpdateLabels(function fn.Function, labels map[st } func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResult, error) { + // Refuse an unbuildable name before the raw deployer creates a Deployment + // and a Service the failure would leave behind. The Route's name is checked + // the same way below, once the exposure conditions are known. + if err := d.validateBridgeName(f); err != nil { + return fn.DeploymentResult{}, err + } + + k8sClientset, err := k8s.NewKubernetesClientset() + if err != nil { + return fn.DeploymentResult{}, fmt.Errorf("failed to create K8sClientset: %v", err) + } + + // Resolved once per deploy and threaded down: every caller wants the same + // answer, and it is needed before the raw deploy so the refusal below can + // run before anything is created. + interceptorNS, exposeRefusal := interceptorNamespace(ctx, k8sClientset) + + // Refuse rather than build a Route to a Service that may not be there: + // such a Route is admitted and then serves nothing. Refusing before the + // raw deploy leaves nothing half-built. The two refusals share the NO but + // not the WHY: "not found" and "could not look" send an operator to + // different fixes. + if d.exposer != nil && fn.ActiveExpose(f.Expose) { + // The Route's name needs the namespace the function will land in; + // k8s.DeployNamespace is the same rule the raw deployer uses, so this + // cannot validate a name the deploy will not use. + exposeNS, err := k8s.DeployNamespace(f) + if err != nil { + return fn.DeploymentResult{}, err + } + if err := validateExposureName(f, exposeNS); err != nil { + return fn.DeploymentResult{}, err + } + + if exposeRefusal != nil { + return fn.DeploymentResult{}, fmt.Errorf("cannot expose function %q: %w", f.Name, exposeRefusal) + } + } + // execute raw deployment deployer deployResult, err := d.Deployer.Deploy(ctx, f) if err != nil { @@ -106,11 +148,6 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu // create additional required keda resources namespace := deployResult.Namespace - k8sClientset, err := k8s.NewKubernetesClientset() - if err != nil { - return fn.DeploymentResult{}, fmt.Errorf("failed to create K8sClientset: %v", err) - } - deployment, err := k8sClientset.AppsV1().Deployments(namespace).Get(ctx, f.Name, metav1.GetOptions{}) if err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to get deployment %s/%s: %v", namespace, f.Name, err) @@ -121,7 +158,7 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu return fn.DeploymentResult{}, fmt.Errorf("failed to get service %s/%s: %v", namespace, f.Name, err) } - if err := d.ensureInterceptorBridgeService(ctx, k8sClientset, f, namespace, deployment); err != nil { + if err := d.ensureInterceptorBridgeService(ctx, k8sClientset, f, namespace, interceptorNS, deployment); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to ensure proxy service exists: %w", err) } @@ -129,16 +166,69 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (fn.DeploymentResu fmt.Sprintf("%s.%s.svc", d.interceptorBridgeServiceName(f), namespace), d.interceptorBridgeServiceName(f), } + url := fmt.Sprintf("http://%s:8080", hosts[0]) // TODO: check on HTTPS too + + // External exposure is reconciled around the HTTPScaledObject: creating + // comes first, because the interceptor 404s any Host header the HSO does + // not register; removing comes last, so a Forbidden in the interceptor's + // namespace cannot leave a function deployed but unscaled. A nil exposer + // means cluster-local, and neither half runs. + var dynClient dynamic.Interface + var exposedHost, appliedExpose string + if d.exposer != nil { + if dynClient, err = k8s.NewDynamicClient(); err != nil { + return fn.DeploymentResult{}, fmt.Errorf("failed to create dynamic client: %w", err) + } + if fn.ActiveExpose(f.Expose) { + // The interceptor refusal is NOT here. It is hoisted above the raw + // deploy, since a refusal at this point leaves the function + // half-built. Only the Route's name is checked here, because it is + // the one check that needs the resolved namespace. + exposedHost, err = d.exposer.Expose(ctx, dynClient, d.interceptorExposure(f, namespace, interceptorNS)) + if err != nil { + return fn.DeploymentResult{}, fmt.Errorf("failed to expose function externally: %w", err) + } + hosts = append(hosts, exposedHost) + // ocproute terminates TLS at the edge and redirects http. + url = fmt.Sprintf("https://%s", exposedHost) + appliedExpose = f.Expose + } + } if err := d.ensureHTTPScaledObject(ctx, f, namespace, deployment, appService, hosts); err != nil { return fn.DeploymentResult{}, fmt.Errorf("failed to ensure http scaled object exists: %w", err) } + if d.exposer != nil { + // Switching exposure off removes the Route BEFORE clearing the record, + // so a failed removal leaves the record for the retry to act on; The + // record outlives the Route, matching the raw deployer and the remover, + // at the price of a dead hostname in describe until the retry lands. + // recordedNS is read from appService, fetched before the exposure work + // above. + if appliedExpose == "" { + if recordedNS := appService.Annotations[k8s.RouteNamespaceAnnotation]; recordedNS != "" { + if err := d.exposer.Unexpose(ctx, dynClient, interceptorExposureRef(f.Name, namespace, recordedNS)); err != nil { + return fn.DeploymentResult{}, fmt.Errorf("failed to remove external exposure: %w", err) + } + } + } + + // The location is recorded together with the hostname, by the code + // that just created the Route and is certain where it went; removal + // reads it instead of resolving the interceptor again. Cleared only + // once the Route above is gone. + if err := d.recordExposure(ctx, k8sClientset, dynClient, f.Name, namespace, interceptorNS, exposedHost); err != nil { + return fn.DeploymentResult{}, err + } + } + return fn.DeploymentResult{ Status: deployResult.Status, - URL: fmt.Sprintf("http://%s:8080", hosts[0]), // TODO: check on HTTPS too + URL: url, Namespace: deployResult.Namespace, Deployer: KedaDeployerName, + Expose: appliedExpose, }, nil } @@ -210,11 +300,38 @@ func (d *Deployer) httpScaledObject(f fn.Function, namespace string, deployment }, nil } +// recordExposure writes the exposure record onto the function's Service. A +// record that cannot be written takes the just-created Route back down: the +// no-record paths deliberately never look for one, so an unrecorded Route +// would serve as an orphan nothing removes until delete's sweep. A kill +// between create and record still orphans; the sweep remains the backstop. +func (d *Deployer) recordExposure(ctx context.Context, clientset kubernetes.Interface, + dynClient dynamic.Interface, name, namespace, interceptorNS, exposedHost string) error { + + routeNS := "" + if exposedHost != "" { + routeNS = interceptorNS + } + err := k8s.SetRouteHostname(ctx, clientset, namespace, name, exposedHost, routeNS) + if err == nil { + return nil + } + if exposedHost == "" { + // A clear, not a record: nothing was created this deploy, so there + // is nothing to roll back. + return err + } + if rbErr := d.exposer.Unexpose(ctx, dynClient, interceptorExposureRef(name, namespace, interceptorNS)); rbErr != nil { + return fmt.Errorf("recording the exposure failed: %w; rolling the Route back failed too: %v", err, rbErr) + } + return fmt.Errorf("recording the exposure failed, the Route was rolled back: %w", err) +} + func (d *Deployer) interceptorBridgeServiceName(f fn.Function) string { - return fmt.Sprintf("%s-interceptor-bridge", f.Name) + return f.Name + interceptorBridgeSuffix } -func (d *Deployer) interceptorBridgeService(f fn.Function, namespace string, deployment *v1.Deployment) *corev1.Service { +func (d *Deployer) interceptorBridgeService(f fn.Function, namespace, interceptorNS string, deployment *v1.Deployment) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: d.interceptorBridgeServiceName(f), @@ -231,7 +348,7 @@ func (d *Deployer) interceptorBridgeService(f fn.Function, namespace string, dep }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeExternalName, - ExternalName: "keda-add-ons-http-interceptor-proxy.keda.svc.cluster.local", + ExternalName: fmt.Sprintf("%s.%s.svc.cluster.local", interceptorServiceName, interceptorNS), }, } } @@ -240,8 +357,8 @@ func (d *Deployer) interceptorBridgeService(f fn.Function, namespace string, dep // this service will server as an external-name service and forward the request to the keda interceptor-proxy by // preserving the host name. This service name is also used in the HTTPScaledObject as host name to allow the // interceptor to match the request with the correct target/scaledObject. -func (d *Deployer) ensureInterceptorBridgeService(ctx context.Context, clientset *kubernetes.Clientset, f fn.Function, namespace string, deployment *v1.Deployment) error { - expected := d.interceptorBridgeService(f, namespace, deployment) +func (d *Deployer) ensureInterceptorBridgeService(ctx context.Context, clientset *kubernetes.Clientset, f fn.Function, namespace, interceptorNS string, deployment *v1.Deployment) error { + expected := d.interceptorBridgeService(f, namespace, interceptorNS, deployment) existing, err := clientset.CoreV1().Services(expected.Namespace).Get(ctx, expected.Name, metav1.GetOptions{}) if err != nil { if k8serrors.IsNotFound(err) { diff --git a/pkg/keda/deployer_unit_test.go b/pkg/keda/deployer_unit_test.go new file mode 100644 index 0000000000..76ee27cb0f --- /dev/null +++ b/pkg/keda/deployer_unit_test.go @@ -0,0 +1,84 @@ +package keda + +import ( + "fmt" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "knative.dev/func/pkg/deployers" + "knative.dev/func/pkg/ocproute" +) + +// Test_recordExposure: keda's Route is created before the record is written +// and carries no owner reference, so a recording failure must take the Route +// back down or it survives as an orphan until delete's sweep. A clear (no +// exposed host) that fails must not touch the Route API at all. +func Test_recordExposure(t *testing.T) { + const ( + fnName = "f" + fnNS = "fn-keda" + interceptorNS = interceptorNamespaceUpstream + ) + routeName := interceptorExposureName(fnName, fnNS) + + newClientset := func(annotations map[string]string) *fake.Clientset { + clientset := fake.NewClientset(&corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: fnName, Namespace: fnNS, Annotations: annotations, + }}) + clientset.PrependReactor("update", "services", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("boom") + }) + return clientset + } + newDynClient := func(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { + return dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{testRouteGVR: "RouteList"}, + objects...) + } + + d := NewDeployer(WithExposer(ocproute.New(deployers.Keda))) + + t.Run("record failure rolls the Route back", func(t *testing.T) { + dynClient := newDynClient(kedaRoute(routeName, interceptorNS, fnName, fnNS)) + + err := d.recordExposure(t.Context(), newClientset(nil), dynClient, + fnName, fnNS, interceptorNS, "f-ns.apps.example.com") + if err == nil { + t.Fatal("expected the record failure to be reported") + } + list, listErr := dynClient.Resource(testRouteGVR).Namespace(interceptorNS).List(t.Context(), metav1.ListOptions{}) + if listErr != nil { + t.Fatal(listErr) + } + if len(list.Items) != 0 { + t.Errorf("expected the just-created Route rolled back, %d left", len(list.Items)) + } + }) + + t.Run("a failed clear touches no Route", func(t *testing.T) { + dynClient := newDynClient(kedaRoute(routeName, interceptorNS, fnName, fnNS)) + // Annotations present so the clear actually attempts a write. + clientset := newClientset(map[string]string{ + "function.knative.dev/route-hostname": "f-ns.apps.example.com", + "function.knative.dev/route-namespace": interceptorNS, + }) + + err := d.recordExposure(t.Context(), clientset, dynClient, fnName, fnNS, interceptorNS, "") + if err == nil { + t.Fatal("expected the failed clear to be reported") + } + for _, a := range dynClient.Actions() { + if a.GetVerb() == "delete" { + t.Error("a failed clear must not delete anything: nothing was created this deploy") + } + } + }) +} diff --git a/pkg/keda/describer.go b/pkg/keda/describer.go index 946f6e2a44..b1fb36e132 100644 --- a/pkg/keda/describer.go +++ b/pkg/keda/describer.go @@ -87,11 +87,15 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In return fn.Instance{}, fmt.Errorf("HTTPScaledObject %q does not have any hosts", name) } - routes := make([]string, 0, len(httpScaledObject.Spec.Hosts)) - for _, host := range httpScaledObject.Spec.Hosts { - routes = append(routes, fmt.Sprintf("http://%s:8080", host)) + // Deploy recorded the externally exposed hostname on the function's own + // Service, so no second lookup is needed to tell the exposed host apart + // from the bridge hosts it sits beside in Spec.Hosts. + hostname := service.Annotations[k8s.RouteHostnameAnnotation] + primaryRouteURL, routes := functionURLs(httpScaledObject.Spec.Hosts, hostname) + expose := "" + if hostname != "" { + expose = fn.ExposeRoute } - primaryRouteURL := routes[0] deploymentClient := clientset.AppsV1().Deployments(namespace) deployment, err := deploymentClient.Get(ctx, name, metav1.GetOptions{}) @@ -121,6 +125,7 @@ func (d *Describer) Describe(ctx context.Context, name, namespace string) (fn.In Name: name, Namespace: namespace, Deployer: KedaDeployerName, + Expose: expose, Labels: deployment.Labels, Route: primaryRouteURL, Routes: routes, diff --git a/pkg/keda/exposure.go b/pkg/keda/exposure.go new file mode 100644 index 0000000000..8b1ba48f18 --- /dev/null +++ b/pkg/keda/exposure.go @@ -0,0 +1,191 @@ +package keda + +import ( + "context" + "fmt" + "strings" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + + "knative.dev/func/pkg/deployer" + fn "knative.dev/func/pkg/functions" + "knative.dev/func/pkg/k8s" +) + +const ( + // Where the keda http-add-on installs its interceptor. The OpenShift Custom + // Metrics Autoscaler uses openshift-keda; the upstream helm chart, which + // pkg/cluster/keda.go and hack/cluster.sh set up on KinD, uses keda. Call + // interceptorNamespace() rather than picking one. + interceptorNamespaceOpenShift = "openshift-keda" + interceptorNamespaceUpstream = "keda" + + // interceptorServiceName is the Service the documented http-add-on + // install creates. A nonstandard helm release name would change it, and + // func would not find the interceptor. + interceptorServiceName = "keda-add-ons-http-interceptor-proxy" + + // interceptorBridgeSuffix is appended to the function's name to name its + // bridge Service. + interceptorBridgeSuffix = "-interceptor-bridge" + + // maxKedaFunctionName is what a DNS-1035 label's 63 characters leave for the + // function's name once the bridge suffix is taken. Derived, so the two + // cannot drift apart. + maxKedaFunctionName = 63 - len(interceptorBridgeSuffix) + + // interceptorServicePortName is the port name on the interceptor's Service: + // "proxy", not the "http" a function's Service uses. + interceptorServicePortName = "proxy" +) + +// interceptorNamespaceCandidates lists where the interceptor can run. CMA +// installs to openshift-keda and is an OpenShift-only product; the upstream +// chart documents installing to keda and runs anywhere, OpenShift included. +// Off OpenShift only keda is possible, so only keda is probed. Order matters +// when every probe is denied: Forbidden says nothing about existence, so the +// first candidate becomes the guess. +func interceptorNamespaceCandidates() []string { + if k8s.IsOpenShift() { + return []string{interceptorNamespaceOpenShift, interceptorNamespaceUpstream} + } + return []string{interceptorNamespaceUpstream} +} + +// interceptorNamespace looks for the interceptor's Service in each candidate +// namespace, in platform preference order; the first read-back wins. When no +// candidate read back, the guessed namespace is still returned, good enough +// for the cluster-local bridge, and exposeRefusal says why it is not good +// enough to build a Route to. +func interceptorNamespace(ctx context.Context, + clientset kubernetes.Interface) (ns string, exposeRefusal error) { + + candidates := interceptorNamespaceCandidates() + + var undetermined []string + for _, candidate := range candidates { + _, err := clientset.CoreV1().Services(candidate).Get(ctx, interceptorServiceName, metav1.GetOptions{}) + if err == nil { + return candidate, nil + } + if !k8serrors.IsNotFound(err) { + undetermined = append(undetermined, candidate) + } + } + + // Something could not be read, so it is not ruled out: the caller was + // denied, not answered. Only the namespaces actually in doubt are named. + if len(undetermined) > 0 { + return undetermined[0], fmt.Errorf( + "could not determine whether the keda interceptor Service %q exists in %s, "+ + "so its Route might point at nothing; this is usually a permissions problem "+ + "rather than a missing interceptor, and needs read access to that namespace", + interceptorServiceName, strings.Join(undetermined, " or ")) + } + // Every candidate answered NotFound. That is an answer. + return candidates[0], fmt.Errorf( + "the keda interceptor Service %q was not found in %s, so its Route would point at nothing", + interceptorServiceName, strings.Join(candidates, " or ")) +} + +// interceptorExposureName builds the name of the object func creates to +// expose a keda function; the interceptor itself is keda's, func only routes +// through it. Every keda function's exposure lands in the one interceptor +// namespace, so the name carries the function's namespace too; without it two +// functions of the same name in different namespaces would collide. +func interceptorExposureName(name, namespace string) string { + return fmt.Sprintf("%s-%s", name, namespace) +} + +// functionURLs returns the URLs to report for a keda function, primary first. +// Every host an HTTPScaledObject registers is a cluster-local bridge address +// answering on :8080, except the exposed hostname: it is registered only so +// the interceptor recognizes requests carrying it, and is reached over https +// through the exposing object. An exposed function leads with that URL, the +// only one reachable from outside. exposedHost is empty for a cluster-local +// function. +func functionURLs(hosts []string, exposedHost string) (primary string, all []string) { + all = make([]string, 0, len(hosts)+1) + if exposedHost != "" { + all = append(all, fmt.Sprintf("https://%s", exposedHost)) + } + for _, host := range hosts { + if host == exposedHost { + continue + } + all = append(all, fmt.Sprintf("http://%s:8080", host)) + } + if len(all) == 0 { + return "", nil + } + return all[0], all +} + +// interceptorExposure describes the external address wanted for a keda +// function. It targets the shared interceptor Service, not the function's +// own: a Route straight to the function would bypass scale-from-zero. Owner +// is nil because Kubernetes rejects cross-namespace owner references, so +// nothing garbage collects this Route; it is removed explicitly instead, +// when exposure is switched off and when the function is deleted. +func (d *Deployer) interceptorExposure(f fn.Function, namespace, interceptorNS string) deployer.Exposure { + return deployer.Exposure{ + Function: f, + // Different from the Route's namespace, and it is what identifies the + // Route as this function's inside a namespace shared with every other + // keda function. + FunctionNamespace: namespace, + Name: interceptorExposureName(f.Name, namespace), + Namespace: interceptorNS, + TargetService: interceptorServiceName, + TargetPort: interceptorServicePortName, + Owner: nil, + Decorator: d.decorator, + } +} + +// interceptorExposureRef identifies a keda function's Route for teardown, +// which needs to find it, not build it: Deploy calls when exposure is +// switched off, Remove on delete. The Route lives beside the interceptor +// rather than beside the function, so the caller names the namespace: the +// one recorded on the Service, or a sweep candidate when nothing was. +func interceptorExposureRef(name, namespace, interceptorNS string) deployer.ExposureRef { + return deployer.ExposureRef{ + FunctionName: name, + FunctionNamespace: namespace, + Namespace: interceptorNS, + } +} + +// validateBridgeName refuses a function whose bridge Service name would not +// be a valid DNS-1035 label: the suffix leaves maxKedaFunctionName +// characters for the function's name, past which the API server rejects the +// Service on a plain keda deploy. This check needs only the name, so it runs +// before the namespace is resolved; validateExposureName handles the name +// that cannot be built until then. +func (d *Deployer) validateBridgeName(f fn.Function) error { + bridge := d.interceptorBridgeServiceName(f) + if errs := validation.IsDNS1035Label(bridge); len(errs) > 0 { + return fmt.Errorf( + "function name %q is too long for the keda deployer: its bridge Service would be named %q, which is not a valid Service name (%s). Keda limits function names to %d characters", + f.Name, bridge, strings.Join(errs, "; "), maxKedaFunctionName) + } + return nil +} + +// validateExposureName refuses a Route name Kubernetes would not accept. It +// needs the resolved namespace, which is why it is separate from +// validateBridgeName. The minted hostname's own 63-character budget is +// deliberately not checked here: the arithmetic and the reasoning live in +// pkg/ocproute, beside the code that builds the host. +func validateExposureName(f fn.Function, namespace string) error { + route := interceptorExposureName(f.Name, namespace) + if errs := validation.IsDNS1123Subdomain(route); len(errs) > 0 { + return fmt.Errorf( + "function %q cannot be exposed: its Route would be named %q, which is not a valid Route name (%s)", + f.Name, route, strings.Join(errs, "; ")) + } + return nil +} diff --git a/pkg/keda/exposure_test.go b/pkg/keda/exposure_test.go new file mode 100644 index 0000000000..f327aad159 --- /dev/null +++ b/pkg/keda/exposure_test.go @@ -0,0 +1,331 @@ +package keda + +import ( + "errors" + "slices" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + fn "knative.dev/func/pkg/functions" + "knative.dev/func/pkg/k8s" +) + +// errDenied fills the cause slot of the synthetic Forbidden errors the test +// reactors return. Only the 403 type matters to the code under test; the +// text is never read. +var errDenied = errors.New("denied") + +// Test_interceptorNamespace: the interceptor's namespace depends on how keda +// was installed, and the two installs func supports disagree. Getting it wrong +// points the bridge Service and every exposing object at a namespace that +// holds nothing. +// +// It is resolved by looking for the interceptor Service, so the platform sets +// only the order tried and the answer given when nothing definite came back. +// +// Note: SetOpenShiftForTest mutates a package-level bool without a mutex, so +// this test must not run with t.Parallel() (see pkg/k8s/openshift.go). +func Test_interceptorNamespace(t *testing.T) { + interceptorServiceIn := func(ns string) *corev1.Service { + return &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: interceptorServiceName, Namespace: ns, + }} + } + + tests := []struct { + name string + openShift bool + installed []string + want string + // wantRefusal is a substring of the refusal's error message; empty + // means the namespace was confirmed. + wantRefusal string + }{ + { + name: "CMA on OpenShift", openShift: true, + installed: []string{interceptorNamespaceOpenShift}, + want: interceptorNamespaceOpenShift, + }, + { + // OpenShift clusters can run either CMA or upstream keda, in + // different namespaces; the probe finds what platform inference + // alone would miss. + name: "upstream keda on OpenShift", openShift: true, + installed: []string{interceptorNamespaceUpstream}, + want: interceptorNamespaceUpstream, + }, + { + name: "upstream keda off OpenShift", openShift: false, + installed: []string{interceptorNamespaceUpstream}, + want: interceptorNamespaceUpstream, + }, + { + // Both present is a broken install (CMA requires removing + // community keda first), but the pick must still be + // deterministic: CMA's namespace wins. + name: "both installs present on OpenShift", openShift: true, + installed: []string{interceptorNamespaceOpenShift, interceptorNamespaceUpstream}, + want: interceptorNamespaceOpenShift, + }, + { + // Both candidates answered NotFound. + name: "neither installed: platform default", openShift: true, + installed: nil, + want: interceptorNamespaceOpenShift, wantRefusal: "was not found", + }, + { + name: "neither installed off OpenShift: platform default", openShift: false, + installed: nil, + want: interceptorNamespaceUpstream, wantRefusal: "was not found", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cleanup := k8s.SetOpenShiftForTest(tt.openShift, nil) + defer cleanup() + + // A fake cluster seeded with whatever interceptor Services this + // row installs. + objects := make([]runtime.Object, 0, len(tt.installed)) + for _, ns := range tt.installed { + objects = append(objects, interceptorServiceIn(ns)) + } + got, refusal := interceptorNamespace(t.Context(), fake.NewClientset(objects...)) + if got != tt.want { + t.Errorf("interceptorNamespace() = %q, want %q", got, tt.want) + } + if tt.wantRefusal == "" { + if refusal != nil { + t.Errorf("interceptorNamespace() refusal = %v, want none", refusal) + } + } else if refusal == nil || !strings.Contains(refusal.Error(), tt.wantRefusal) { + t.Errorf("interceptorNamespace() refusal = %v, want one containing %q", refusal, tt.wantRefusal) + } + }) + } +} + +// Test_interceptorNamespace_AllDeniedUsesPlatformDefault: a restricted +// account that cannot read any candidate gets pure platform inference, the +// same answer probing nothing would give. Denial degrades the probe to a +// no-op, never to an error or a wrong claim. +// +// This case cannot distinguish reading Forbidden as "absent" from reading it +// as "unknown", because both answer with the same namespace when every +// candidate is denied. Test_interceptorNamespace_DeniedBeatsRuledOut is the +// case that separates them; this one only holds the floor. +func Test_interceptorNamespace_AllDeniedUsesPlatformDefault(t *testing.T) { + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() + + // An interceptor really is installed; denial hides it, so the guess + // below is genuinely blind. + clientset := fake.NewClientset(&corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: interceptorServiceName, Namespace: interceptorNamespaceUpstream, + }}) + clientset.PrependReactor("get", "services", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, k8serrors.NewForbidden( + schema.GroupResource{Resource: "services"}, interceptorServiceName, errDenied) + }) + + got, refusal := interceptorNamespace(t.Context(), clientset) + if got != interceptorNamespaceOpenShift { + t.Errorf("interceptorNamespace() = %q, want the platform default %q when every lookup is denied", + got, interceptorNamespaceOpenShift) + } + // The namespace is the same as the absent case; the refusal is what must + // differ, since it is the only thing telling the caller it never looked. + if refusal == nil || !strings.Contains(refusal.Error(), "could not determine") { + t.Errorf("refusal = %v, want one saying it could not determine: a denied lookup must not report absence", refusal) + } +} + +// Test_interceptorNamespace_DeniedBeatsRuledOut: when one candidate is +// definitely absent and the other cannot be seen, the one that could not be +// ruled out wins. Answering with a namespace known to hold nothing would be +// strictly worse than admitting ignorance. +func Test_interceptorNamespace_DeniedBeatsRuledOut(t *testing.T) { + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() + + clientset := fake.NewClientset() + clientset.PrependReactor("get", "services", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetNamespace() == interceptorNamespaceUpstream { + return true, nil, k8serrors.NewForbidden( + schema.GroupResource{Resource: "services"}, interceptorServiceName, errDenied) + } + return false, nil, nil // openshift-keda falls through to a real NotFound + }) + + got, refusal := interceptorNamespace(t.Context(), clientset) + if got != interceptorNamespaceUpstream { + t.Errorf("interceptorNamespace() = %q, want %q: the ruled-out candidate must lose to the unseen one", + got, interceptorNamespaceUpstream) + } + if refusal == nil || !strings.Contains(refusal.Error(), "could not determine") { + t.Errorf("refusal = %v, want one saying it could not determine: one candidate was never ruled out", refusal) + } +} + +// Test_interceptorExposureName: every keda function's exposing object lands in +// the one interceptor namespace, so the name has to separate two functions +// that share a name in different namespaces. Without the namespace in the +// name, the second deploy would retarget the first function's object. +func Test_interceptorExposureName(t *testing.T) { + a := interceptorExposureName("f", "alice") + b := interceptorExposureName("f", "bob") + if a == b { + t.Fatalf("same-named functions in different namespaces collided on %q", a) + } + if a != "f-alice" { + t.Errorf("interceptorExposureName = %q, want %q", a, "f-alice") + } +} + +// Test_interceptorExposure: keda's exposure must target the shared interceptor +// Service, not the function's own. A Route to the function's Service would +// bypass the interceptor, and a function scaled to zero would answer nothing. +// It must also carry no owner reference, since Kubernetes rejects one across +// namespaces and the interceptor's namespace is not the function's. +func Test_interceptorExposure(t *testing.T) { + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() + + d := NewDeployer() + e := d.interceptorExposure(fn.Function{Name: "f", Runtime: "go"}, "ns", interceptorNamespaceOpenShift) + + if e.TargetService != interceptorServiceName { + t.Errorf("TargetService = %q, want the interceptor %q; targeting anything else, like the function's own Service, would bypass scale-from-zero", + e.TargetService, interceptorServiceName) + } + if e.TargetPort != interceptorServicePortName { + t.Errorf("TargetPort = %q, want %q", e.TargetPort, interceptorServicePortName) + } + if e.Namespace != interceptorNamespaceOpenShift { + t.Errorf("Namespace = %q, want %q", e.Namespace, interceptorNamespaceOpenShift) + } + if e.Name != "f-ns" { + t.Errorf("Name = %q, want %q", e.Name, "f-ns") + } + if e.Owner != nil { + t.Errorf("Owner = %+v, want nil: an owner reference cannot cross namespaces", e.Owner) + } +} + +// Test_functionURLs: the exposed hostname is registered among the +// HTTPScaledObject's hosts so the interceptor matches requests carrying it, +// which means the reporting paths cannot treat every host alike. The exposed +// one is reached over https and never on :8080, and it leads, being the only +// address reachable from outside the cluster. +func Test_functionURLs(t *testing.T) { + bridges := []string{"f-interceptor-bridge.ns.svc", "f-interceptor-bridge"} + + t.Run("cluster-local reports only bridge addresses", func(t *testing.T) { + primary, all := functionURLs(bridges, "") + want := []string{"http://f-interceptor-bridge.ns.svc:8080", "http://f-interceptor-bridge:8080"} + if !slices.Equal(all, want) { + t.Errorf("all = %v, want %v", all, want) + } + if primary != want[0] { + t.Errorf("primary = %q, want %q", primary, want[0]) + } + }) + + t.Run("exposed leads with https and never repeats the host on :8080", func(t *testing.T) { + const host = "f-ns.apps.example.com" + primary, all := functionURLs(append(slices.Clone(bridges), host), host) + + if primary != "https://"+host { + t.Errorf("primary = %q, want %q", primary, "https://"+host) + } + want := []string{ + "https://" + host, + "http://f-interceptor-bridge.ns.svc:8080", + "http://f-interceptor-bridge:8080", + } + if !slices.Equal(all, want) { + t.Errorf("all = %v, want %v", all, want) + } + if slices.Contains(all, "http://"+host+":8080") { + t.Error("exposed host reported as a bridge address on :8080") + } + }) + + t.Run("no hosts at all", func(t *testing.T) { + if primary, all := functionURLs(nil, ""); primary != "" || len(all) != 0 { + t.Errorf("expected no URLs, got primary %q and %v", primary, all) + } + }) +} + +// Test_validateBridgeName: a function name that ValidateFunctionName accepts +// can still produce a bridge Service name the API server will not, because +// the suffix pushes it past a DNS-1035 label's 63 characters. +func Test_validateBridgeName(t *testing.T) { + name := func(n int) string { + s := "a" + for len(s) < n { + s += "b" + } + return s + } + + d := NewDeployer() + + t.Run("ordinary name", func(t *testing.T) { + if err := d.validateBridgeName(fn.Function{Name: "myfunc"}); err != nil { + t.Errorf("expected an ordinary name to pass, got %v", err) + } + }) + + t.Run("longest name that fits", func(t *testing.T) { + if err := d.validateBridgeName(fn.Function{Name: name(maxKedaFunctionName)}); err != nil { + t.Errorf("expected a %d character name to pass, got %v", maxKedaFunctionName, err) + } + }) + + t.Run("one character too long", func(t *testing.T) { + f := fn.Function{Name: name(maxKedaFunctionName + 1)} + err := d.validateBridgeName(f) + if err == nil { + t.Fatalf("expected a %d character name to be refused", maxKedaFunctionName+1) + } + if !strings.Contains(err.Error(), f.Name) { + t.Errorf("expected the error to name the function, got %v", err) + } + }) + + t.Run("the longest name func itself accepts", func(t *testing.T) { + // utils.ValidateFunctionName admits any DNS-1035 label, so 63 is + // legal as a function name and still too long here. + if err := d.validateBridgeName(fn.Function{Name: name(63)}); err == nil { + t.Error("expected a 63 character name, legal as a function name, to be refused by keda") + } + }) +} + +// Test_validateExposureName: the Route's name is built from the function's +// name AND its namespace, so it cannot be checked until the namespace is +// resolved. An unresolved namespace yields a name ending in a hyphen, which is +// not a valid DNS-1123 subdomain, and checking too early would refuse a first +// deploy that is perfectly fine. +func Test_validateExposureName(t *testing.T) { + f := fn.Function{Name: "myfunc"} + + if err := validateExposureName(f, "myns"); err != nil { + t.Errorf("expected a resolved namespace to pass, got %v", err) + } + + if err := validateExposureName(f, ""); err == nil { + t.Error("expected an unresolved namespace to be refused rather than silently producing 'myfunc-'") + } +} diff --git a/pkg/keda/lister.go b/pkg/keda/lister.go index 4d100430ae..cf2c78001e 100644 --- a/pkg/keda/lister.go +++ b/pkg/keda/lister.go @@ -49,7 +49,8 @@ func (l *Lister) List(ctx context.Context, namespace string) ([]fn.ListItem, err continue } - item, err := l.get(ctx, httpScaledObjectClientset, service.Name, service.Namespace) + item, err := l.get(ctx, httpScaledObjectClientset, service.Name, + service.Namespace, service.Annotations[k8s.RouteHostnameAnnotation]) if err != nil { return nil, fmt.Errorf("unable to get details about function: %v", err) } @@ -60,8 +61,11 @@ func (l *Lister) List(ctx context.Context, namespace string) ([]fn.ListItem, err return listItems, nil } -// Get a function, optionally specifying a namespace. -func (l *Lister) get(ctx context.Context, httpScaledObjectClientset *versioned.Clientset, name, namespace string) (fn.ListItem, error) { +// Get a function, optionally specifying a namespace. exposedHost is the +// hostname Deploy recorded on the function's Service, empty when the function +// is cluster-local; List reads it there rather than looking the exposing +// object up again. +func (l *Lister) get(ctx context.Context, httpScaledObjectClientset *versioned.Clientset, name, namespace, exposedHost string) (fn.ListItem, error) { httpScaledObject, err := httpScaledObjectClientset.HttpV1alpha1().HTTPScaledObjects(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return fn.ListItem{}, fmt.Errorf("unable to get HTTPScaledObject: %v", err) @@ -74,10 +78,7 @@ func (l *Lister) get(ctx context.Context, httpScaledObjectClientset *versioned.C ready = v1.ConditionFalse } - url := "" - if len(httpScaledObject.Spec.Hosts) > 0 { - url = fmt.Sprintf("http://%s:8080", httpScaledObject.Spec.Hosts[0]) - } + url, _ := functionURLs(httpScaledObject.Spec.Hosts, exposedHost) runtimeLabel := "" listItem := fn.ListItem{ diff --git a/pkg/keda/remover.go b/pkg/keda/remover.go index 097d020856..589a56b5b2 100644 --- a/pkg/keda/remover.go +++ b/pkg/keda/remover.go @@ -7,18 +7,38 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "knative.dev/func/pkg/deployer" fn "knative.dev/func/pkg/functions" "knative.dev/func/pkg/k8s" ) -func NewRemover(verbose bool) *Remover { - return &Remover{ +type RemoverOpt func(*Remover) + +func NewRemover(verbose bool, opts ...RemoverOpt) *Remover { + r := &Remover{ verbose: verbose, } + for _, opt := range opts { + opt(r) + } + return r +} + +// WithRemoverExposer gives the Remover the mechanism that exposed the +// function, so Remove can take that exposure away. A keda function's exposing +// object lives in the interceptor's namespace and carries no owner reference, +// since Kubernetes rejects one across namespaces, so deleting the Deployment +// does not garbage collect it the way it collects everything else. +func WithRemoverExposer(exposer deployer.Exposer) RemoverOpt { + return func(r *Remover) { + r.exposer = exposer + } } type Remover struct { verbose bool + exposer deployer.Exposer } func (remover *Remover) Remove(ctx context.Context, name, ns string) error { @@ -32,27 +52,45 @@ func (remover *Remover) Remove(ctx context.Context, name, ns string) error { return fmt.Errorf("could not setup kubernetes clientset: %w", err) } - serviceClient := clientset.CoreV1().Services(ns) - svc, err := serviceClient.Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if apiErrors.IsNotFound(err) { - // Service doesn't exist - we don't handle this - return fn.ErrNotHandled - } + svc, err := clientset.CoreV1().Services(ns).Get(ctx, name, metav1.GetOptions{}) + switch { + case apiErrors.IsNotFound(err): + // No Service means nothing to inspect: the record lived on it, and so + // did the annotation saying this was keda's. + // + // A Route may survive this, since keda's carries no owner reference and + // nothing collects it. That leftover is manual cleanup. + return fn.ErrNotHandled + + case err != nil: return err - } - if !UsesKedaDeployer(svc.Annotations) { + case !UsesKedaDeployer(svc.Annotations): + // Another deployer's function, and its own objects are all still here + // to say so. return fn.ErrNotHandled } - // We're responsible, for this function --> proceed... + // Where the Route is, read off the Service before anything is deleted. + recordedNS := svc.Annotations[k8s.RouteNamespaceAnnotation] + + // Remove the Route before deleting anything, and fail the whole delete if + // it cannot be removed. Failing here touches nothing, so the user can + // simply retry. + if remover.exposer != nil { + dynClient, err := k8s.NewDynamicClient() + if err != nil { + return fmt.Errorf("could not setup dynamic client: %w", err) + } + if err := remover.unexpose(ctx, dynClient, recordedNS, name, ns); err != nil { + return err + } + } deploymentClient := clientset.AppsV1().Deployments(ns) - // delete only the deployment and let the api server handle the others via the owner reference - err = deploymentClient.Delete(ctx, name, metav1.DeleteOptions{}) - if err != nil { + // Delete only the Deployment; owner references take the rest with it. + if err := deploymentClient.Delete(ctx, name, metav1.DeleteOptions{}); err != nil { if apiErrors.IsNotFound(err) { return fn.ErrFunctionNotFound } @@ -60,8 +98,34 @@ func (remover *Remover) Remove(ctx context.Context, name, ns string) error { } if err := k8s.WaitForServiceRemoved(ctx, clientset, ns, name, k8s.DefaultWaitingTimeout); err != nil { - return fmt.Errorf("k8s remover failed to propagate service deletion: %v", err) + return fmt.Errorf("keda remover failed to propagate service deletion: %v", err) } return nil } + +// unexpose removes the function's Route, found by its name and namespace +// labels. A record (recordedNS non-empty) names the one namespace to act +// on, and failing there fails the delete: the record says removal is owed. +// With no record, the candidate namespaces are swept best-effort for a +// Route a crash left unrecorded: a candidate that cannot be checked is +// warned about and skipped. dynClient is a parameter so a test can reach +// this. +func (remover *Remover) unexpose(ctx context.Context, dynClient dynamic.Interface, recordedNS, name, ns string) error { + if recordedNS == "" { + for _, candidate := range interceptorNamespaceCandidates() { + if err := remover.exposer.Unexpose(ctx, dynClient, interceptorExposureRef(name, ns, candidate)); err != nil { + fmt.Fprintf(os.Stderr, "Warning: namespace %q could not be checked for a leftover Route of function %q: %v\n", + candidate, name, err) + } + } + return nil + } + + if err := remover.exposer.Unexpose(ctx, dynClient, interceptorExposureRef(name, ns, recordedNS)); err != nil { + return fmt.Errorf("could not remove the Route exposing function %q in namespace %q; "+ + "nothing was deleted and the function is still running, if you fix this you can run delete again: %w", + name, recordedNS, err) + } + return nil +} diff --git a/pkg/keda/remover_unit_test.go b/pkg/keda/remover_unit_test.go new file mode 100644 index 0000000000..55b51ce8b5 --- /dev/null +++ b/pkg/keda/remover_unit_test.go @@ -0,0 +1,207 @@ +package keda + +import ( + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + + "knative.dev/func/pkg/deployer" + "knative.dev/func/pkg/deployers" + "knative.dev/func/pkg/k8s" + fnlabels "knative.dev/func/pkg/k8s/labels" + "knative.dev/func/pkg/ocproute" +) + +var testRouteGVR = schema.GroupVersionResource{ + Group: "route.openshift.io", Version: "v1", Resource: "routes", +} + +// kedaRoute is a Route as keda's Exposer stamps one: labelled with the +// function's name AND namespace, since every keda function's Route shares the +// interceptor's namespace, and annotated with the deployer that owns it. +func kedaRoute(routeName, routeNS, fnName, fnNS string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{ + "name": routeName, + "namespace": routeNS, + "labels": map[string]any{ + fnlabels.FunctionKey: "true", + fnlabels.FunctionNameKey: fnName, + fnlabels.FunctionNamespaceKey: fnNS, + }, + "annotations": map[string]any{ + deployer.DeployerNameAnnotation: deployers.Keda, + }, + }, + }} +} + +// forbidIn makes every route verb in ns answer Forbidden, which is what a +// namespace the account cannot list answers - AND what a namespace that does +// not exist answers, since RBAC is evaluated before existence. +func forbidIn(client *dynamicfake.FakeDynamicClient, ns string) { + client.PrependReactor("*", "routes", func(a k8stesting.Action) (bool, runtime.Object, error) { + if a.GetNamespace() != ns { + return false, nil, nil + } + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: testRouteGVR.Group, Resource: testRouteGVR.Resource}, "", nil) + }) +} + +// Test_unexpose pins the whole of what a removal decides about exposure: a +// record is removed as written, no record means sweeping the candidates. +// +// Assertions are on the fake's object state and on returned refusals: what +// left the cluster and what was reported are all a caller can observe. +func Test_unexpose(t *testing.T) { + const ( + recorded = interceptorNamespaceUpstream // where the Route actually is + other = interceptorNamespaceOpenShift // the platform convention here + fnName = "f" + fnNS = "fn-keda" + ) + routeName := interceptorExposureName(fnName, fnNS) + + // Off OpenShift the candidate list shrinks to keda alone, so the + // two-candidate sweep under test here is the OpenShift one. Seeding + // mutates package state; this test must not run with t.Parallel() (see + // pkg/k8s/openshift.go). + cleanup := k8s.SetOpenShiftForTest(true, nil) + defer cleanup() + + newClient := func(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { + return dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{testRouteGVR: "RouteList"}, + objects...) + } + routesIn := func(t *testing.T, c *dynamicfake.FakeDynamicClient, ns string) int { + t.Helper() + list, err := c.Resource(testRouteGVR).Namespace(ns).List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("counting Routes in %q: %v", ns, err) + } + return len(list.Items) + } + + remover := NewRemover(false, WithRemoverExposer(ocproute.New(deployers.Keda))) + + // The Route is in the namespace the deploy recorded, which is not the + // platform convention. A removal that guessed would look in the wrong + // place and report nothing to remove; this one is told, so it removes it. + t.Run("removes the Route from the recorded namespace", func(t *testing.T) { + client := newClient(kedaRoute(routeName, recorded, fnName, fnNS)) + forbidIn(client, other) + + if err := remover.unexpose(t.Context(), client, recorded, fnName, fnNS); err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if n := routesIn(t, client, recorded); n != 0 { + t.Errorf("expected the Route to be gone, %d left in %q", n, recorded) + } + }) + + // No record sweeps, and only on delete. Deploy can stay silent because + // the next deploy reuses the same deterministic name, but delete is the + // last moment this function's identity exists and keda's Route carries no + // owner reference to collect it afterwards. A stray left by a crash + // between create and annotate is therefore hunted in every candidate + // namespace. + t.Run("no record sweeps a stray from either candidate", func(t *testing.T) { + for _, stray := range []string{recorded, other} { + t.Run(stray, func(t *testing.T) { + client := newClient(kedaRoute(routeName, stray, fnName, fnNS)) + + if err := remover.unexpose(t.Context(), client, "", fnName, fnNS); err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if n := routesIn(t, client, stray); n != 0 { + t.Errorf("expected the stray Route in %q to be swept, %d left", stray, n) + } + }) + } + }) + + // The ordinary case: nothing was ever exposed. Finding nothing is an + // answer, not a failure. + t.Run("no record and no Route anywhere is clean silence", func(t *testing.T) { + client := newClient() + + if err := remover.unexpose(t.Context(), client, "", fnName, fnNS); err != nil { + t.Fatalf("a sweep that finds nothing must not fail: %v", err) + } + }) + + // The sweep is best-effort: no record claims a Route exists, and a + // project user often cannot read the interceptor's namespace at all, so + // a denied candidate must not block deleting a function that was never + // exposed. It is skipped with a warning, and readable candidates are + // still swept. A denial on a RECORDED namespace stays fatal (below). + t.Run("a denied candidate is skipped; readable ones still swept", func(t *testing.T) { + client := newClient(kedaRoute(routeName, recorded, fnName, fnNS)) + forbidIn(client, other) + + if err := remover.unexpose(t.Context(), client, "", fnName, fnNS); err != nil { + t.Fatalf("a denied candidate must not fail the best-effort sweep: %v", err) + } + if n := routesIn(t, client, recorded); n != 0 { + t.Errorf("expected the readable candidate still swept, %d left in %q", n, recorded) + } + }) + + // The sweep widens where it looks, never what it will remove: the stamp + // and label checks still decide, so another deployer's Route survives it. + t.Run("a Route with another deployer's stamp survives the sweep", func(t *testing.T) { + stranger := kedaRoute(routeName, recorded, fnName, fnNS) + if err := unstructured.SetNestedField(stranger.Object, deployers.Kubernetes, + "metadata", "annotations", deployer.DeployerNameAnnotation); err != nil { + t.Fatal(err) + } + client := newClient(stranger) + + if err := remover.unexpose(t.Context(), client, "", fnName, fnNS); err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if n := routesIn(t, client, recorded); n != 1 { + t.Errorf("a Route stamped by another deployer must survive, found %d in %q", n, recorded) + } + }) + + // A record means a Route WAS made, so failing to take it away is a fact the + // caller is owed. It is reported before the function is dismantled, which + // is what makes a retry after the grant an ordinary delete. + t.Run("denial on the recorded namespace is fatal", func(t *testing.T) { + client := newClient(kedaRoute(routeName, recorded, fnName, fnNS)) + forbidIn(client, recorded) + + if err := remover.unexpose(t.Context(), client, recorded, fnName, fnNS); err == nil { + t.Fatal("expected a denial on a recorded Route to fail the removal") + } + // The Route's survival is not assertable here: the reactor forbids + // listing in that namespace too, which is the whole point of the case. + // The refusal IS the observable. + }) + + // The record is acted on as written. A recorded namespace that holds no + // Route is a clean answer, not an error, and nothing widens the search to + // the other candidate afterwards. + t.Run("a record naming an empty namespace removes nothing and does not widen", func(t *testing.T) { + client := newClient(kedaRoute(routeName, other, fnName, fnNS)) + + if err := remover.unexpose(t.Context(), client, recorded, fnName, fnNS); err != nil { + t.Fatalf("unexpected failure: %v", err) + } + if n := routesIn(t, client, other); n != 1 { + t.Errorf("a Route outside the recorded namespace must be left alone, found %d in %q", n, other) + } + }) +} diff --git a/pkg/lister/testing/integration_test_helper.go b/pkg/lister/testing/integration_test_helper.go index e794d0f452..d38dee86c0 100644 --- a/pkg/lister/testing/integration_test_helper.go +++ b/pkg/lister/testing/integration_test_helper.go @@ -42,7 +42,7 @@ func TestInt_List(t *testing.T, lister fn.Lister, deployer fn.Deployer, describe // Explicit opt-out: keeps this integration deploy cluster-local and // platform-deterministic under exposed-by-default; ignored entirely // by the knative deployer. - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) diff --git a/pkg/mock/deployer.go b/pkg/mock/deployer.go index 74ccd5badc..874e48bc93 100644 --- a/pkg/mock/deployer.go +++ b/pkg/mock/deployer.go @@ -39,6 +39,10 @@ func NewDeployer() *Deployer { } else { result.Deployer = f.Deploy.Deployer // redeploy with current } + // Observed exposure mirrors intent when active (same as real deployers). + if fn.ActiveExpose(f.Expose) { + result.Expose = f.Expose + } if err == nil { result.Status = fn.Deployed } diff --git a/pkg/mock/remover.go b/pkg/mock/remover.go index 69572f4fef..f8dfc6caba 100644 --- a/pkg/mock/remover.go +++ b/pkg/mock/remover.go @@ -1,6 +1,8 @@ package mock -import "context" +import ( + "context" +) type Remover struct { RemoveInvoked bool diff --git a/pkg/ocproute/route.go b/pkg/ocproute/route.go new file mode 100644 index 0000000000..881e0f4e36 --- /dev/null +++ b/pkg/ocproute/route.go @@ -0,0 +1,410 @@ +/* +Package ocproute exposes a function through an OpenShift Route. + +Routes are an OpenShift-only resource, but both binaries attach this Exposer on +every platform: on a cluster with no Route API the cost is one List that comes +back NotFound. Choosing it is the caller's job: nothing here detects the +platform. +*/ +package ocproute + +import ( + "context" + "fmt" + "maps" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8slabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/util/retry" + + "knative.dev/func/pkg/deployer" + fnlabels "knative.dev/func/pkg/k8s/labels" +) + +// routeGVR identifies the OpenShift Route resource. No typed client is used +// here: adding github.com/openshift/api as a direct dependency for a handful +// of fields is disproportionate, and this project already has a precedent for +// reading Routes through the dynamic client (see pkg/pipelines/tekton/pac/pac.go +// DetectPACOpenShiftRoute). Route's structure is also small and stable (a v1, +// GA API since OpenShift 3.x), so hand-built unstructured content carries little +// maintenance risk. +var routeGVR = schema.GroupVersionResource{ + Group: "route.openshift.io", + Version: "v1", + Resource: "routes", +} + +// admissionTimeout bounds the wait for a router to accept a Route. A router +// that has not answered in this long is not going to. +const admissionTimeout = 30 * time.Second + +// Exposer creates and removes the OpenShift Route fronting a function. +type Exposer struct { + // deployerName goes onto every Route this Exposer creates and is + // checked again before deleting one, so the Route minted for a keda + // function is never removed by the raw deployer, or the other way + // round. + deployerName string +} + +// New returns an Exposer stamping its Routes with deployerName, one of the +// names in pkg/deployers. +func New(deployerName string) *Exposer { + return &Exposer{deployerName: deployerName} +} + +// Expose creates or updates the Route for 'e' and returns the hostname a +// router admitted it at. A Route this call created that fails admission is +// removed again; a pre-existing one is kept, it may be serving. +func (x *Exposer) Expose(ctx context.Context, client dynamic.Interface, e deployer.Exposure) (string, error) { + route, err := x.generate(e) + if err != nil { + return "", fmt.Errorf("failed to generate Route: %w", err) + } + + name, created, err := x.ensure(ctx, client, e, route) + if err != nil { + return "", err + } + + host, err := waitForAdmitted(ctx, client, e.Namespace, name, admissionTimeout) + if err != nil && created { + delErr := client.Resource(routeGVR).Namespace(e.Namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if delErr != nil && !apierrors.IsNotFound(delErr) { + return "", fmt.Errorf("%w; rolling the Route back failed too: %v", err, delErr) + } + return "", fmt.Errorf("%w; the Route was rolled back", err) + } + return host, err +} + +// Unexpose deletes the Route belonging to the function named by ref, leaving +// in place any Route this Exposer did not create. +func (x *Exposer) Unexpose(ctx context.Context, client dynamic.Interface, ref deployer.ExposureRef) error { + route, err := x.find(ctx, client, ref) + if err != nil || route == nil { + return err + } + + err = client.Resource(routeGVR).Namespace(ref.Namespace).Delete(ctx, route.GetName(), metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete Route %q: %w", route.GetName(), err) + } + return nil +} + +// selector matches the Routes this Exposer creates for one function. The +// deployer stamp is deliberately not in it: that is an annotation, which no +// selector can filter on, so find() applies it afterwards through isManaged. +func selector(ref deployer.ExposureRef) string { + return k8slabels.SelectorFromSet(k8slabels.Set{ + fnlabels.FunctionKey: "true", + fnlabels.FunctionNameKey: ref.FunctionName, + fnlabels.FunctionNamespaceKey: ref.FunctionNamespace, + }).String() +} + +// find returns the Route this Exposer manages for the function named by ref, +// or nil when there is none. Lookup is by label: labels are what func stamped +// on the Route it created. A missing Route API or namespace means nothing to +// find; a lookup that fails or is denied wraps deployer.ErrExposureNotVisible, +// so denial is never read as absence. Finding two managed Routes is a refusal +// of its own, not wrapped: the cluster answered, and the answer is ambiguous. +func (x *Exposer) find(ctx context.Context, client dynamic.Interface, ref deployer.ExposureRef) (*unstructured.Unstructured, error) { + list, err := client.Resource(routeGVR).Namespace(ref.Namespace). + List(ctx, metav1.ListOptions{LabelSelector: selector(ref)}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("%w: looking for the Route of function %q: %w", + deployer.ErrExposureNotVisible, ref.FunctionName, err) + } + + var found *unstructured.Unstructured + for i := range list.Items { + if !x.isManaged(&list.Items[i]) { + continue + } + if found != nil { + return nil, fmt.Errorf( + "found %d Routes for function %q in namespace %q (%q and %q); func cannot tell which one it should manage, remove the stale one by hand", + len(list.Items), ref.FunctionName, ref.Namespace, found.GetName(), list.Items[i].GetName()) + } + found = &list.Items[i] + } + return found, nil +} + +// generate builds, but does not create, the Route exposing e's target Service. +// With no domain, spec.host stays empty: the router mints +// "-.", and naming a host here would mean +// discovering the domain first. A --domain is the exception: it is used as +// the host verbatim, and the router admits or refuses the claim. +// +// The minted host's first label gives the function's name and namespace 63 +// shared characters, fewer for keda, whose Route also carries the +// interceptor's namespace. Deliberately not checked up front: a check would +// encode OpenShift's minting template and refuse names a retemplated +// cluster accepts; the API server rejects an over-budget Route at creation +// and reports the real limit. +func (x *Exposer) generate(e deployer.Exposure) (*unstructured.Unstructured, error) { + labels, err := deployer.GenerateCommonLabels(e.Function, e.Decorator) + if err != nil { + return nil, err + } + + labels[fnlabels.FunctionNamespaceKey] = e.FunctionNamespace + + annotations := deployer.GenerateCommonAnnotations(e.Function, e.Decorator, false /* dapr n/a for routing */, x.deployerName) + + spec := map[string]any{ + "to": map[string]any{ + "kind": "Service", + "name": e.TargetService, + }, + "port": map[string]any{ + "targetPort": e.TargetPort, + }, + // Edge TLS via the router's wildcard cert - zero cert + // management; Redirect upgrades http requests to https. + "tls": map[string]any{ + "termination": "edge", + "insecureEdgeTerminationPolicy": "Redirect", + }, + } + // A custom domain is used verbatim as the host. DNS and the certificate + // are the user's: point DNS at the router, and have something like + // cert-manager inject the cert (ensure carries it over). The router + // reports a host collision at admission. + if e.Function.Domain != "" { + spec["host"] = e.Function.Domain + } + + route := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": routeGVR.GroupVersion().String(), + "kind": "Route", + "spec": spec, + }, + } + route.SetName(e.Name) + route.SetNamespace(e.Namespace) + route.SetLabels(labels) + route.SetAnnotations(annotations) + if e.Owner != nil { + route.SetOwnerReferences([]metav1.OwnerReference{*e.Owner}) + } + + return route, nil +} + +// ensure creates or updates the Route for e, returning the name it settled +// on and whether this call created the object, so Expose can undo a failed +// admission without touching a pre-existing Route. An existing managed +// Route is found by label and updated under whatever name it carries, so a +// naming-scheme change cannot strand one. An update +// reconciles func's owned fields onto the live object, so state written by +// others - a certificate controller's annotations and cert material - stays +// put. A changed domain is the one update done by replacement, since +// editing the host in place needs a grant project admins lack; an injected +// certificate dies with the old Route. A foreign Route occupying the wanted +// name is never adopted or overwritten; the create's AlreadyExists is +// reported instead. Updates retry on 409, since a controller status write +// can race. +func (x *Exposer) ensure(ctx context.Context, client dynamic.Interface, e deployer.Exposure, route *unstructured.Unstructured) (string, bool, error) { + routes := client.Resource(routeGVR).Namespace(e.Namespace) + + existing, err := x.find(ctx, client, e.Ref()) + if err != nil { + return "", false, err + } + + if existing == nil { + // create new Route + route.SetResourceVersion("") + if _, err := routes.Create(ctx, route, metav1.CreateOptions{}); err != nil { + if apierrors.IsAlreadyExists(err) { + return "", false, fmt.Errorf( + "cannot expose function %q: a Route named %q already exists in namespace %q and was not created by func; rename or remove it, or deploy with --expose=none", + e.Function.Name, e.Name, e.Namespace) + } + return "", false, fmt.Errorf("failed to create Route %q: %w", e.Name, err) + } + return e.Name, true, nil + } + + // A changed domain replaces the Route rather than editing it: an + // injected certificate names the old host, so it must die with the old + // Route instead of surviving as a mismatch on the new one. Editing + // spec.host in place is also closed off, needing "update" on + // routes/custom-host, which project admins lack by default; delete and + // create need only "create", which they have. + if existing.GetLabels()[deployer.DomainLabel] != e.Function.Domain { + if err := routes.Delete(ctx, existing.GetName(), metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return "", false, fmt.Errorf("failed to replace Route %q for a changed domain: %w", existing.GetName(), err) + } + route.SetResourceVersion("") + if _, err := routes.Create(ctx, route, metav1.CreateOptions{}); err != nil { + return "", false, fmt.Errorf("failed to recreate Route %q for domain %q: %w", e.Name, e.Function.Domain, err) + } + return e.Name, true, nil + } + + // Update by reconciling the live object rather than replacing it: third + // parties write to this Route too - cert-manager's trigger annotations + // and issued certificate, at least - and a regenerated object would wipe + // whatever func did not author. Func overlays only the fields it owns + // and leaves the rest, spec.host included, since the domain-change + // branch above already diverted any host change. The cost of merging: a + // key func once wrote but no longer generates lingers, since a merge + // cannot tell foreign from formerly ours. + name := existing.GetName() + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + current, getErr := routes.Get(ctx, name, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + merged := current.DeepCopy() + + labels := merged.GetLabels() + if labels == nil { + labels = map[string]string{} + } + maps.Copy(labels, route.GetLabels()) + merged.SetLabels(labels) + + annotations := merged.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + maps.Copy(annotations, route.GetAnnotations()) + merged.SetAnnotations(annotations) + + for _, owned := range [][]string{ + {"spec", "to"}, + {"spec", "port"}, + {"spec", "tls", "termination"}, + {"spec", "tls", "insecureEdgeTerminationPolicy"}, + // The owner must track the current Service UID: after a Service + // recreate, a Route keeping the old UID is garbage collected as + // the dead Service's dependent. Absent on keda's ownerless + // Exposure, so the copy below skips it there. + {"metadata", "ownerReferences"}, + } { + v, found, err := unstructured.NestedFieldCopy(route.Object, owned...) + if err != nil { + return err + } + if !found { + continue + } + if err := unstructured.SetNestedField(merged.Object, v, owned...); err != nil { + return err + } + } + + _, updateErr := routes.Update(ctx, merged, metav1.UpdateOptions{}) + return updateErr + }) + if err != nil { + return "", false, fmt.Errorf("failed to update Route %q: %w", name, err) + } + return name, false, nil +} + +// isManaged reports whether route was created by this Exposer - as opposed +// to a user-authored or third-party Route, which must never be touched. +// Both signals are required: a bare boson.dev/function label, or a deployer +// annotation written by some other component, alone does not prove ownership. +// The deployer name is checked here rather than in the selector because it is +// an annotation, and no label selector can filter on those. +func (x *Exposer) isManaged(route *unstructured.Unstructured) bool { + return route.GetLabels()[fnlabels.FunctionKey] == "true" && + route.GetAnnotations()[deployer.DeployerNameAnnotation] == x.deployerName +} + +// waitForAdmitted polls the Route status until any ingress entry (one per +// router shard) reports Admitted=True, returning that entry's host. Admitted +// with no hostname yet keeps polling. It fails early only when a shard +// reports Admitted=False and none reports True; entries with no verdict poll +// through to the timeout. +func waitForAdmitted(ctx context.Context, client dynamic.Interface, ns, name string, timeout time.Duration) (string, error) { + routes := client.Resource(routeGVR).Namespace(ns) + + var host string + var lastErr error + pollErr := wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + route, err := routes.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + lastErr = fmt.Errorf("failed to get Route %q: %w", name, err) + return false, nil + } + + ingresses, found, err := unstructured.NestedSlice(route.Object, "status", "ingress") + if err != nil || !found { + return false, nil + } + + // Scan every entry before deciding: verdicts are per router, and a + // rejection by one shard must not override an admission by another. + var rejection error + for _, raw := range ingresses { + ingress, ok := raw.(map[string]any) + if !ok { + continue + } + conditions, found, err := unstructured.NestedSlice(ingress, "conditions") + if err != nil || !found { + continue + } + for _, rawCond := range conditions { + cond, ok := rawCond.(map[string]any) + if !ok || cond["type"] != "Admitted" { + continue + } + status, _ := cond["status"].(string) + switch status { + case "True": + host, _, _ = unstructured.NestedString(ingress, "host") + if host == "" { + // Admitted with no hostname is not a usable answer, + // and returning it would hand the caller a bare + // "https://" and record the function as exposed. + // Another entry may still carry one; otherwise keep + // polling and say what happened if it never appears. + lastErr = fmt.Errorf( + "route %q was admitted by a router but reports no hostname", name) + continue + } + return true, nil + case "False": + if rejection == nil { + reason, _ := cond["reason"].(string) + message, _ := cond["message"].(string) + rejection = fmt.Errorf("route %q was rejected by the router: %s: %s", name, reason, message) + } + } + // Unknown or missing status: keep polling. + } + } + if rejection != nil { + lastErr = rejection + return false, rejection + } + return false, nil + }) + if pollErr != nil { + if lastErr != nil { + return "", lastErr + } + return "", fmt.Errorf("route %q was not admitted by any router within %s: %w", name, timeout, pollErr) + } + return host, nil +} diff --git a/pkg/ocproute/route_test.go b/pkg/ocproute/route_test.go new file mode 100644 index 0000000000..a6a6ae9dc9 --- /dev/null +++ b/pkg/ocproute/route_test.go @@ -0,0 +1,820 @@ +package ocproute + +import ( + "errors" + "strings" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8stesting "k8s.io/client-go/testing" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "knative.dev/func/pkg/deployer" + "knative.dev/func/pkg/deployers" + fn "knative.dev/func/pkg/functions" +) + +// errDenied stands in for the reason an API server gives with a 403. +var errDenied = errors.New("denied") + +func newFakeDynamicClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { + return dynamicfake.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{routeGVR: "RouteList"}, + objects..., + ) +} + +func testDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "f", + Namespace: "ns", + UID: types.UID("abc-123"), + }, + } +} + +// testExposure is what the raw deployer asks for: the function's own Service, +// in the function's own namespace, owned by its Deployment. +func testExposure() deployer.Exposure { + d := testDeployment() + controller := true + return deployer.Exposure{ + Function: fn.Function{Name: "f", Runtime: "go"}, + FunctionNamespace: d.Namespace, + Name: d.Name, + Namespace: d.Namespace, + TargetService: d.Name, + TargetPort: "http", + Owner: &metav1.OwnerReference{ + APIVersion: appsv1.SchemeGroupVersion.WithKind("Deployment").GroupVersion().String(), + Kind: "Deployment", + Name: d.Name, + UID: d.UID, + Controller: &controller, + }, + } +} + +func testExposer() *Exposer { + return New(deployers.Kubernetes) +} + +func Test_generate(t *testing.T) { + route, err := testExposer().generate(testExposure()) + if err != nil { + t.Fatal(err) + } + + if route.GetName() != "f" || route.GetNamespace() != "ns" { + t.Errorf("expected name/namespace f/ns, got %s/%s", route.GetName(), route.GetNamespace()) + } + toName, _, _ := unstructured.NestedString(route.Object, "spec", "to", "name") + if toName != "f" { + t.Errorf("expected spec.to.name %q, got %q", "f", toName) + } + toKind, _, _ := unstructured.NestedString(route.Object, "spec", "to", "kind") + if toKind != "Service" { + t.Errorf("expected spec.to.kind Service, got %q", toKind) + } + targetPort, _, _ := unstructured.NestedString(route.Object, "spec", "port", "targetPort") + if targetPort != "http" { + t.Errorf("expected spec.port.targetPort http, got %q", targetPort) + } + if host, found, _ := unstructured.NestedString(route.Object, "spec", "host"); found && host != "" { + t.Errorf("expected spec.host to be unset (router-minted), got %q", host) + } + termination, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "termination") + if termination != "edge" { + t.Errorf("expected spec.tls.termination edge, got %q", termination) + } + insecurePolicy, _, _ := unstructured.NestedString(route.Object, "spec", "tls", "insecureEdgeTerminationPolicy") + if insecurePolicy != "Redirect" { + t.Errorf("expected spec.tls.insecureEdgeTerminationPolicy Redirect, got %q", insecurePolicy) + } + if !testExposer().isManaged(route) { + t.Error("expected a freshly generated Route to be self-managed") + } + owners := route.GetOwnerReferences() + if len(owners) != 1 || owners[0].Name != "f" || owners[0].Kind != "Deployment" { + t.Errorf("expected a single Deployment ownerRef named f, got %+v", owners) + } +} + +// Test_generate_Domain: a custom domain becomes spec.host verbatim; without +// one the field is absent and the router mints the hostname. +func Test_generate_Domain(t *testing.T) { + e := testExposure() + e.Function.Domain = "hello.tester1.com" + route, err := testExposer().generate(e) + if err != nil { + t.Fatal(err) + } + if host, _, _ := unstructured.NestedString(route.Object, "spec", "host"); host != "hello.tester1.com" { + t.Errorf("expected spec.host %q, got %q", "hello.tester1.com", host) + } + + route, err = testExposer().generate(testExposure()) + if err != nil { + t.Fatal(err) + } + if host, found, _ := unstructured.NestedString(route.Object, "spec", "host"); found { + t.Errorf("expected no spec.host without a domain, got %q", host) + } +} + +// Test_ensure_PreservesInjectedTLS: an update must carry over TLS material a +// certificate controller wrote into spec.tls (cert-manager's openshift-routes +// plugin injects a custom domain's cert there); regenerating the spec must +// not wipe it on redeploy. +func Test_ensure_PreservesInjectedTLS(t *testing.T) { + ctx := t.Context() + x := testExposer() + + existing, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(existing.Object, "PEM-CERT", "spec", "tls", "certificate"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(existing.Object, "PEM-KEY", "spec", "tls", "key"); err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(existing) + + fresh, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + if _, _, err := x.ensure(ctx, client, testExposure(), fresh); err != nil { + t.Fatal(err) + } + + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if cert, _, _ := unstructured.NestedString(got.Object, "spec", "tls", "certificate"); cert != "PEM-CERT" { + t.Errorf("expected the injected certificate carried over, got %q", cert) + } + if key, _, _ := unstructured.NestedString(got.Object, "spec", "tls", "key"); key != "PEM-KEY" { + t.Errorf("expected the injected key carried over, got %q", key) + } + if term, _, _ := unstructured.NestedString(got.Object, "spec", "tls", "termination"); term != "edge" { + t.Errorf("expected func's own tls fields still applied, got termination %q", term) + } +} + +// Test_ensure_DomainChangeRecreates: updating spec.host in place is gated on +// routes/custom-host update permission, so a changed domain replaces the +// Route instead of updating it; a stale injected cert dies with the old host. +func Test_ensure_DomainChangeRecreates(t *testing.T) { + ctx := t.Context() + x := testExposer() + + existing, err := x.generate(testExposure()) // no domain: router-minted host + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(existing) + + e := testExposure() + e.Function.Domain = "hello.tester1.com" + fresh, err := x.generate(e) + if err != nil { + t.Fatal(err) + } + _, created, err := x.ensure(ctx, client, e, fresh) + if err != nil { + t.Fatal(err) + } + if !created { + t.Error("expected the replacement to count as created: its Route is this call's to roll back") + } + + var deleted bool + for _, a := range client.Actions() { + if a.GetVerb() == "delete" { + deleted = true + } + } + if !deleted { + t.Error("expected the old Route deleted, not updated: spec.host is immutable") + } + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if host, _, _ := unstructured.NestedString(got.Object, "spec", "host"); host != "hello.tester1.com" { + t.Errorf("expected the recreated Route to carry spec.host %q, got %q", "hello.tester1.com", host) + } +} + +// Test_generate_NoOwner covers the keda-shaped Exposure: a Route in a +// namespace the function's Deployment does not live in, which Kubernetes +// forbids owning across, so it carries no ownerReference at all. +func Test_generate_NoOwner(t *testing.T) { + e := testExposure() + e.Owner = nil + e.Namespace = "openshift-keda" + e.Name = "f-ns" + e.TargetService = "keda-add-ons-http-interceptor-proxy" + e.TargetPort = "proxy" + + route, err := testExposer().generate(e) + if err != nil { + t.Fatal(err) + } + + if owners := route.GetOwnerReferences(); len(owners) != 0 { + t.Errorf("expected no ownerRef when Owner is nil, got %+v", owners) + } + if route.GetName() != "f-ns" || route.GetNamespace() != "openshift-keda" { + t.Errorf("expected name/namespace f-ns/openshift-keda, got %s/%s", route.GetName(), route.GetNamespace()) + } + toName, _, _ := unstructured.NestedString(route.Object, "spec", "to", "name") + if toName != "keda-add-ons-http-interceptor-proxy" { + t.Errorf("expected spec.to.name to be the interceptor Service, got %q", toName) + } + targetPort, _, _ := unstructured.NestedString(route.Object, "spec", "port", "targetPort") + if targetPort != "proxy" { + t.Errorf("expected spec.port.targetPort proxy, got %q", targetPort) + } +} + +// Test_isManaged_ForeignDeployer: a Route stamped by one deployer is not +// managed by another's Exposer, so neither deletes the other's Route. +func Test_isManaged_ForeignDeployer(t *testing.T) { + route, err := New(deployers.Keda).generate(testExposure()) + if err != nil { + t.Fatal(err) + } + if New(deployers.Kubernetes).isManaged(route) { + t.Error("expected a keda-stamped Route not to be managed by the raw Exposer") + } + if !New(deployers.Keda).isManaged(route) { + t.Error("expected a keda-stamped Route to be managed by the keda Exposer") + } +} + +func Test_ensure_CreateThenUpdate(t *testing.T) { + ctx := t.Context() + x := testExposer() + client := newFakeDynamicClient() + + route, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + name, created, err := x.ensure(ctx, client, testExposure(), route) + if err != nil { + t.Fatalf("create: %v", err) + } + if name != "f" { + t.Errorf("expected the generated name f, got %q", name) + } + if !created { + t.Error("expected ensure to report it created the Route") + } + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatalf("expected Route to exist after create: %v", err) + } + toName, _, _ := unstructured.NestedString(got.Object, "spec", "to", "name") + if toName != "f" { + t.Errorf("expected spec.to.name f, got %q", toName) + } + + // Update path: regenerate (idempotent) and ensure again, no error. + route2, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + _, created, err = x.ensure(ctx, client, testExposure(), route2) + if err != nil { + t.Fatalf("update: %v", err) + } + if created { + t.Error("expected ensure to report the Route pre-existed on update") + } +} + +// Test_ensure_RefusesForeignRoute pins the half of the ownership rule that +// used to be missing. Unexpose has always declined to delete a Route func did +// not create; ensure used to fetch by name and update whatever it found, so +// the same object delete protected was silently taken over by create. Both +// paths now ask isManaged, and creation fails loudly on the name collision +// rather than overwriting somebody's object and reporting success. +func Test_ensure_RefusesForeignRoute(t *testing.T) { + ctx := t.Context() + foreign := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{ + "name": "f", + "namespace": "ns", + }, + "spec": map[string]any{ + "to": map[string]any{"kind": "Service", "name": "someone-elses-service"}, + }, + }} + client := newFakeDynamicClient(foreign) + + x := testExposer() + route, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + + if _, _, err := x.ensure(ctx, client, testExposure(), route); err == nil { + t.Fatal("expected ensure to refuse a Route func did not create, got nil error") + } + + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + toName, _, _ := unstructured.NestedString(got.Object, "spec", "to", "name") + if toName != "someone-elses-service" { + t.Errorf("the foreign Route was overwritten: spec.to.name is now %q", toName) + } +} + +// Test_find_ByLabelNotName is the point of selecting on labels: func's own +// Route is found even under a name func would not choose today, so changing +// the naming scheme cannot strand one, and a foreign Route sitting at the +// name func WOULD choose is not found at all. +func Test_find_ByLabelNotName(t *testing.T) { + ctx := t.Context() + x := testExposer() + + renamed, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + renamed.SetName("f-under-some-older-scheme") + + foreign := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + }} + + client := newFakeDynamicClient(renamed, foreign) + + found, err := x.find(ctx, client, testExposure().Ref()) + if err != nil { + t.Fatal(err) + } + if found == nil { + t.Fatal("expected func's own Route to be found under its old name") + } + if found.GetName() != "f-under-some-older-scheme" { + t.Errorf("found the wrong Route: %q", found.GetName()) + } +} + +// Test_find_IgnoresAnotherFunction: the name label alone does not separate two +// functions called the same in different namespaces, which is the whole reason +// the namespace label exists. Keda puts every function's Route in one shared +// namespace, so without it a teardown would find a stranger's. +func Test_find_IgnoresAnotherFunction(t *testing.T) { + ctx := t.Context() + x := testExposer() + + other := testExposure() + other.FunctionNamespace = "somebody-else" + other.Name = "f-somebody-else" + otherRoute, err := x.generate(other) + if err != nil { + t.Fatal(err) + } + // Both Routes land in one namespace, as keda's do. + otherRoute.SetNamespace("ns") + + client := newFakeDynamicClient(otherRoute) + + found, err := x.find(ctx, client, testExposure().Ref()) + if err != nil { + t.Fatal(err) + } + if found != nil { + t.Errorf("found another function's Route %q", found.GetName()) + } +} + +// Test_find_DeniedIsNotVisible: a denied lookup must not be reported as "no +// Route here". A denied account is told the same thing whether one exists or +// not, and callers differ on whether that should be fatal, so the two answers +// have to stay distinguishable. +func Test_find_DeniedIsNotVisible(t *testing.T) { + client := newFakeDynamicClient() + client.PrependReactor("list", "routes", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: routeGVR.Group, Resource: routeGVR.Resource}, "", errDenied) + }) + + route, err := testExposer().find(t.Context(), client, testExposure().Ref()) + if err == nil { + t.Fatal("expected a denied lookup to report an error, got nil") + } + if route != nil { + t.Error("expected no Route alongside the error") + } + if !errors.Is(err, deployer.ErrExposureNotVisible) { + t.Errorf("expected ErrExposureNotVisible so callers can tell denial from absence, got %v", err) + } +} + +// Test_Unexpose_DeniedPropagates: Unexpose must not report "nothing to remove" +// when it simply could not look. Reporting removed=false with no error would +// let a caller conclude the Route is gone. +func Test_Unexpose_DeniedPropagates(t *testing.T) { + client := newFakeDynamicClient() + client.PrependReactor("list", "routes", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: routeGVR.Group, Resource: routeGVR.Resource}, "", errDenied) + }) + + err := testExposer().Unexpose(t.Context(), client, testExposure().Ref()) + if err == nil { + t.Fatal("expected a denied Unexpose to report an error") + } + if !errors.Is(err, deployer.ErrExposureNotVisible) { + t.Errorf("expected ErrExposureNotVisible, got %v", err) + } +} + +func Test_Unexpose(t *testing.T) { + ctx := t.Context() + + t.Run("nothing to remove: no-op", func(t *testing.T) { + client := newFakeDynamicClient() + if err := testExposer().Unexpose(ctx, client, testExposure().Ref()); err != nil { + t.Errorf("expected nil for an absent Route, got %v", err) + } + }) + + t.Run("managed: deleted", func(t *testing.T) { + x := testExposer() + route, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(route) + + if err := x.Unexpose(ctx, client, testExposure().Ref()); err != nil { + t.Fatal(err) + } + if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err == nil { + t.Error("expected Route to be gone after removal") + } + }) + + t.Run("foreign Route at the same name: kept", func(t *testing.T) { + foreign := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + }} + client := newFakeDynamicClient(foreign) + + if err := testExposer().Unexpose(ctx, client, testExposure().Ref()); err != nil { + t.Fatalf("expected nil for a foreign Route, got %v", err) + } + if _, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}); err != nil { + t.Error("expected the foreign Route to be left in place") + } + }) +} + +func Test_waitForAdmitted(t *testing.T) { + ctx := t.Context() + + t.Run("admitted: returns host", func(t *testing.T) { + client := newFakeDynamicClient(admittedRoute("f-ns.apps.example.com")) + host, err := waitForAdmitted(ctx, client, "ns", "f", time.Second) + if err != nil { + t.Fatal(err) + } + if host != "f-ns.apps.example.com" { + t.Errorf("expected host f-ns.apps.example.com, got %q", host) + } + }) + + t.Run("rejected: fails fast with reason", func(t *testing.T) { + rejected := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": "", + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "False", "reason": "HostAlreadyClaimed", "message": "taken"}, + }, + }, + }, + }, + }} + client := newFakeDynamicClient(rejected) + _, err := waitForAdmitted(ctx, client, "ns", "f", 5*time.Second) + if err == nil { + t.Fatal("expected an error for a rejected Route") + } + }) + + t.Run("rejected by one shard, admitted by another: admission wins", func(t *testing.T) { + // The rejection entry comes FIRST: list order must not decide. + mixed := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": "", + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "False", "reason": "HostAlreadyClaimed", "message": "taken"}, + }, + }, + map[string]any{ + "host": "f-ns.apps.example.com", + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "True"}, + }, + }, + }, + }, + }} + client := newFakeDynamicClient(mixed) + host, err := waitForAdmitted(ctx, client, "ns", "f", time.Second) + if err != nil { + t.Fatalf("a rejection by one shard must not override an admission by another: %v", err) + } + if host != "f-ns.apps.example.com" { + t.Errorf("expected the admitting shard's host, got %q", host) + } + }) + + t.Run("never admitted: times out cleanly", func(t *testing.T) { + route, err := testExposer().generate(testExposure()) + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(route) + + _, err = waitForAdmitted(ctx, client, "ns", "f", 100*time.Millisecond) + if err == nil { + t.Fatal("expected a timeout error when no router ever admits the route") + } + }) +} + +// Test_waitForAdmitted_EmptyHost: a router that sets Admitted=True without a +// hostname has not given a usable answer. Returning it would hand the caller +// a bare "https://" and record the function as externally exposed, so the +// wait must not succeed on it. +func Test_waitForAdmitted_EmptyHost(t *testing.T) { + route := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{map[string]any{ + "host": "", + "conditions": []any{map[string]any{ + "type": "Admitted", "status": "True", + }}, + }}, + }, + }} + + host, err := waitForAdmitted(t.Context(), newFakeDynamicClient(route), "ns", "f", 2*time.Second) + if err == nil { + t.Fatalf("expected an error for an admitted Route with no hostname, got host %q", host) + } + if host != "" { + t.Errorf("expected no host alongside the error, got %q", host) + } + if !strings.Contains(err.Error(), "no hostname") { + t.Errorf("expected the error to name the cause, got %v", err) + } +} + +// A Route Expose itself created that fails admission is deleted again. +func Test_Expose_RollsBackUnadmittedRoute(t *testing.T) { + x := testExposer() + client := newFakeDynamicClient() + // no router in the fake: serve a rejected status to every poll + client.PrependReactor("get", "routes", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, rejectedRoute(), nil + }) + + if _, err := x.Expose(t.Context(), client, testExposure()); err == nil { + t.Fatal("expected Expose to fail when no router admits the Route") + } + + var deleted bool + for _, a := range client.Actions() { + if a.GetVerb() == "delete" { + deleted = true + } + } + if !deleted { + t.Error("expected the just-created Route to be removed after the admission failure") + } +} + +// A pre-existing Route survives an admission failure: it may be serving. +func Test_Expose_KeepsPreexistingRouteOnAdmissionFailure(t *testing.T) { + x := testExposer() + existing, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(existing) + client.PrependReactor("get", "routes", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, rejectedRoute(), nil + }) + + if _, err := x.Expose(t.Context(), client, testExposure()); err == nil { + t.Fatal("expected Expose to fail when no router admits the Route") + } + + for _, a := range client.Actions() { + if a.GetVerb() == "delete" { + t.Error("a pre-existing Route was deleted on admission failure") + } + } +} + +func rejectedRoute() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": "", + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "False", "reason": "HostAlreadyClaimed", "message": "taken"}, + }, + }, + }, + }, + }} +} + +func admittedRoute(host string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "route.openshift.io/v1", + "kind": "Route", + "metadata": map[string]any{"name": "f", "namespace": "ns"}, + "status": map[string]any{ + "ingress": []any{ + map[string]any{ + "host": host, + "conditions": []any{ + map[string]any{"type": "Admitted", "status": "True"}, + }, + }, + }, + }, + }} +} + +// Test_ensure_UpdatePreservesForeignStateAndReassertsOwned: an update +// reconciles the live Route instead of replacing it. Cert-manager's trigger +// annotations are the state whose loss hurts most: the issued certificate +// survives a wipe, so everything looks fine until renewal never happens. +// The other direction matters equally: fields func owns are reasserted when +// something drifted them. +func Test_ensure_UpdatePreservesForeignStateAndReassertsOwned(t *testing.T) { + ctx := t.Context() + x := testExposer() + + existing, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + // Foreign state: the management contract and a spec.tls field func does + // not know about. + ann := existing.GetAnnotations() + ann["cert-manager.io/issuer-name"] = "letsencrypt-prod" + existing.SetAnnotations(ann) + labels := existing.GetLabels() + labels["team"] = "a" + existing.SetLabels(labels) + if err := unstructured.SetNestedField(existing.Object, "PEM-CERT", "spec", "tls", "certificate"); err != nil { + t.Fatal(err) + } + // Drift on owned fields: something changed what func asserts. + if err := unstructured.SetNestedField(existing.Object, "passthrough", "spec", "tls", "termination"); err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(existing) + + fresh, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + if _, _, err := x.ensure(ctx, client, testExposure(), fresh); err != nil { + t.Fatal(err) + } + + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if v := got.GetAnnotations()["cert-manager.io/issuer-name"]; v != "letsencrypt-prod" { + t.Errorf("expected the cert-manager annotation to survive the update, got %q", v) + } + if v := got.GetLabels()["team"]; v != "a" { + t.Errorf("expected the foreign label to survive the update, got %q", v) + } + if cert, _, _ := unstructured.NestedString(got.Object, "spec", "tls", "certificate"); cert != "PEM-CERT" { + t.Errorf("expected the issued certificate to survive the update, got %q", cert) + } + if term, _, _ := unstructured.NestedString(got.Object, "spec", "tls", "termination"); term != "edge" { + t.Errorf("expected func to reassert its termination, got %q", term) + } + if v := got.GetAnnotations()[deployer.DeployerNameAnnotation]; v != deployers.Kubernetes { + t.Errorf("expected func's own annotations reasserted, got %q", v) + } +} + +// Test_ensure_UpdateReassertsOwner: the update must point the Route's owner +// reference at the current owner. A Route surviving a Service recreate keeps +// the dead UID otherwise; the update then reports success and garbage +// collection deletes the Route as the dead Service's dependent. Keda's +// exposure carries no owner, and its ownerless Route must stay that way. +func Test_ensure_UpdateReassertsOwner(t *testing.T) { + ctx := t.Context() + x := testExposer() + + t.Run("raw: stale owner UID replaced with the current one", func(t *testing.T) { + stale, err := x.generate(testExposure()) + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(stale) + + e := testExposure() + e.Owner.UID = "recreated-uid" + fresh, err := x.generate(e) + if err != nil { + t.Fatal(err) + } + if _, _, err := x.ensure(ctx, client, e, fresh); err != nil { + t.Fatal(err) + } + + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs := got.GetOwnerReferences() + if len(refs) != 1 || string(refs[0].UID) != "recreated-uid" { + t.Errorf("expected the owner reference reasserted to the current UID, got %+v", refs) + } + }) + + t.Run("keda: ownerless Route stays ownerless", func(t *testing.T) { + e := testExposure() + e.Owner = nil + existing, err := x.generate(e) + if err != nil { + t.Fatal(err) + } + client := newFakeDynamicClient(existing) + + fresh, err := x.generate(e) + if err != nil { + t.Fatal(err) + } + if _, _, err := x.ensure(ctx, client, e, fresh); err != nil { + t.Fatal(err) + } + + got, err := client.Resource(routeGVR).Namespace("ns").Get(ctx, "f", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if refs := got.GetOwnerReferences(); len(refs) != 0 { + t.Errorf("expected no owner reference on keda's Route, got %+v", refs) + } + }) +} diff --git a/pkg/pipelines/tekton/pipelines_provider.go b/pkg/pipelines/tekton/pipelines_provider.go index 4758380e03..ef937b29a8 100644 --- a/pkg/pipelines/tekton/pipelines_provider.go +++ b/pkg/pipelines/tekton/pipelines_provider.go @@ -113,7 +113,8 @@ func NewPipelinesProvider(opts ...Opt) *PipelinesProvider { // definition, sending it to the cluster to be run via Tekton. // Progress is by default piped to stdtout. // Returned is the final url, and the input Function with the final results of the run populated -// (f.Deploy.Image and f.Deploy.Namespace) or an error. +// (f.Deploy.Image, f.Deploy.Namespace, f.Deploy.Deployer and f.Deploy.Expose) +// or an error. func (pp *PipelinesProvider) Run(ctx context.Context, f fn.Function) (string, fn.Function, error) { var err error @@ -159,6 +160,12 @@ func (pp *PipelinesProvider) Run(ctx context.Context, f fn.Function) (string, fn } f.Deploy.Deployer = deployer + // Applied exposure (f.Deploy.Expose) is deliberately NOT derived from intent + // here: the pipeline runs a published func-util image this build does not + // compile, so what it did with expose is established by looking. Recorded + // after the run from the describer, which reads the annotation the on-cluster + // deployer wrote at exposure time. + // Client for the given namespace client, err := NewTektonClient(namespace) if err != nil { @@ -274,6 +281,7 @@ func (pp *PipelinesProvider) Run(ctx context.Context, f fn.Function) (string, fn if err != nil { return "", f, fmt.Errorf("problem in retrieving status of deployed function: %v", err) } + f.Deploy.Expose = obj.Expose verb := "deployed" if obj.Generation != 1 { diff --git a/pkg/remover/testing/integration_test_helper.go b/pkg/remover/testing/integration_test_helper.go index ce0634048c..9783cbf64f 100644 --- a/pkg/remover/testing/integration_test_helper.go +++ b/pkg/remover/testing/integration_test_helper.go @@ -39,7 +39,7 @@ func TestInt_Remove(t *testing.T, remover fn.Remover, deployer fn.Deployer, desc Runtime: "go", Namespace: ns, Registry: Registry(), - Deploy: fn.DeploySpec{Expose: "none"}, + Expose: fn.ExposeNone, }) if err != nil { t.Fatal(err) diff --git a/schema/func_yaml-schema.json b/schema/func_yaml-schema.json index 79e7af6b3d..f931de1e40 100644 --- a/schema/func_yaml-schema.json +++ b/schema/func_yaml-schema.json @@ -132,8 +132,13 @@ "description": "ManagementDisabled disables automatic creation/update of a Function CR\nfor operator management after deploy. The zero value (false) means\nthe function is managed by default when the func-operator is installed." }, "expose": { + "enum": [ + "route", + "none", + "" + ], "type": "string", - "description": "Expose controls external access for the raw and keda deployers (the\nknative deployer manages its own exposure and ignores it). Optional.\nValues: \"route\" (create an OpenShift Route; OpenShift clusters only -\na hard error elsewhere), \"none\" (cluster-local only, explicit\nopt-out). Defaults to \"route\" behavior on OpenShift - a deployed\nfunction being externally reachable is the expected outcome - and to\ncluster-local on any other cluster, since a Route is an\nOpenShift-only mechanism and the unset default must not impose a\nplatform requirement." + "description": "Expose records the external exposure mode CURRENTLY applied on the\ncluster for raw/keda (observed state). Written after successful deploy,\ncleared on undeploy alongside Namespace and Deployer. Empty means\ncluster-local (or never exposed). User intent lives on Function.Expose." } }, "additionalProperties": false, @@ -215,6 +220,15 @@ "type": "string", "description": "Deployer with which to deploy the Function: the requested (intended)\ndeployer. This is the user's choice and persists across undeploy.\nThe deployer a Function is CURRENTLY deployed with is recorded separately\nin .Deploy.Deployer, which is cleared on undeploy." }, + "expose": { + "enum": [ + "route", + "none", + "" + ], + "type": "string", + "description": "Expose is the requested (intended) external exposure mode for the raw\nand keda deployers (knative manages its own networking and ignores it).\nValues: \"route\" (OpenShift Route; OpenShift only), \"none\" (cluster-local).\nEmpty means cluster-local. Persists across undeploy like Deployer.\nThe mode CURRENTLY applied on the cluster is recorded separately in\n.Deploy.Expose, which is cleared on undeploy." + }, "created": { "type": "string", "description": "Created time is the moment that creation was successfully completed\naccording to the client which is in charge of what constitutes being\nfully \"Created\" (aka initialized)",