diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..c0455e60995 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @Workable/systems diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc1f188733d..b893123f475 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,19 +1,18 @@ -name: CI +name: External Secrets Workable CI on: push: + tags: + - workable-* + pull_request: branches: - - main - pull_request: {} + - workable-* env: # Common versions GOLANGCI_VERSION: 'v2.4.0' KUBERNETES_VERSION: '1.33.x' - # Sonar - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - permissions: contents: read @@ -26,9 +25,6 @@ jobs: outputs: noop: ${{ steps.noop.outputs.should_skip }} steps: - - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - name: Detect No-op Changes id: noop uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 @@ -47,9 +43,6 @@ jobs: if: needs.detect-noop.outputs.noop != 'true' && github.ref != 'refs/heads/main' steps: - - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -70,32 +63,12 @@ jobs: skip-pkg-cache: true skip-build-cache: true - license-check: - permissions: - contents: read # for actions/checkout to fetch code - pull-requests: read # for golangci/golangci-lint-action to fetch pull requests - runs-on: ubuntu-latest - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' && github.ref != 'refs/heads/main' - - steps: - - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - name: Check License Headers - uses: apache/skywalking-eyes/header@5c5b974209f0de5d905f37deb69369068ebfc15c # v0.7.0 - check-diff: runs-on: ubuntu-latest needs: detect-noop if: needs.detect-noop.outputs.noop != 'true' && github.ref != 'refs/heads/main' steps: - - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - uses: hashicorp/setup-terraform@c529327889820530c60b4ce5bbc8d6099e166666 # v3 @@ -124,9 +97,6 @@ jobs: if: needs.detect-noop.outputs.noop != 'true' steps: - - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 - with: - egress-policy: audit - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -154,6 +124,7 @@ jobs: make test - name: Publish Unit Test Coverage + if: false uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -162,39 +133,75 @@ jobs: file: ./cover.out publish-artifacts: - needs: detect-noop - if: needs.detect-noop.outputs.noop != 'true' - uses: ./.github/workflows/publish.yml + needs: [lint, check-diff, unit-tests] + if: ${{ needs.detect-noop.outputs.noop != 'true' && startsWith(github.ref, 'refs/tags/workable-') }} permissions: - contents: read #actions/checkout - packages: write #for publishing artifacts - id-token: write #for keyless sign - strategy: - matrix: - include: - - dockerfile: "Dockerfile" - build-args: "CGO_ENABLED=0" - build-arch: "amd64 arm64 s390x ppc64le" - build-platform: "linux/amd64,linux/arm64,linux/s390x,linux/ppc64le" - tag-suffix: "" # distroless - - dockerfile: "Dockerfile.ubi" - build-args: "CGO_ENABLED=0" - build-arch: "amd64 arm64 ppc64le" - build-platform: "linux/amd64,linux/arm64,linux/ppc64le" - tag-suffix: "-ubi" - - dockerfile: "Dockerfile.ubi" - build-args: "CGO_ENABLED=0 GOEXPERIMENT=boringcrypto" - build-arch: "amd64 ppc64le" - build-platform: "linux/amd64,linux/ppc64le" - tag-suffix: "-ubi-boringssl" - with: - dockerfile: ${{ matrix.dockerfile }} - tag-suffix: ${{ matrix.tag-suffix }} - image-name: ghcr.io/${{ github.repository }} - build-platform: ${{ matrix.build-platform }} - build-args: ${{ matrix.build-args }} - build-arch: ${{ matrix.build-arch }} - ref: ${{ github.ref }} - secrets: - IS_FORK: ${{ secrets.GHCR_USERNAME }} # this is just a secret to verify it is a fork or not, no other utility + id-token: write + contents: read + runs-on: ubuntu-latest + environment: Workable + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Get image tag + id: container-info + run: | + echo "image-tag=${GITHUB_REF#refs/tags/workable-}" >> $GITHUB_OUTPUT + + - name: Build image + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.1 + with: + context: . + file: Dockerfile.standalone + push: false + tags: Workable/external-secrets:${{ steps.container-info.outputs.image-tag }} + provenance: false + + # DISTRIBUTION OF SRE IMAGE + - name: Login to sre registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.SRE_GCR_SA }} + + - name: Push image to sre registry + env: + REGISTRY: us-docker.pkg.dev/sre-artifacts-20e4/gcr.io + run: | + docker tag Workable/external-secrets:${{ steps.container-info.outputs.image-tag }} \ + ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} + docker push ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} + + # DISTRIBUTION OF STAGING IMAGE + - name: Login to staging registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.STAGING_GCR_SA }} + + - name: Push image to staging registry + env: + REGISTRY: us-docker.pkg.dev/staging-artifacts-786a/gcr.io + run: | + docker tag Workable/external-secrets:${{ steps.container-info.outputs.image-tag }} \ + ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} + docker push ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} + + # DISTRIBUTION OF PRODUCTION IMAGE + - name: Login to production registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: us-docker.pkg.dev + username: _json_key + password: ${{ secrets.PRODUCTION_GCR_SA }} + - name: Push image to production registry + env: + REGISTRY: us-docker.pkg.dev/production-artifacts-0b0d/gcr.io + run: | + docker tag Workable/external-secrets:${{ steps.container-info.outputs.image-tag }} \ + ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} + docker push ${{ env.REGISTRY }}/external-secrets:${{ steps.container-info.outputs.image-tag }} diff --git a/.licenserc.yaml b/.licenserc.yaml index f4f114c549d..ea418264984 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -21,5 +21,7 @@ header: - "**/*.go" paths-ignore: - "pkg/template/v2/pem_chain.go" + - "pkg/provider/vault/symlink.go" + - "pkg/provider/vault/symlink_test.go" comment: on-failure diff --git a/Makefile b/Makefile index 2d8b6ff8cc8..bcb34f2e63f 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ FAIL = (echo ${TIME} ${RED}[FAIL]${CNone} && false) # ==================================================================================== # Conformance -reviewable: generate docs manifests helm.generate helm.schema.update helm.docs lint license.check helm.test.update test.crds.update tf.fmt ## Ensure a PR is ready for review. +reviewable: generate docs manifests helm.generate helm.schema.update helm.docs lint license.check helm.test.update test.crds.update ## Ensure a PR is ready for review. @go mod tidy @cd e2e/ && go mod tidy @@ -321,36 +321,6 @@ docker.promote: ## Promote the docker image to the registry $(DOCKER) manifest push $(IMAGE_NAME):$(RELEASE_TAG) @$(OK) $(DOCKER) push $(RELEASE_TAG) \ -# ==================================================================================== -# Terraform - -define run_terraform - @cd $(TF_DIR)/$1/infrastructure && \ - terraform init && \ - $2 && \ - cd ../kubernetes && \ - terraform init && \ - $3 -endef - -tf.plan.%: - $(call run_terraform,$*,terraform plan,terraform plan) - -tf.apply.%: - $(call run_terraform,$*,terraform apply -auto-approve,terraform apply -auto-approve) - -tf.destroy.%: - @cd $(TF_DIR)/$*/kubernetes && \ - terraform init && \ - terraform destroy -auto-approve && \ - cd ../infrastructure && \ - terraform init && \ - terraform destroy -auto-approve - -tf.fmt: - @cd $(TF_DIR) && \ - terraform fmt -recursive - # ==================================================================================== # Help diff --git a/pkg/provider/vault/client_get.go b/pkg/provider/vault/client_get.go index c423ddf5e2b..d14bb290191 100644 --- a/pkg/provider/vault/client_get.go +++ b/pkg/provider/vault/client_get.go @@ -72,6 +72,12 @@ func (c *client) GetSecret(ctx context.Context, ref esv1.ExternalSecretDataRemot } } + // Replace symlinks + data, err = c.resolveSymlink(ctx, data) + if err != nil { + return nil, err + } + return getSecretValue(data, ref.Property) } diff --git a/pkg/provider/vault/symlink.go b/pkg/provider/vault/symlink.go new file mode 100644 index 00000000000..85bb7d580b1 --- /dev/null +++ b/pkg/provider/vault/symlink.go @@ -0,0 +1,76 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "context" + "regexp" + "strings" +) + +const ( + // Symlink must start with the prefix vault:// to be valid. + vaultSymlink = `vault://` + // Path can be anything and matches the last # as a separator of key. + vaultSymlinkPath = `(?P.*)#` + // Key can be any alphanumeric character and stops with the first @. + vaultSymlinkSecret = `(?P\w+)` + // Version is optional and will match any number after @. + vaultSymlinkVersion = `(@(?P\d+)?)?` + vaultSymlinkPattern = vaultSymlink + vaultSymlinkPath + vaultSymlinkSecret + vaultSymlinkVersion +) + +// isSymlink tests if secret can be converted to string and if it matches the symlink pattern. +func isSymlink(secret any) bool { + if s, ok := secret.(string); ok { + return strings.HasPrefix(s, vaultSymlink) + } + + return false +} + +// extractSymlinkParts extract capture group items of regex to a map. +func extractSymlinkParts(secret any) (paramsMap map[string]string) { + r := regexp.MustCompile(vaultSymlinkPattern) + match := r.FindStringSubmatch(secret.(string)) + paramsMap = make(map[string]string) + + for i, name := range r.SubexpNames() { + if i > 0 && i <= len(match) { + paramsMap[name] = match[i] + } + } + + return paramsMap +} + +// resolveSymlink test if the data passed has symlinks and resolve them. +func (c *client) resolveSymlink(ctx context.Context, data map[string]any) (map[string]any, error) { + for key, secret := range data { + for isSymlink(secret) { + symlink := extractSymlinkParts(secret) + + s, err := c.readSecret(ctx, symlink["Path"], symlink["Version"]) + if err != nil { + return nil, err + } + + secret = s[symlink["Secret"]] + data[key] = secret + } + } + + return data, nil +} diff --git a/pkg/provider/vault/symlink_test.go b/pkg/provider/vault/symlink_test.go new file mode 100644 index 00000000000..782204815af --- /dev/null +++ b/pkg/provider/vault/symlink_test.go @@ -0,0 +1,81 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestIsSymlink(t *testing.T) { + cases := map[string]struct { + got string + want bool + }{ + "ShouldResolveSymlink": { + got: "vault://test", + want: true, + }, + "ShouldNotResolveSymlink": { + got: "test", + want: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if diff := cmp.Diff(tc.want, isSymlink(tc.got), EquateErrors()); diff != "" { + t.Errorf("\nvault.isSymlink(...): -want error, +got error:\n%s", diff) + } + }) + } +} + +func TestExtractSymlinkParts(t *testing.T) { + cases := map[string]struct { + pattern string + expectedPath string + expectedSecret string + expectedVersion string + }{ + "ShouldExtractPathAndSecret": { + pattern: "vault://test#KEY", + expectedPath: "test", + expectedSecret: "KEY", + expectedVersion: "", + }, + "ShouldExtractPathAndSecretAndVersion": { + pattern: "vault://test#KEY@21", + expectedPath: "test", + expectedSecret: "KEY", + expectedVersion: "21", + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if diff := cmp.Diff(tc.expectedPath, extractSymlinkParts(tc.pattern)["Path"], EquateErrors()); diff != "" { + t.Errorf("\nvault.extractSymlinkParts(...): -want error, +got error:\n%s", diff) + } + if diff := cmp.Diff(tc.expectedSecret, extractSymlinkParts(tc.pattern)["Secret"], EquateErrors()); diff != "" { + t.Errorf("\nvault.extractSymlinkParts(...): -want error, +got error:\n%s", diff) + } + if diff := cmp.Diff(tc.expectedVersion, extractSymlinkParts(tc.pattern)["Version"], EquateErrors()); diff != "" { + t.Errorf("\nvault.extractSymlinkParts(...): -want error, +got error:\n%s", diff) + } + }) + } +}