diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index d8a48677523..3cbd1b4312f 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" collectorversion "github.com/prometheus/client_golang/prometheus/collectors/version" "github.com/prometheus/common/version" - _ "go.uber.org/automaxprocs" + "go.uber.org/automaxprocs/maxprocs" "gopkg.in/yaml.v2" "github.com/cortexproject/cortex/pkg/cortex" @@ -170,6 +170,9 @@ func main() { } util_log.InitLogger(&cfg.Server) + if _, err := maxprocs.Set(maxprocs.Logger(util_log.AutomaxprocsLogger(util_log.Logger))); err != nil { + level.Warn(util_log.Logger).Log("msg", "failed to set GOMAXPROCS from CPU quota", "err", err) + } util.InitEvents(eventSampleRate) ctx, cancelFn := context.WithCancel(context.Background()) diff --git a/pkg/util/log/external.go b/pkg/util/log/external.go new file mode 100644 index 00000000000..a2f68ef4d78 --- /dev/null +++ b/pkg/util/log/external.go @@ -0,0 +1,159 @@ +package log + +import ( + "fmt" + "os" + "strconv" + "strings" + + kitlog "github.com/go-kit/log" + "github.com/go-kit/log/level" + "google.golang.org/grpc/grpclog" +) + +// InitExternalLoggers configures package-level loggers from dependencies that +// do not take Cortex's logger through normal constructors. +func InitExternalLoggers() { + grpclog.SetLoggerV2(NewGRPCLogger(Logger)) +} + +// AutomaxprocsLogger adapts the printf-style logger used by automaxprocs to +// Cortex's configured go-kit logger. +func AutomaxprocsLogger(logger kitlog.Logger) func(string, ...interface{}) { + return func(format string, args ...interface{}) { + level.Info(logger).Log("msg", fmt.Sprintf(format, args...)) + } +} + +// NewGRPCLogger adapts gRPC's package-level logger to Cortex's configured +// go-kit logger so gRPC transport logs use the selected Cortex log format. +func NewGRPCLogger(logger kitlog.Logger) grpclog.LoggerV2 { + return &grpcLogger{ + logger: logger, + severity: grpcSeverityFromEnv(), + verbose: grpcVerbosityFromEnv(), + } +} + +type grpcSeverity int + +const ( + grpcSeverityInfo grpcSeverity = iota + grpcSeverityWarning + grpcSeverityError +) + +type grpcLogger struct { + logger kitlog.Logger + severity grpcSeverity + verbose int +} + +func (l *grpcLogger) Info(args ...any) { + l.log(grpcSeverityInfo, fmt.Sprint(args...)) +} + +func (l *grpcLogger) Infoln(args ...any) { + l.log(grpcSeverityInfo, trimPrintln(fmt.Sprintln(args...))) +} + +func (l *grpcLogger) Infof(format string, args ...any) { + l.log(grpcSeverityInfo, fmt.Sprintf(format, args...)) +} + +func (l *grpcLogger) Warning(args ...any) { + l.log(grpcSeverityWarning, fmt.Sprint(args...)) +} + +func (l *grpcLogger) Warningln(args ...any) { + l.log(grpcSeverityWarning, trimPrintln(fmt.Sprintln(args...))) +} + +func (l *grpcLogger) Warningf(format string, args ...any) { + l.log(grpcSeverityWarning, fmt.Sprintf(format, args...)) +} + +func (l *grpcLogger) Error(args ...any) { + l.log(grpcSeverityError, fmt.Sprint(args...)) +} + +func (l *grpcLogger) Errorln(args ...any) { + l.log(grpcSeverityError, trimPrintln(fmt.Sprintln(args...))) +} + +func (l *grpcLogger) Errorf(format string, args ...any) { + l.log(grpcSeverityError, fmt.Sprintf(format, args...)) +} + +func (l *grpcLogger) Fatal(args ...any) { + l.log(grpcSeverityError, fmt.Sprint(args...)) + os.Exit(1) +} + +func (l *grpcLogger) Fatalln(args ...any) { + l.log(grpcSeverityError, trimPrintln(fmt.Sprintln(args...))) + os.Exit(1) +} + +func (l *grpcLogger) Fatalf(format string, args ...any) { + l.log(grpcSeverityError, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +func (l *grpcLogger) V(level int) bool { + return level <= l.verbose +} + +func (l *grpcLogger) InfoDepth(_ int, args ...any) { + l.Infoln(args...) +} + +func (l *grpcLogger) WarningDepth(_ int, args ...any) { + l.Warningln(args...) +} + +func (l *grpcLogger) ErrorDepth(_ int, args ...any) { + l.Errorln(args...) +} + +func (l *grpcLogger) FatalDepth(_ int, args ...any) { + l.Fatalln(args...) +} + +func (l *grpcLogger) log(severity grpcSeverity, msg string) { + if severity < l.severity { + return + } + + switch severity { + case grpcSeverityInfo: + level.Info(l.logger).Log("msg", msg) + case grpcSeverityWarning: + level.Warn(l.logger).Log("msg", msg) + default: + level.Error(l.logger).Log("msg", msg) + } +} + +func grpcSeverityFromEnv() grpcSeverity { + switch strings.ToLower(os.Getenv("GRPC_GO_LOG_SEVERITY_LEVEL")) { + case "info": + return grpcSeverityInfo + case "warning": + return grpcSeverityWarning + default: + return grpcSeverityError + } +} + +func grpcVerbosityFromEnv() int { + verbosity, err := strconv.Atoi(os.Getenv("GRPC_GO_LOG_VERBOSITY_LEVEL")) + if err != nil { + return 0 + } + return verbosity +} + +func trimPrintln(msg string) string { + return strings.TrimSuffix(msg, "\n") +} diff --git a/pkg/util/log/external_test.go b/pkg/util/log/external_test.go new file mode 100644 index 00000000000..42080e14364 --- /dev/null +++ b/pkg/util/log/external_test.go @@ -0,0 +1,58 @@ +package log + +import ( + "bytes" + "encoding/json" + "testing" + + kitlog "github.com/go-kit/log" + "github.com/stretchr/testify/require" + "github.com/weaveworks/common/logging" +) + +func TestGRPCLoggerUsesConfiguredLogger(t *testing.T) { + t.Setenv("GRPC_GO_LOG_SEVERITY_LEVEL", "") + t.Setenv("GRPC_GO_LOG_VERBOSITY_LEVEL", "") + + logger, logs := testJSONLogger(t) + grpcLogger := NewGRPCLogger(logger) + + grpcLogger.Info("not logged by default") + grpcLogger.Error("transport failure") + + entry := readSingleJSONLogEntry(t, logs) + require.Equal(t, "error", entry["level"]) + require.Equal(t, "transport failure", entry["msg"]) + require.NotContains(t, logs.String(), "not logged by default") +} + +func TestAutomaxprocsLoggerUsesConfiguredLogger(t *testing.T) { + logger, logs := testJSONLogger(t) + + AutomaxprocsLogger(logger)("maxprocs: Leaving GOMAXPROCS=%v: CPU quota undefined", 4) + + entry := readSingleJSONLogEntry(t, logs) + require.Equal(t, "info", entry["level"]) + require.Equal(t, "maxprocs: Leaving GOMAXPROCS=4: CPU quota undefined", entry["msg"]) +} + +func testJSONLogger(t *testing.T) (kitlog.Logger, *bytes.Buffer) { + t.Helper() + + var logLevel logging.Level + require.NoError(t, logLevel.Set("debug")) + + logs := &bytes.Buffer{} + return newPrometheusLoggerFrom(kitlog.NewJSONLogger(logs), logLevel), logs +} + +func readSingleJSONLogEntry(t *testing.T, logs *bytes.Buffer) map[string]interface{} { + t.Helper() + + lines := bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n")) + require.Len(t, lines, 1) + + var entry map[string]interface{} + require.NoError(t, json.Unmarshal(lines[0], &entry)) + return entry +} diff --git a/pkg/util/log/log.go b/pkg/util/log/log.go index 9b58c72c0a9..6d04f60d8d9 100644 --- a/pkg/util/log/log.go +++ b/pkg/util/log/log.go @@ -44,6 +44,7 @@ func InitLogger(cfg *server.Config) { // when use util_log.Logger, skip 6 stack frames. Logger = newPrometheusLoggerFrom(l, cfg.LogLevel, "caller", log.Caller(6)) SLogger = GoKitLogToSlog(Logger) + InitExternalLoggers() // cfg.Log wraps log function, skip 7 stack frames to get caller information. // this works in go 1.12, but doesn't work in versions earlier. diff --git a/vendor/go.uber.org/automaxprocs/.codecov.yml b/vendor/go.uber.org/automaxprocs/.codecov.yml deleted file mode 100644 index 9a2ed4a9969..00000000000 --- a/vendor/go.uber.org/automaxprocs/.codecov.yml +++ /dev/null @@ -1,14 +0,0 @@ -coverage: - range: 80..100 - round: down - precision: 2 - - status: - project: # measuring the overall project coverage - default: # context, you can create multiple ones with custom titles - enabled: yes # must be yes|true to enable this status - target: 90% # specify the target coverage for each commit status - # option: "auto" (must increase from parent commit or pull request base) - # option: "X%" a static target percentage to hit - if_not_found: success # if parent is not found report status as success, error, or failure - if_ci_failed: error # if ci fails report status as success, error, or failure diff --git a/vendor/go.uber.org/automaxprocs/.gitignore b/vendor/go.uber.org/automaxprocs/.gitignore deleted file mode 100644 index dd7bcf5130b..00000000000 --- a/vendor/go.uber.org/automaxprocs/.gitignore +++ /dev/null @@ -1,33 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test -vendor - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof -*.pprof -*.out -*.log -coverage.txt - -/bin -cover.out -cover.html diff --git a/vendor/go.uber.org/automaxprocs/CHANGELOG.md b/vendor/go.uber.org/automaxprocs/CHANGELOG.md deleted file mode 100644 index f421056ae82..00000000000 --- a/vendor/go.uber.org/automaxprocs/CHANGELOG.md +++ /dev/null @@ -1,52 +0,0 @@ -# Changelog - -## v1.6.0 (2024-07-24) - -- Add RoundQuotaFunc option that allows configuration of rounding - behavior for floating point CPU quota. - -## v1.5.3 (2023-07-19) - -- Fix mountinfo parsing when super options have fields with spaces. -- Fix division by zero while parsing cgroups. - -## v1.5.2 (2023-03-16) - -- Support child control cgroups -- Fix file descriptor leak -- Update dependencies - -## v1.5.1 (2022-04-06) - -- Fix cgroups v2 mountpoint detection. - -## v1.5.0 (2022-04-05) - -- Add support for cgroups v2. - -Thanks to @emadolsky for their contribution to this release. - -## v1.4.0 (2021-02-01) - -- Support colons in cgroup names. -- Remove linters from runtime dependencies. - -## v1.3.0 (2020-01-23) - -- Migrate to Go modules. - -## v1.2.0 (2018-02-22) - -- Fixed quota clamping to always round down rather than up; Rather than - guaranteeing constant throttling at saturation, instead assume that the - fractional CPU was added as a hedge for factors outside of Go's scheduler. - -## v1.1.0 (2017-11-10) - -- Log the new value of `GOMAXPROCS` rather than the current value. -- Make logs more explicit about whether `GOMAXPROCS` was modified or not. -- Allow customization of the minimum `GOMAXPROCS`, and modify default from 2 to 1. - -## v1.0.0 (2017-08-09) - -- Initial release. diff --git a/vendor/go.uber.org/automaxprocs/CODE_OF_CONDUCT.md b/vendor/go.uber.org/automaxprocs/CODE_OF_CONDUCT.md deleted file mode 100644 index e327d9aa5cd..00000000000 --- a/vendor/go.uber.org/automaxprocs/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,75 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, -body size, disability, ethnicity, gender identity and expression, level of -experience, nationality, personal appearance, race, religion, or sexual -identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an -appointed representative at an online or offline event. Representation of a -project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at oss-conduct@uber.com. The project -team will review and investigate all complaints, and will respond in a way -that it deems appropriate to the circumstances. The project team is obligated -to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 1.4, available at -[http://contributor-covenant.org/version/1/4][version]. - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ diff --git a/vendor/go.uber.org/automaxprocs/CONTRIBUTING.md b/vendor/go.uber.org/automaxprocs/CONTRIBUTING.md deleted file mode 100644 index 2b6a6040d78..00000000000 --- a/vendor/go.uber.org/automaxprocs/CONTRIBUTING.md +++ /dev/null @@ -1,81 +0,0 @@ -# Contributing - -We'd love your help improving this package! - -If you'd like to add new exported APIs, please [open an issue][open-issue] -describing your proposal — discussing API changes ahead of time makes -pull request review much smoother. In your issue, pull request, and any other -communications, please remember to treat your fellow contributors with -respect! We take our [code of conduct](CODE_OF_CONDUCT.md) seriously. - -Note that you'll need to sign [Uber's Contributor License Agreement][cla] -before we can accept any of your contributions. If necessary, a bot will remind -you to accept the CLA when you open your pull request. - -## Setup - -[Fork][fork], then clone the repository: - -``` -mkdir -p $GOPATH/src/go.uber.org -cd $GOPATH/src/go.uber.org -git clone git@github.com:your_github_username/automaxprocs.git -cd automaxprocs -git remote add upstream https://github.com/uber-go/automaxprocs.git -git fetch upstream -``` - -Install the test dependencies: - -``` -make dependencies -``` - -Make sure that the tests and the linters pass: - -``` -make test -make lint -``` - -If you're not using the minor version of Go specified in the Makefile's -`LINTABLE_MINOR_VERSIONS` variable, `make lint` doesn't do anything. This is -fine, but it means that you'll only discover lint failures after you open your -pull request. - -## Making Changes - -Start by creating a new branch for your changes: - -``` -cd $GOPATH/src/go.uber.org/automaxprocs -git checkout master -git fetch upstream -git rebase upstream/master -git checkout -b cool_new_feature -``` - -Make your changes, then ensure that `make lint` and `make test` still pass. If -you're satisfied with your changes, push them to your fork. - -``` -git push origin cool_new_feature -``` - -Then use the GitHub UI to open a pull request. - -At this point, you're waiting on us to review your changes. We *try* to respond -to issues and pull requests within a few business days, and we may suggest some -improvements or alternatives. Once your changes are approved, one of the -project maintainers will merge them. - -We're much more likely to approve your changes if you: - -* Add tests for new functionality. -* Write a [good commit message][commit-message]. -* Maintain backward compatibility. - -[fork]: https://github.com/uber-go/automaxprocs/fork -[open-issue]: https://github.com/uber-go/automaxprocs/issues/new -[cla]: https://cla-assistant.io/uber-go/automaxprocs -[commit-message]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html diff --git a/vendor/go.uber.org/automaxprocs/Makefile b/vendor/go.uber.org/automaxprocs/Makefile deleted file mode 100644 index 1642b714801..00000000000 --- a/vendor/go.uber.org/automaxprocs/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -export GOBIN ?= $(shell pwd)/bin - -GO_FILES := $(shell \ - find . '(' -path '*/.*' -o -path './vendor' ')' -prune \ - -o -name '*.go' -print | cut -b3-) - -GOLINT = $(GOBIN)/golint -STATICCHECK = $(GOBIN)/staticcheck - -.PHONY: build -build: - go build ./... - -.PHONY: install -install: - go mod download - -.PHONY: test -test: - go test -race ./... - -.PHONY: cover -cover: - go test -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./... - go tool cover -html=cover.out -o cover.html - -$(GOLINT): tools/go.mod - cd tools && go install golang.org/x/lint/golint - -$(STATICCHECK): tools/go.mod - cd tools && go install honnef.co/go/tools/cmd/staticcheck@2023.1.2 - -.PHONY: lint -lint: $(GOLINT) $(STATICCHECK) - @rm -rf lint.log - @echo "Checking gofmt" - @gofmt -d -s $(GO_FILES) 2>&1 | tee lint.log - @echo "Checking go vet" - @go vet ./... 2>&1 | tee -a lint.log - @echo "Checking golint" - @$(GOLINT) ./... | tee -a lint.log - @echo "Checking staticcheck" - @$(STATICCHECK) ./... 2>&1 | tee -a lint.log - @echo "Checking for license headers..." - @./.build/check_license.sh | tee -a lint.log - @[ ! -s lint.log ] diff --git a/vendor/go.uber.org/automaxprocs/README.md b/vendor/go.uber.org/automaxprocs/README.md deleted file mode 100644 index bfed32adae8..00000000000 --- a/vendor/go.uber.org/automaxprocs/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# automaxprocs [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov] - -Automatically set `GOMAXPROCS` to match Linux container CPU quota. - -## Installation - -`go get -u go.uber.org/automaxprocs` - -## Quick Start - -```go -import _ "go.uber.org/automaxprocs" - -func main() { - // Your application logic here. -} -``` - -# Performance -Data measured from Uber's internal load balancer. We ran the load balancer with 200% CPU quota (i.e., 2 cores): - -| GOMAXPROCS | RPS | P50 (ms) | P99.9 (ms) | -| ------------------ | --------- | -------- | ---------- | -| 1 | 28,893.18 | 1.46 | 19.70 | -| 2 (equal to quota) | 44,715.07 | 0.84 | 26.38 | -| 3 | 44,212.93 | 0.66 | 30.07 | -| 4 | 41,071.15 | 0.57 | 42.94 | -| 8 | 33,111.69 | 0.43 | 64.32 | -| Default (24) | 22,191.40 | 0.45 | 76.19 | - -When `GOMAXPROCS` is increased above the CPU quota, we see P50 decrease slightly, but see significant increases to P99. We also see that the total RPS handled also decreases. - -When `GOMAXPROCS` is higher than the CPU quota allocated, we also saw significant throttling: - -``` -$ cat /sys/fs/cgroup/cpu,cpuacct/system.slice/[...]/cpu.stat -nr_periods 42227334 -nr_throttled 131923 -throttled_time 88613212216618 -``` - -Once `GOMAXPROCS` was reduced to match the CPU quota, we saw no CPU throttling. - -## Development Status: Stable - -All APIs are finalized, and no breaking changes will be made in the 1.x series -of releases. Users of semver-aware dependency management systems should pin -automaxprocs to `^1`. - -## Contributing - -We encourage and support an active, healthy community of contributors — -including you! Details are in the [contribution guide](CONTRIBUTING.md) and -the [code of conduct](CODE_OF_CONDUCT.md). The automaxprocs maintainers keep -an eye on issues and pull requests, but you can also report any negative -conduct to oss-conduct@uber.com. That email list is a private, safe space; -even the automaxprocs maintainers don't have access, so don't hesitate to hold -us to a high standard. - -