From ab7a57f28da6e2df6c43c8d76652a3c9ea2106b6 Mon Sep 17 00:00:00 2001 From: Igor Grzankowski Date: Thu, 29 Jan 2026 10:12:53 +0100 Subject: [PATCH 01/19] Draft: CSPL-4347 (#1662) * a * Tests for slog * Use slog in main * Use assertion * black box --------- Co-authored-by: igor.grzankowski <@splunk.com> --- cmd/main.go | 46 ++- pkg/logging/slog.go | 184 ++++++++++ pkg/logging/slog_test.go | 768 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 993 insertions(+), 5 deletions(-) create mode 100644 pkg/logging/slog.go create mode 100644 pkg/logging/slog_test.go diff --git a/cmd/main.go b/cmd/main.go index f8aba0ae1..266dd012d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -20,12 +20,14 @@ import ( "crypto/tls" "flag" "fmt" + "log/slog" "os" "time" intController "github.com/splunk/splunk-operator/internal/controller" "github.com/splunk/splunk-operator/internal/controller/debug" "github.com/splunk/splunk-operator/pkg/config" + "github.com/splunk/splunk-operator/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -54,6 +56,11 @@ import ( var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") + + // Version information (set via ldflags at build time) + version = "dev" + buildTime = "unknown" + gitCommit = "unknown" ) func init() { @@ -70,8 +77,11 @@ func main() { var enableLeaderElection bool var probeAddr string var pprofActive bool - var logEncoder string - var logLevel int + + // Structured logging flags + var logLevel string + var logFormat string + var logAddSource bool var leaseDuration time.Duration var renewDeadline time.Duration @@ -80,13 +90,16 @@ func main() { var tlsOpts []func(*tls.Config) - flag.StringVar(&logEncoder, "log-encoder", "json", "log encoding ('json' or 'console')") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&pprofActive, "pprof", true, "Enable pprof endpoint") - flag.IntVar(&logLevel, "log-level", int(zapcore.InfoLevel), "set log level") + + // Structured logging flags (can also be set via LOG_LEVEL, LOG_FORMAT, LOG_ADD_SOURCE env vars) + flag.StringVar(&logLevel, "log-level", "", "log level: debug, info, warn, error (overrides LOG_LEVEL env var)") + flag.StringVar(&logFormat, "log-format", "", "log output format: json, text (overrides LOG_FORMAT env var)") + flag.BoolVar(&logAddSource, "log-add-source", false, "add source file:line to log output (overrides LOG_ADD_SOURCE env var)") flag.IntVar(&leaseDurationSecond, "lease-duration", int(leaseDurationSecond), "manager lease duration in seconds") flag.IntVar(&renewDeadlineSecond, "renew-duration", int(renewDeadlineSecond), "manager renew duration in seconds") flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metrics endpoint binds to. "+ @@ -140,7 +153,30 @@ func main() { opts.BindFlags(flag.CommandLine) flag.Parse() - // Logging setup + // Initialize structured logging infrastructure + // Flags take precedence over environment variables + var addSourcePtr *bool + if logAddSource { + addSourcePtr = &logAddSource + } + logCfg := logging.LoadConfigWithFlags(logLevel, logFormat, addSourcePtr) + logger := logging.SetupLogger(logCfg, + slog.String("component", "splunk-operator"), + slog.String("version", version), + slog.String("build", gitCommit)) + + // Log startup information using slog + slog.Info("Splunk Operator starting", + slog.String("version", version), + slog.String("build_time", buildTime), + slog.String("git_commit", gitCommit), + slog.String("log_level", logging.LevelToString(logCfg.Level)), + slog.String("log_format", logCfg.Format), + slog.Bool("log_add_source", logCfg.AddSource)) + + _ = logger // logger is available for future use + + // Logging setup (existing zap logger for controller-runtime compatibility) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) baseOptions := ctrl.Options{ diff --git a/pkg/logging/slog.go b/pkg/logging/slog.go new file mode 100644 index 000000000..ace2877a5 --- /dev/null +++ b/pkg/logging/slog.go @@ -0,0 +1,184 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "io" + "log/slog" + "os" + "strings" + "time" +) + +const ( + EnvLogLevel = "LOG_LEVEL" + EnvLogFormat = "LOG_FORMAT" + EnvLogAddSource = "LOG_ADD_SOURCE" + + FormatJSON = "json" + FormatText = "text" + + DefaultLogLevel = slog.LevelInfo + DefaultLogFormat = FormatJSON +) + +// Config holds the logging configuration +type Config struct { + Level slog.Level + Format string + AddSource bool +} + +// sensitiveKeys contains field names that should be redacted +var sensitiveKeys = []string{ + "password", + "token", + "secret", + "apikey", + "api_key", + "credential", + "auth", +} + +// LoadConfig loads logging configuration from environment variables +// Environment variables take precedence over defaults +func LoadConfig() Config { + cfg := Config{ + Level: DefaultLogLevel, + Format: DefaultLogFormat, + AddSource: false, + } + + if level := os.Getenv(EnvLogLevel); level != "" { + cfg.Level = LevelFromString(level) + } + + if format := os.Getenv(EnvLogFormat); format != "" { + cfg.Format = strings.ToLower(format) + } + + if addSource := os.Getenv(EnvLogAddSource); addSource != "" { + cfg.AddSource = strings.ToLower(addSource) == "true" + } + + if cfg.Level == slog.LevelDebug && os.Getenv(EnvLogAddSource) == "" { + cfg.AddSource = true + } + + return cfg +} + +// LoadConfigWithFlags loads configuration with command-line flag overrides +// Flags take precedence over environment variables +func LoadConfigWithFlags(levelFlag, formatFlag string, addSourceFlag *bool) Config { + cfg := LoadConfig() + + if levelFlag != "" { + cfg.Level = LevelFromString(levelFlag) + } + + if formatFlag != "" { + cfg.Format = strings.ToLower(formatFlag) + } + + if addSourceFlag != nil { + cfg.AddSource = *addSourceFlag + } + + if cfg.Level == slog.LevelDebug && addSourceFlag == nil && os.Getenv(EnvLogAddSource) == "" { + cfg.AddSource = true + } + + return cfg +} + +// NewHandler creates a new slog.Handler based on the configuration +func NewHandler(cfg Config) slog.Handler { + return NewHandlerWithWriter(cfg, os.Stdout) +} + +// NewHandlerWithWriter creates a new slog.Handler that writes to the specified writer +func NewHandlerWithWriter(cfg Config, w io.Writer) slog.Handler { + opts := &slog.HandlerOptions{ + Level: cfg.Level, + AddSource: cfg.AddSource, + ReplaceAttr: redactSensitiveData, + } + + if cfg.Format == FormatText { + return slog.NewTextHandler(w, opts) + } + + return slog.NewJSONHandler(w, opts) +} + +// redactSensitiveData replaces sensitive field values with [REDACTED] +func redactSensitiveData(groups []string, a slog.Attr) slog.Attr { + keyLower := strings.ToLower(a.Key) + for _, sensitive := range sensitiveKeys { + if strings.Contains(keyLower, sensitive) { + return slog.String(a.Key, "[REDACTED]") + } + } + + if a.Key == slog.TimeKey { + if t, ok := a.Value.Any().(time.Time); ok { + return slog.String(slog.TimeKey, t.Format(time.RFC3339)) + } + } + + return a +} + +// SetupLogger initializes the global slog logger with the given configuration +// and optional attributes, then returns the configured logger +func SetupLogger(cfg Config, attrs ...slog.Attr) *slog.Logger { + handler := NewHandler(cfg) + logger := slog.New(handler) + for _, attr := range attrs { + logger = logger.With(attr) + } + slog.SetDefault(logger) + return logger +} + +// LevelFromString converts a string to slog.Level +func LevelFromString(s string) slog.Level { + switch strings.ToLower(s) { + case "debug": + return slog.LevelDebug + case "warn", "warning": + return slog.LevelWarn + case "error": + return slog.LevelError + default: + return slog.LevelInfo + } +} + +// LevelToString converts slog.Level to string +func LevelToString(level slog.Level) string { + switch level { + case slog.LevelDebug: + return "debug" + case slog.LevelWarn: + return "warn" + case slog.LevelError: + return "error" + default: + return "info" + } +} diff --git a/pkg/logging/slog_test.go b/pkg/logging/slog_test.go new file mode 100644 index 000000000..5887392d8 --- /dev/null +++ b/pkg/logging/slog_test.go @@ -0,0 +1,768 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging_test + +import ( + "bytes" + "log/slog" + "os" + "testing" + + "github.com/splunk/splunk-operator/pkg/logging" + "github.com/stretchr/testify/assert" +) + +const ( + envLogLevel = "LOG_LEVEL" + envLogFormat = "LOG_FORMAT" + envLogAddSource = "LOG_ADD_SOURCE" +) + +func TestLoadConfig(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expectedLevel slog.Level + expectedFormat string + expectedSource bool + }{ + { + name: "default values", + envVars: map[string]string{}, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "debug level with auto source", + envVars: map[string]string{ + envLogLevel: "debug", + }, + expectedLevel: slog.LevelDebug, + expectedFormat: "json", + expectedSource: true, + }, + { + name: "warn level with text format", + envVars: map[string]string{ + envLogLevel: "warn", + envLogFormat: "text", + }, + expectedLevel: slog.LevelWarn, + expectedFormat: "text", + expectedSource: false, + }, + { + name: "error level with explicit source", + envVars: map[string]string{ + envLogLevel: "error", + envLogAddSource: "true", + }, + expectedLevel: slog.LevelError, + expectedFormat: "json", + expectedSource: true, + }, + { + name: "debug level with source disabled", + envVars: map[string]string{ + envLogLevel: "debug", + envLogAddSource: "false", + }, + expectedLevel: slog.LevelDebug, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "invalid level defaults to info", + envVars: map[string]string{ + envLogLevel: "invalid", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearEnv() + for k, v := range tt.envVars { + os.Setenv(k, v) + } + + cfg := logging.LoadConfig() + + assert.Equal(t, tt.expectedLevel, cfg.Level) + assert.Equal(t, tt.expectedFormat, cfg.Format) + assert.Equal(t, tt.expectedSource, cfg.AddSource) + + clearEnv() + }) + } +} + +func TestLoadConfigWithFlags(t *testing.T) { + clearEnv() + os.Setenv(envLogLevel, "info") + os.Setenv(envLogFormat, "json") + defer clearEnv() + + addSource := true + cfg := logging.LoadConfigWithFlags("debug", "text", &addSource) + + assert.Equal(t, slog.LevelDebug, cfg.Level) + assert.Equal(t, "text", cfg.Format) + assert.True(t, cfg.AddSource) +} + +func TestLevelFromString(t *testing.T) { + tests := []struct { + input string + expected slog.Level + }{ + {"debug", slog.LevelDebug}, + {"DEBUG", slog.LevelDebug}, + {"info", slog.LevelInfo}, + {"INFO", slog.LevelInfo}, + {"warn", slog.LevelWarn}, + {"WARN", slog.LevelWarn}, + {"warning", slog.LevelWarn}, + {"error", slog.LevelError}, + {"ERROR", slog.LevelError}, + {"invalid", slog.LevelInfo}, + {"", slog.LevelInfo}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := logging.LevelFromString(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestLevelToString(t *testing.T) { + tests := []struct { + level slog.Level + expected string + }{ + {slog.LevelDebug, "debug"}, + {slog.LevelInfo, "info"}, + {slog.LevelWarn, "warn"}, + {slog.LevelError, "error"}, + {slog.Level(100), "info"}, // unknown level defaults to info + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + result := logging.LevelToString(tt.level) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestNewHandler(t *testing.T) { + tests := []struct { + name string + config logging.Config + }{ + { + name: "JSON handler", + config: logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + }, + }, + { + name: "Text handler", + config: logging.Config{ + Level: slog.LevelDebug, + Format: "text", + AddSource: true, + }, + }, + { + name: "Invalid format defaults to JSON", + config: logging.Config{ + Level: slog.LevelInfo, + Format: "invalid_format", + AddSource: false, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler := logging.NewHandler(tt.config) + assert.NotNil(t, handler) + }) + } +} + +func TestSetupLogger(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + } + + logger := logging.SetupLogger(cfg) + assert.NotNil(t, logger) + assert.Equal(t, logger, slog.Default()) +} + +func TestSetupLoggerWithAttrs(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + } + + logger := logging.SetupLogger(cfg, + slog.String("component", "splunk-operator"), + slog.String("version", "1.0.0"), + slog.String("build", "abc123")) + assert.NotNil(t, logger) + assert.Equal(t, logger, slog.Default()) +} + +func TestSetupLoggerWithEmptyAttrs(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + } + + logger := logging.SetupLogger(cfg) + assert.NotNil(t, logger) + + // Test logging still works + var buf bytes.Buffer + handler := slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo}) + testLogger := slog.New(handler) + testLogger.Info("test message") + assert.NotEmpty(t, buf.String()) +} + +func TestSetupLogger_ConcurrentAccess(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + } + + logger := logging.SetupLogger(cfg) + + done := make(chan bool, 10) + for i := 0; i < 10; i++ { + go func(id int) { + defer func() { + if r := recover(); r != nil { + t.Errorf("Panic in goroutine %d: %v", id, r) + } + done <- true + }() + + for j := 0; j < 100; j++ { + logger.Info("concurrent test", + slog.Int("goroutine", id), + slog.Int("iteration", j)) + } + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} + +func TestLoadConfigWithFlags_EdgeCases(t *testing.T) { + tests := []struct { + name string + envLevel string + envFormat string + flagLevel string + flagFormat string + flagAddSource *bool + expectedLevel slog.Level + expectedFormat string + expectedSource bool + }{ + { + name: "empty flags use env vars", + envLevel: "warn", + envFormat: "text", + flagLevel: "", + flagFormat: "", + flagAddSource: nil, + expectedLevel: slog.LevelWarn, + expectedFormat: "text", + expectedSource: false, + }, + { + name: "flags override env vars", + envLevel: "info", + envFormat: "json", + flagLevel: "error", + flagFormat: "text", + flagAddSource: boolPtr(true), + expectedLevel: slog.LevelError, + expectedFormat: "text", + expectedSource: true, + }, + { + name: "invalid flag level defaults to info", + envLevel: "warn", + envFormat: "json", + flagLevel: "invalid", + flagFormat: "", + flagAddSource: nil, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "nil addSource pointer uses env/default", + envLevel: "info", + envFormat: "json", + flagLevel: "", + flagFormat: "", + flagAddSource: nil, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "false addSource pointer explicitly disables", + envLevel: "debug", + envFormat: "json", + flagLevel: "", + flagFormat: "", + flagAddSource: boolPtr(false), + expectedLevel: slog.LevelDebug, + expectedFormat: "json", + expectedSource: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearEnv() + if tt.envLevel != "" { + os.Setenv(envLogLevel, tt.envLevel) + } + if tt.envFormat != "" { + os.Setenv(envLogFormat, tt.envFormat) + } + + cfg := logging.LoadConfigWithFlags(tt.flagLevel, tt.flagFormat, tt.flagAddSource) + + assert.Equal(t, tt.expectedLevel, cfg.Level) + assert.Equal(t, tt.expectedFormat, cfg.Format) + assert.Equal(t, tt.expectedSource, cfg.AddSource) + + clearEnv() + }) + } +} + +func TestLoadConfig_InvalidEnvironmentValues(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expectedLevel slog.Level + expectedFormat string + expectedSource bool + }{ + { + name: "garbage log level", + envVars: map[string]string{ + envLogLevel: "notavalidlevel", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "numeric log level (invalid)", + envVars: map[string]string{ + envLogLevel: "123", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "special characters in log level", + envVars: map[string]string{ + envLogLevel: "!@#$%", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "invalid format stored as-is", + envVars: map[string]string{ + envLogFormat: "xml", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "xml", + expectedSource: false, + }, + { + name: "invalid add_source value", + envVars: map[string]string{ + envLogAddSource: "yes", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: false, + }, + { + name: "mixed case add_source", + envVars: map[string]string{ + envLogAddSource: "TRUE", + }, + expectedLevel: slog.LevelInfo, + expectedFormat: "json", + expectedSource: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearEnv() + for k, v := range tt.envVars { + os.Setenv(k, v) + } + + cfg := logging.LoadConfig() + + assert.Equal(t, tt.expectedLevel, cfg.Level) + assert.Equal(t, tt.expectedFormat, cfg.Format) + assert.Equal(t, tt.expectedSource, cfg.AddSource) + + clearEnv() + }) + } +} + +func TestLogOutput_LevelFiltering(t *testing.T) { + tests := []struct { + name string + handlerLevel slog.Level + logLevel slog.Level + shouldContain bool + }{ + {"debug log at info level", slog.LevelInfo, slog.LevelDebug, false}, + {"info log at info level", slog.LevelInfo, slog.LevelInfo, true}, + {"warn log at info level", slog.LevelInfo, slog.LevelWarn, true}, + {"error log at info level", slog.LevelInfo, slog.LevelError, true}, + {"info log at error level", slog.LevelError, slog.LevelInfo, false}, + {"error log at error level", slog.LevelError, slog.LevelError, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + opts := &slog.HandlerOptions{Level: tt.handlerLevel} + handler := slog.NewTextHandler(&buf, opts) + logger := slog.New(handler) + + switch tt.logLevel { + case slog.LevelDebug: + logger.Debug("test message") + case slog.LevelInfo: + logger.Info("test message") + case slog.LevelWarn: + logger.Warn("test message") + case slog.LevelError: + logger.Error("test message") + } + + hasOutput := buf.Len() > 0 + assert.Equal(t, tt.shouldContain, hasOutput) + }) + } +} + +func TestSensitiveDataRedaction(t *testing.T) { + var buf bytes.Buffer + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "text", + AddSource: false, + } + + handler := logging.NewHandlerWithWriter(cfg, &buf) + logger := slog.New(handler) + + logger.Info("test", + slog.String("password", "secret123"), + slog.String("token", "abc123"), + slog.String("username", "admin")) + + output := buf.String() + + // Verify sensitive data is redacted + assert.NotContains(t, output, "secret123", "password should be redacted") + assert.NotContains(t, output, "abc123", "token should be redacted") + assert.Contains(t, output, "[REDACTED]", "redaction marker should be present") + assert.Contains(t, output, "admin", "non-sensitive data should not be redacted") +} + +func clearEnv() { + os.Unsetenv(envLogLevel) + os.Unsetenv(envLogFormat) + os.Unsetenv(envLogAddSource) +} + +func boolPtr(b bool) *bool { + return &b +} + +func TestLevelConversions_Roundtrip(t *testing.T) { + levels := []slog.Level{ + slog.LevelDebug, + slog.LevelInfo, + slog.LevelWarn, + slog.LevelError, + } + + for _, level := range levels { + str := logging.LevelToString(level) + result := logging.LevelFromString(str) + assert.Equal(t, level, result, "Roundtrip failed for level %v", level) + } +} + +func TestNewHandler_OutputFormat(t *testing.T) { + t.Run("JSON format produces JSON output", func(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "json", + AddSource: false, + } + handler := logging.NewHandler(cfg) + assert.NotNil(t, handler) + }) + + t.Run("Text format produces text output", func(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "text", + AddSource: false, + } + handler := logging.NewHandler(cfg) + assert.NotNil(t, handler) + }) +} + +func TestSensitiveDataRedactionAllKeys(t *testing.T) { + var buf bytes.Buffer + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "text", + AddSource: false, + } + + handler := logging.NewHandlerWithWriter(cfg, &buf) + logger := slog.New(handler) + + // Test all sensitive key patterns + logger.Info("test", + slog.String("password", "pass1"), + slog.String("token", "tok1"), + slog.String("secret", "sec1"), + slog.String("apikey", "key1"), + slog.String("api_key", "key2"), + slog.String("credential", "cred1"), + slog.String("auth", "auth1"), + slog.String("normal_field", "visible")) + + output := buf.String() + + // Verify ALL sensitive values are redacted + sensitiveValues := []string{"pass1", "tok1", "sec1", "key1", "key2", "cred1", "auth1"} + for _, val := range sensitiveValues { + assert.NotContains(t, output, val, "sensitive value %q should be redacted", val) + } + + // Verify normal value is NOT redacted + assert.Contains(t, output, "visible", "normal field should not be redacted") +} + +func TestConfigStruct(t *testing.T) { + cfg := logging.Config{ + Level: slog.LevelDebug, + Format: "text", + AddSource: true, + } + + assert.Equal(t, slog.LevelDebug, cfg.Level) + assert.Equal(t, "text", cfg.Format) + assert.True(t, cfg.AddSource) +} + +func TestRedactionThroughHandler(t *testing.T) { + tests := []struct { + name string + key string + value string + shouldRedact bool + }{ + {"password field", "password", "secret123", true}, + {"Password field", "Password", "secret456", true}, + {"admin_password field", "admin_password", "secret789", true}, + {"token field", "token", "tok123", true}, + {"auth_token field", "auth_token", "tok456", true}, + {"secret field", "secret", "sec123", true}, + {"apikey field", "apikey", "key123", true}, + {"api_key field", "api_key", "key456", true}, + {"credential field", "credential", "cred123", true}, + {"auth field", "auth", "auth123", true}, + {"normal field", "namespace", "default", false}, + {"name field", "name", "myapp", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + cfg := logging.Config{ + Level: slog.LevelInfo, + Format: "text", + AddSource: false, + } + handler := logging.NewHandlerWithWriter(cfg, &buf) + logger := slog.New(handler) + + logger.Info("test", slog.String(tt.key, tt.value)) + output := buf.String() + + if tt.shouldRedact { + assert.NotContains(t, output, tt.value, "value should be redacted") + assert.Contains(t, output, "[REDACTED]") + } else { + assert.Contains(t, output, tt.value, "value should NOT be redacted") + } + }) + } +} + +func TestDebugLevelAutoEnablesSource(t *testing.T) { + clearEnv() + os.Setenv(envLogLevel, "debug") + defer clearEnv() + + cfg := logging.LoadConfig() + + assert.Equal(t, slog.LevelDebug, cfg.Level) + assert.True(t, cfg.AddSource, "Debug level should auto-enable source") +} + +func TestDebugLevelSourceCanBeDisabled(t *testing.T) { + clearEnv() + os.Setenv(envLogLevel, "debug") + os.Setenv(envLogAddSource, "false") + defer clearEnv() + + cfg := logging.LoadConfig() + + assert.Equal(t, slog.LevelDebug, cfg.Level) + assert.False(t, cfg.AddSource, "Source should be disabled when explicitly set to false") +} + +func TestWarningAlias(t *testing.T) { + // "warning" should be treated same as "warn" + result := logging.LevelFromString("warning") + assert.Equal(t, slog.LevelWarn, result) +} + +func TestCaseInsensitiveLevel(t *testing.T) { + tests := []string{"DEBUG", "Debug", "dEbUg", "debug"} + for _, input := range tests { + result := logging.LevelFromString(input) + assert.Equal(t, slog.LevelDebug, result, "Input: %s", input) + } +} + +func TestFormatCaseInsensitive(t *testing.T) { + clearEnv() + os.Setenv(envLogFormat, "JSON") + defer clearEnv() + + cfg := logging.LoadConfig() + assert.Equal(t, "json", cfg.Format) +} + +func TestAddSourceCaseInsensitive(t *testing.T) { + clearEnv() + os.Setenv(envLogAddSource, "TRUE") + defer clearEnv() + + cfg := logging.LoadConfig() + assert.True(t, cfg.AddSource) +} + +func TestFlagsOverrideEnvVars(t *testing.T) { + clearEnv() + os.Setenv(envLogLevel, "info") + os.Setenv(envLogFormat, "json") + os.Setenv(envLogAddSource, "false") + defer clearEnv() + + addSource := true + cfg := logging.LoadConfigWithFlags("debug", "text", &addSource) + + assert.Equal(t, slog.LevelDebug, cfg.Level, "Flag should override env var for level") + assert.Equal(t, "text", cfg.Format, "Flag should override env var for format") + assert.True(t, cfg.AddSource, "Flag should override env var for addSource") +} + +func TestEmptyFlagsUseEnvVars(t *testing.T) { + clearEnv() + os.Setenv(envLogLevel, "warn") + os.Setenv(envLogFormat, "text") + os.Setenv(envLogAddSource, "true") + defer clearEnv() + + cfg := logging.LoadConfigWithFlags("", "", nil) + + assert.Equal(t, slog.LevelWarn, cfg.Level) + assert.Equal(t, "text", cfg.Format) + assert.True(t, cfg.AddSource) +} + +func TestNilAddSourcePointer(t *testing.T) { + clearEnv() + os.Setenv(envLogAddSource, "true") + defer clearEnv() + + cfg := logging.LoadConfigWithFlags("", "", nil) + assert.True(t, cfg.AddSource, "nil pointer should use env var value") +} + +func TestExplicitFalseAddSource(t *testing.T) { + clearEnv() + os.Setenv(envLogAddSource, "true") + defer clearEnv() + + addSource := false + cfg := logging.LoadConfigWithFlags("", "", &addSource) + assert.False(t, cfg.AddSource, "explicit false should override env var") +} From 29397ccaa6e984b6c51ae45b4eb7787985be315e Mon Sep 17 00:00:00 2001 From: Igor Grzankowski Date: Thu, 26 Feb 2026 09:44:13 +0100 Subject: [PATCH 02/19] CSPL-4357: Secrets logs (#1711) * Secrets logs * Get rid of log handle * Differentiate errors --- pkg/logging/slog.go | 17 +++ pkg/splunk/enterprise/clustermanager.go | 2 +- pkg/splunk/enterprise/clustermaster.go | 2 +- pkg/splunk/enterprise/configuration.go | 4 +- pkg/splunk/enterprise/indexercluster.go | 2 +- pkg/splunk/enterprise/searchheadcluster.go | 2 +- pkg/splunk/enterprise/util.go | 4 +- pkg/splunk/util/secrets.go | 168 +++++++++++++++------ pkg/splunk/util/secrets_test.go | 46 +++++- 9 files changed, 192 insertions(+), 55 deletions(-) diff --git a/pkg/logging/slog.go b/pkg/logging/slog.go index ace2877a5..152d41d8d 100644 --- a/pkg/logging/slog.go +++ b/pkg/logging/slog.go @@ -16,6 +16,7 @@ package logging import ( + "context" "io" "log/slog" "os" @@ -23,6 +24,22 @@ import ( "time" ) +type slogContextKey struct{} + +// WithLogger returns a new context with the given slog.Logger attached. +func WithLogger(ctx context.Context, l *slog.Logger) context.Context { + return context.WithValue(ctx, slogContextKey{}, l) +} + +// FromContext extracts the slog.Logger from the context. +// Falls back to slog.Default() if none is set. +func FromContext(ctx context.Context) *slog.Logger { + if l, ok := ctx.Value(slogContextKey{}).(*slog.Logger); ok { + return l + } + return slog.Default() +} + const ( EnvLogLevel = "LOG_LEVEL" EnvLogFormat = "LOG_FORMAT" diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index 269753c5c..b7a54910c 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -404,7 +404,7 @@ func PushManagerAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr eventPublisher, _ := newK8EventPublisher(c, cr) defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) - defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), cr.GetName(), defaultSecretObjName) + defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), defaultSecretObjName) if err != nil { eventPublisher.Warning(ctx, "PushManagerAppsBundle", fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) return fmt.Errorf("could not access default secret object to fetch admin password. Reason %v", err) diff --git a/pkg/splunk/enterprise/clustermaster.go b/pkg/splunk/enterprise/clustermaster.go index 85fcb70cc..496ecb31e 100644 --- a/pkg/splunk/enterprise/clustermaster.go +++ b/pkg/splunk/enterprise/clustermaster.go @@ -385,7 +385,7 @@ func PushMasterAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr eventPublisher, _ := newK8EventPublisher(c, cr) defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) - defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), cr.GetName(), defaultSecretObjName) + defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), defaultSecretObjName) if err != nil { eventPublisher.Warning(ctx, "PushMasterAppsBundle", fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) return fmt.Errorf("could not access default secret object to fetch admin password. Reason %v", err) diff --git a/pkg/splunk/enterprise/configuration.go b/pkg/splunk/enterprise/configuration.go index a0d90b354..78ed47b2f 100644 --- a/pkg/splunk/enterprise/configuration.go +++ b/pkg/splunk/enterprise/configuration.go @@ -428,7 +428,7 @@ func ValidateImagePullSecrets(ctx context.Context, c splcommon.ControllerClient, // If configured, validated if the secret/s exist for _, secret := range spec.ImagePullSecrets { - _, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), cr.GetName(), secret.Name) + _, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), secret.Name) if err != nil { scopedLog.Error(err, "Couldn't get secret in the imagePullSecrets config", "Secret", secret.Name) } @@ -1251,7 +1251,7 @@ func AreRemoteVolumeKeysChanged(ctx context.Context, client splcommon.Controller volList := smartstore.VolList for _, volume := range volList { if volume.SecretRef != "" { - namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), volume.SecretRef) + namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), volume.SecretRef) // Ideally, this should have been detected in Spec validation time if err != nil { *retError = fmt.Errorf("not able to access secret object = %s, reason: %s", volume.SecretRef, err) diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index 2d135d84f..4215fa273 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -750,7 +750,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if len(mgr.cr.Status.IdxcPasswordChangedSecrets) > 0 { for podSecretName := range mgr.cr.Status.IdxcPasswordChangedSecrets { if mgr.cr.Status.IdxcPasswordChangedSecrets[podSecretName] { - podSecret, err := splutil.GetSecretByName(ctx, mgr.c, mgr.cr.GetNamespace(), mgr.cr.GetName(), podSecretName) + podSecret, err := splutil.GetSecretByName(ctx, mgr.c, mgr.cr.GetNamespace(), podSecretName) if err != nil { return fmt.Errorf("could not read secret %s, reason - %v", podSecretName, err) } diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index d5b4fd12f..bb55671e2 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -404,7 +404,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli */ if len(mgr.cr.Status.AdminPasswordChangedSecrets) > 0 { for podSecretName := range mgr.cr.Status.AdminPasswordChangedSecrets { - podSecret, err := splutil.GetSecretByName(ctx, mgr.c, mgr.cr.GetNamespace(), mgr.cr.GetName(), podSecretName) + podSecret, err := splutil.GetSecretByName(ctx, mgr.c, mgr.cr.GetNamespace(), podSecretName) if err != nil { return fmt.Errorf("could not read secret %s, reason - %v", podSecretName, err) } diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index abd96482a..f616074b0 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -154,7 +154,7 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie secretAccessKey = "" } else { // Get credentials through the secretRef - remoteDataClientSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), appSecretRef) + remoteDataClientSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), appSecretRef) if err != nil { return remoteDataClient, err } @@ -398,7 +398,7 @@ func getSearchHeadExtraEnv(cr splcommon.MetaObject, replicas int32) []corev1.Env // GetSmartstoreRemoteVolumeSecrets is used to retrieve S3 access key and secrete keys. func GetSmartstoreRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi.VolumeSpec, client splcommon.ControllerClient, cr splcommon.MetaObject, smartstore *enterpriseApi.SmartStoreSpec) (string, string, string, error) { - namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), volume.SecretRef) + namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), volume.SecretRef) if err != nil { return "", "", "", err } diff --git a/pkg/splunk/util/secrets.go b/pkg/splunk/util/secrets.go index 35071e80a..e2caa1842 100644 --- a/pkg/splunk/util/secrets.go +++ b/pkg/splunk/util/secrets.go @@ -19,30 +19,40 @@ import ( "context" "errors" "fmt" + "log/slog" "reflect" "strconv" "strings" "time" + "github.com/splunk/splunk-operator/pkg/logging" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) // GetSpecificSecretTokenFromPod retrieves a specific secret token's value from a Pod func GetSpecificSecretTokenFromPod(ctx context.Context, c splcommon.ControllerClient, PodName string, namespace string, secretToken string) (string, error) { + logger := logging.FromContext(ctx).With("func", "GetSpecificSecretTokenFromPod") + logger.DebugContext(ctx, "Retrieving secret token from pod", + "pod", PodName, + "namespace", namespace, + "token_key", secretToken) + // Get Pod data secret, err := GetSecretFromPod(ctx, c, PodName, namespace) if err != nil { return "", err } - // Sanity checks if secret.Data == nil { + logger.WarnContext(ctx, "Secret has nil data. Update secret with required data", + "secret_name", secret.Name, + "namespace", namespace) return "", errors.New(invalidSecretDataError) } @@ -51,6 +61,10 @@ func GetSpecificSecretTokenFromPod(ctx context.Context, c splcommon.ControllerCl } if _, ok := secret.Data[secretToken]; !ok { + logger.WarnContext(ctx, "Secret is missing required field. Update secret with required data", + "secret_name", secret.Name, + "namespace", namespace, + "missing_field", secretToken) return "", errors.New(invalidSecretDataError) } @@ -63,16 +77,28 @@ func GetSecretFromPod(ctx context.Context, c splcommon.ControllerClient, PodName var currentSecret corev1.Secret var secretName string + logger := slog.With("func", "GetSecretFromPod") + logger.DebugContext(ctx, "Retrieving secret from pod", + "pod", PodName, + "namespace", namespace) + // Get Pod namespacedName := types.NamespacedName{Namespace: namespace, Name: PodName} err := c.Get(ctx, namespacedName, ¤tPod) if err != nil { + logger.WarnContext(ctx, "Pod not found", + "pod", PodName, + "namespace", namespace, + "error", err) return nil, errors.New(splcommon.PodNotFoundError) } // Get Pod Spec Volumes podSpecVolumes := currentPod.Spec.Volumes if len(podSpecVolumes) == 0 { + logger.WarnContext(ctx, "Pod has no volumes configured", + "pod", PodName, + "namespace", namespace) return nil, errors.New("empty pod spec volumes") } @@ -96,6 +122,10 @@ func GetSecretFromPod(ctx context.Context, c splcommon.ControllerClient, PodName namespacedName = types.NamespacedName{Namespace: namespace, Name: secretName} err = c.Get(ctx, namespacedName, ¤tSecret) if err != nil { + logger.WarnContext(ctx, "Secret not found for pod. Create secret to proceed", + "secret_name", secretName, + "pod", PodName, + "namespace", namespace) return nil, errors.New(splcommon.SecretNotFoundError) } @@ -114,7 +144,7 @@ func GetSecretLabels() map[string]string { func SetSecretOwnerRef(ctx context.Context, client splcommon.ControllerClient, secretObjectName string, cr splcommon.MetaObject) error { var err error - secret, err := GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), secretObjectName) + secret, err := GetSecretByName(ctx, client, cr.GetNamespace(), secretObjectName) if err != nil { return err } @@ -140,7 +170,7 @@ func RemoveSecretOwnerRef(ctx context.Context, client splcommon.ControllerClient var err error var refCount uint = 0 - secret, err := GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), secretObjectName) + secret, err := GetSecretByName(ctx, client, cr.GetNamespace(), secretObjectName) if err != nil { return 0, err } @@ -167,6 +197,7 @@ func RemoveSecretOwnerRef(ctx context.Context, client splcommon.ControllerClient // RemoveUnwantedSecrets deletes all secrets whose version preceeds (latestVersion - MinimumVersionedSecrets) func RemoveUnwantedSecrets(ctx context.Context, c splcommon.ControllerClient, versionedSecretIdentifier, namespace string) error { + logger := slog.With("func", "RemoveUnwantedSecrets") // retrieve the list of versioned namespace scoped secrets _, latestVersion, list := GetExistingLatestVersionedSecret(ctx, c, namespace, versionedSecretIdentifier, true) if latestVersion != -1 { @@ -181,8 +212,18 @@ func RemoveUnwantedSecrets(ctx context.Context, c splcommon.ControllerClient, ve // Delete secret err := DeleteResource(ctx, c, &secret) if err != nil { + logger.ErrorContext(ctx, "Failed to delete old versioned secret", + "secret_name", secret.GetName(), + "version", version, + "namespace", namespace, + "error", err) return err } + logger.InfoContext(ctx, "Deleted old versioned secret", + "secret_name", secret.GetName(), + "version", version, + "latest_version", latestVersion, + "namespace", namespace) } } } @@ -194,11 +235,19 @@ func RemoveUnwantedSecrets(ctx context.Context, c splcommon.ControllerClient, ve func GetNamespaceScopedSecret(ctx context.Context, c splcommon.ControllerClient, namespace string) (*corev1.Secret, error) { var namespaceScopedSecret corev1.Secret - // Check if a namespace scoped secret exists - namespacedName := types.NamespacedName{Namespace: namespace, Name: splcommon.GetNamespaceScopedSecretName(namespace)} + logger := slog.With("func", "GetNamespaceScopedSecret") + name := splcommon.GetNamespaceScopedSecretName(namespace) + logger.DebugContext(ctx, "Retrieving namespace-scoped secret", + "secret_name", name, + "namespace", namespace) + + namespacedName := types.NamespacedName{Namespace: namespace, Name: name} err := c.Get(ctx, namespacedName, &namespaceScopedSecret) if err != nil { - // Didn't find it + logger.WarnContext(ctx, "Namespace-scoped secret not found. Create secret to proceed", + "secret_name", name, + "namespace", namespace, + "error", err) return nil, err } @@ -232,11 +281,7 @@ func GetVersionedSecretVersion(secretName string, versionedSecretIdentifier stri // GetExistingLatestVersionedSecret retrieves latest EXISTING versionedSecretIdentifier based secret existing currently in the namespace func GetExistingLatestVersionedSecret(ctx context.Context, c splcommon.ControllerClient, namespace string, versionedSecretIdentifier string, list bool) (*corev1.Secret, int, map[int]corev1.Secret) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetExistingLatestVersionedSecret").WithValues( - "versionedSecretIdentifier", versionedSecretIdentifier, - "namespace", namespace) - + logger := slog.With("func", "GetExistingLatestVersionedSecret") // Get list of secrets in K8S cluster secretList := corev1.SecretList{} @@ -251,7 +296,9 @@ func GetExistingLatestVersionedSecret(ctx context.Context, c splcommon.Controlle err := c.List(ctx, &secretList, listOpts...) if err != nil || len(secretList.Items) == 0 { - scopedLog.Info("Secrets not found in namespace") + logger.DebugContext(ctx, "No versioned secrets found in namespace", + "versioned_secret_identifier", versionedSecretIdentifier, + "namespace", namespace) return nil, -1, nil } @@ -291,20 +338,7 @@ func GetExistingLatestVersionedSecret(ctx context.Context, c splcommon.Controlle // GetLatestVersionedSecret is used to create/retrieve latest versionedSecretIdentifier based secret, cr is optional for owner references(pass nil if not required) func GetLatestVersionedSecret(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, namespace string, versionedSecretIdentifier string) (*corev1.Secret, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetLatestVersionedSecret").WithValues( - "versionedSecretIdentifier", versionedSecretIdentifier, - "namespace", namespace) - - // If CR is passed log it as well - if cr != nil { - scopedLog = reqLogger.WithName("GetLatestVersionedSecret").WithValues( - "versionedSecretIdentifier", versionedSecretIdentifier, - "cr", cr.GetName(), - "kind", cr.GetObjectKind(), - "namespace", namespace) - } - + logger := slog.With("func", "GetLatestVersionedSecret") var latestVersionedSecret *corev1.Secret var err error @@ -320,15 +354,23 @@ func GetLatestVersionedSecret(ctx context.Context, c splcommon.ControllerClient, // Check if there is atleast one versionedSecretIdentifier based secret if existingLatestVersion == -1 { // No secret based on versionedSecretIdentifier, create one with version v1 - scopedLog.Info("Creating first version secret") + logger.InfoContext(ctx, "Creating first version secret", + "versioned_secret_identifier", versionedSecretIdentifier, + "namespace", namespace) latestVersionedSecret, _ = ApplySplunkSecret(ctx, c, cr, splunkReadableData, splcommon.GetVersionedSecretName(versionedSecretIdentifier, splcommon.FirstVersion), namespace) } else { // Check if contents of latest versionedSecretIdentifier based secret is different from that of namespace scoped secrets object if !reflect.DeepEqual(splunkReadableData, existingLatestVersionedSecret.Data) { // Different, create a newer version versionedSecretIdentifier based secret latestVersionedSecret, err = ApplySplunkSecret(ctx, c, cr, splunkReadableData, splcommon.GetVersionedSecretName(versionedSecretIdentifier, strconv.Itoa(existingLatestVersion+1)), namespace) - scopedLog.Info("Data in the latest versioned secret is different from the namespace scoped secret, hence creating a new secret", "newSecretName", latestVersionedSecret.GetName(), - "newSecretVersion", existingLatestVersion+1, "oldSecretName", existingLatestVersionedSecret.GetName(), "oldSecretVersion", existingLatestVersion) + if latestVersionedSecret != nil { + logger.InfoContext(ctx, "Secret version changed", + "new_secret_name", latestVersionedSecret.GetName(), + "new_version", existingLatestVersion+1, + "old_secret_name", existingLatestVersionedSecret.GetName(), + "old_version", existingLatestVersion, + "namespace", namespace) + } return latestVersionedSecret, err } @@ -379,6 +421,7 @@ splunk: // ApplySplunkSecret creates/updates a secret using secretData(which HAS to be of ansible readable format) or namespace scoped secret data if not specified func ApplySplunkSecret(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, secretData map[string][]byte, secretName string, namespace string) (*corev1.Secret, error) { + logger := slog.With("func", "ApplySplunkSecret") var current corev1.Secret var newSecretData map[string][]byte var err error @@ -417,6 +460,10 @@ func ApplySplunkSecret(ctx context.Context, c splcommon.ControllerClient, cr spl // Didn't find secret, create it err = CreateResource(ctx, c, ¤t) if err != nil { + logger.ErrorContext(ctx, "Failed to create secret", + "secret_name", secretName, + "namespace", namespace, + "error", err) return nil, err } } else { @@ -425,6 +472,10 @@ func ApplySplunkSecret(ctx context.Context, c splcommon.ControllerClient, cr spl current.Data = newSecretData err = UpdateResource(ctx, c, ¤t) if err != nil { + logger.ErrorContext(ctx, "Failed to update secret", + "secret_name", secretName, + "namespace", namespace, + "error", err) return nil, err } } @@ -437,13 +488,9 @@ func ApplySplunkSecret(ctx context.Context, c splcommon.ControllerClient, cr spl func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.ControllerClient, namespace string) (*corev1.Secret, error) { var current corev1.Secret + logger := slog.With("func", "ApplyNamespaceScopedSecretObject") name := splcommon.GetNamespaceScopedSecretName(namespace) - log := log.FromContext(ctx) - scopedLog := log.WithName("ApplyNamespaceScopedSecretObject").WithValues( - "name", splcommon.GetNamespaceScopedSecretName(namespace), - "namespace", namespace) - // Check if a namespace scoped K8S secrets object exists namespacedName := types.NamespacedName{Namespace: namespace, Name: splcommon.GetNamespaceScopedSecretName(namespace)} err := client.Get(ctx, namespacedName, ¤t) @@ -452,7 +499,10 @@ func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.Cont var updateNeeded bool = false for _, tokenType := range splcommon.GetSplunkSecretTokenTypes() { if _, ok := current.Data[tokenType]; !ok { - scopedLog.Info("Namespace scoped secret exists, missing value for token", "missingTokenType", tokenType) + logger.WarnContext(ctx, "Secret is missing required field. Update secret with required data", + "secret_name", name, + "namespace", namespace, + "missing_field", tokenType) if current.Data == nil || reflect.ValueOf(current.Data).Kind() != reflect.Map { current.Data = make(map[string][]byte) } @@ -468,7 +518,9 @@ func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.Cont // Updated the secret if needed if updateNeeded { - scopedLog.Info("Updating namespace scoped secret due to a missing value for token") + logger.InfoContext(ctx, "Updating namespace-scoped secret with generated token values", + "secret_name", name, + "namespace", namespace) err = UpdateResource(ctx, client, ¤t) if err != nil { return nil, err @@ -477,12 +529,17 @@ func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.Cont return ¤t, nil } else if err != nil && !k8serrors.IsNotFound(err) { - // get secret call failed with other than NotFound error return the err + logger.ErrorContext(ctx, "Unexpected API error retrieving namespace-scoped secret", + "secret_name", name, + "namespace", namespace, + "error", err) return nil, err } // Make data - scopedLog.Info("Namespace scoped secret does not exist, creating and filling it with new values for all token types") + logger.InfoContext(ctx, "Namespace-scoped secret does not exist, creating with new token values", + "secret_name", name, + "namespace", namespace) current.Data = make(map[string][]byte) // Not found, update data by generating values for all types of tokens for _, tokenType := range splcommon.GetSplunkSecretTokenTypes() { @@ -508,7 +565,10 @@ func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.Cont retryCnt := 0 gerr := client.Get(ctx, namespacedName, ¤t) for ; gerr != nil; gerr = client.Get(ctx, namespacedName, ¤t) { - scopedLog.Error(gerr, "Newly created resource still not in cache sleeping for 10 micro second", "secret", name, "error", gerr.Error()) + logger.DebugContext(ctx, "Newly created secret not yet in cache, retrying", + "secret_name", name, + "namespace", namespace, + "error", gerr) time.Sleep(10 * time.Microsecond) // Avoid infinite loop @@ -520,20 +580,36 @@ func ApplyNamespaceScopedSecretObject(ctx context.Context, client splcommon.Cont return ¤t, nil } -// GetSecretByName retrieves namespace scoped secret object for a given name -func GetSecretByName(ctx context.Context, c splcommon.ControllerClient, namespace string, logHandle string, name string) (*corev1.Secret, error) { +// GetSecretByName retrieves namespace scoped secret object for a given name. +func GetSecretByName(ctx context.Context, c splcommon.ControllerClient, namespace string, name string) (*corev1.Secret, error) { var namespaceScopedSecret corev1.Secret - log := log.FromContext(ctx) - scopedLog := log.WithName("GetSecretByName").WithValues("logHandle: ", logHandle, "namespace: ", namespace) + + logger := slog.With("func", "GetSecretByName") + logger.DebugContext(ctx, "Retrieving secret", + "secret_name", name, + "namespace", namespace) // Check if a namespace scoped secret exists namespacedName := types.NamespacedName{Namespace: namespace, Name: name} err := c.Get(ctx, namespacedName, &namespaceScopedSecret) if err != nil { - // Didn't find it - scopedLog.Error(err, "Unable to get secret", "secret name", name) + if k8serrors.IsNotFound(err) { + logger.WarnContext(ctx, "Secret not found. Create secret to proceed", + "secret_name", name, + "namespace", namespace) + } else { + logger.ErrorContext(ctx, "Failed to retrieve secret", + "secret_name", name, + "namespace", namespace, + "error", err) + } return nil, err } + logger.DebugContext(ctx, "Secret retrieved successfully", + "secret_name", name, + "namespace", namespace, + "resource_version", namespaceScopedSecret.ResourceVersion) + return &namespaceScopedSecret, nil } diff --git a/pkg/splunk/util/secrets_test.go b/pkg/splunk/util/secrets_test.go index bd6817f6e..a67490dfe 100644 --- a/pkg/splunk/util/secrets_test.go +++ b/pkg/splunk/util/secrets_test.go @@ -1023,8 +1023,52 @@ func TestGetNamespaceScopedSecretByName(t *testing.T) { _, _ = ApplyNamespaceScopedSecretObject(ctx, c, "test") secretName := splcommon.GetNamespaceScopedSecretName("test") - secret, err := GetSecretByName(ctx, c, cr.GetNamespace(), cr.GetName(), secretName) + secret, err := GetSecretByName(ctx, c, cr.GetNamespace(), secretName) if secret == nil || err != nil { t.Error(err.Error()) } } + +func TestGetSecretByNameNotFound(t *testing.T) { + ctx := context.TODO() + c := spltest.NewMockClient() + + notFoundErr := k8serrors.NewNotFound( + schema.GroupResource{Group: "", Resource: "secrets"}, + "nonexistent-secret", + ) + c.InduceErrorKind[splcommon.MockClientInduceErrorGet] = notFoundErr + + secret, err := GetSecretByName(ctx, c, "test", "nonexistent-secret") + if secret != nil { + t.Error("Expected nil secret for NotFound error") + } + if err == nil { + t.Fatal("Expected error for NotFound") + } + if !k8serrors.IsNotFound(err) { + t.Errorf("Expected NotFound error, got: %v", err) + } +} + +func TestGetSecretByNameAPIError(t *testing.T) { + ctx := context.TODO() + c := spltest.NewMockClient() + + apiErr := errors.New("connection refused") + c.InduceErrorKind[splcommon.MockClientInduceErrorGet] = apiErr + + secret, err := GetSecretByName(ctx, c, "test", "some-secret") + if secret != nil { + t.Error("Expected nil secret for API error") + } + if err == nil { + t.Fatal("Expected error for API failure") + } + if k8serrors.IsNotFound(err) { + t.Error("Expected non-NotFound error") + } + if err.Error() != "connection refused" { + t.Errorf("Expected 'connection refused' error, got: %v", err) + } +} From 4da56b70acecefc47463707fb1568dfade0dd7e8 Mon Sep 17 00:00:00 2001 From: Kasia Koziol Date: Fri, 27 Feb 2026 08:33:20 +0100 Subject: [PATCH 03/19] Fixes --- cmd/main.go | 1 - pkg/splunk/enterprise/indexercluster_test.go | 8 +++----- pkg/splunk/enterprise/licensemanager.go | 2 +- pkg/splunk/enterprise/telemetry.go | 2 +- pkg/splunk/enterprise/util.go | 2 +- 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 8776b2720..7fe545b29 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -32,7 +32,6 @@ import ( "github.com/splunk/splunk-operator/internal/controller/debug" "github.com/splunk/splunk-operator/pkg/config" "github.com/splunk/splunk-operator/pkg/logging" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "github.com/splunk/splunk-operator/pkg/splunk/enterprise/validation" "sigs.k8s.io/controller-runtime/pkg/certwatcher" diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index ad97ab2e8..0629e27ec 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -1588,11 +1588,9 @@ func TestIndexerClusterWithReadyState(t *testing.T) { mclient.AddHandler(wantRequest2, 200, string(response2), nil) mclient.AddHandler(wantRequest3, 200, string(response3), nil) - // Mock GetSpecificSecretTokenFromPod to return a dummy password - // This allows VerifyRFPeers to execute its real logic with HTTP calls mocked via MockHTTPClient - savedGetSpecificSecretTokenFromPod := splutil.GetSpecificSecretTokenFromPod - defer func() { splutil.GetSpecificSecretTokenFromPod = savedGetSpecificSecretTokenFromPod }() - splutil.GetSpecificSecretTokenFromPod = func(ctx context.Context, c splcommon.ControllerClient, podName string, namespace string, secretToken string) (string, error) { + savedGetSpecificSecretTokenFromPod := splutil.GetSpecificSecretTokenFromPodMock + defer func() { splutil.GetSpecificSecretTokenFromPodMock = savedGetSpecificSecretTokenFromPod }() + splutil.GetSpecificSecretTokenFromPodMock = func(ctx context.Context, c splcommon.ControllerClient, podName string, namespace string, secretToken string) (string, error) { return "dummypassword", nil } diff --git a/pkg/splunk/enterprise/licensemanager.go b/pkg/splunk/enterprise/licensemanager.go index 49e01e0cc..dd580433b 100644 --- a/pkg/splunk/enterprise/licensemanager.go +++ b/pkg/splunk/enterprise/licensemanager.go @@ -262,7 +262,7 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro // Get admin password from namespace-scoped secret defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) - defaultSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), defaultSecretObjName) + defaultSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), defaultSecretObjName) if err != nil { return fmt.Errorf("failed to get namespace secret for license check: %w", err) } diff --git a/pkg/splunk/enterprise/telemetry.go b/pkg/splunk/enterprise/telemetry.go index a095e1e6e..3d356fc8e 100644 --- a/pkg/splunk/enterprise/telemetry.go +++ b/pkg/splunk/enterprise/telemetry.go @@ -471,7 +471,7 @@ func SendTelemetry(ctx context.Context, client splcommon.ControllerClient, cr sp scopedLog.Info("Got service FQDN", "serviceFQDN", serviceFQDN) defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) - defaultSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), defaultSecretObjName) + defaultSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), defaultSecretObjName) if err != nil { scopedLog.Error(err, "Could not access default secret object") return false diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index 406028ca0..4b5d77e8f 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -456,7 +456,7 @@ func GetSmartstoreRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi. // GetQueueRemoteVolumeSecrets is used to retrieve access key and secrete key for Index & Ingestion separation func GetQueueRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi.VolumeSpec, client splcommon.ControllerClient, cr splcommon.MetaObject) (string, string, string, error) { - namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), volume.SecretRef) + namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), volume.SecretRef) if err != nil { return "", "", "", err } From e7f00eb0b315b3f15d43e9731c9eac70e9dd421a Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Thu, 26 Feb 2026 09:45:06 +0100 Subject: [PATCH 04/19] Add logs to controllers --- cmd/main.go | 10 ++- .../controller/clustermanager_controller.go | 6 ++ .../controller/clustermaster_controller.go | 6 ++ .../controller/indexercluster_controller.go | 6 ++ .../controller/licensemanager_controller.go | 6 ++ .../controller/licensemaster_controller.go | 6 ++ .../monitoringconsole_controller.go | 7 ++ .../searchheadcluster_controller.go | 6 ++ internal/controller/standalone_controller.go | 6 ++ pkg/splunk/enterprise/configuration.go | 85 ++++++++----------- pkg/splunk/enterprise/finalizers.go | 14 ++- pkg/splunk/enterprise/searchheadcluster.go | 2 +- .../enterprise/searchheadcluster_test.go | 18 ++-- .../enterprise/searchheadclusterpodmanager.go | 76 +++++++++++------ pkg/splunk/enterprise/standalone.go | 17 ++-- 15 files changed, 164 insertions(+), 107 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7fe545b29..d3e2a716e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -187,8 +187,6 @@ func main() { slog.String("log_format", logCfg.Format), slog.Bool("log_add_source", logCfg.AddSource)) - _ = logger // logger is available for future use - // Logging setup (existing zap logger for controller-runtime compatibility) ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -237,6 +235,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("clustermanager-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { fmt.Printf(" error - %v", err) setupLog.Error(err, "unable to create controller", "controller", "ClusterManager ") @@ -247,6 +246,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("clustermaster-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterMaster") os.Exit(1) @@ -255,6 +255,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("indexercluster-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "IndexerCluster") os.Exit(1) @@ -263,6 +264,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("licensemaster-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LicenseMaster") os.Exit(1) @@ -271,6 +273,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("licensemanager-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LicenseManager") os.Exit(1) @@ -279,6 +282,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("monitoringconsole-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "MonitoringConsole") os.Exit(1) @@ -287,6 +291,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("searchheadcluster-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SearchHeadCluster") os.Exit(1) @@ -295,6 +300,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("standalone-controller"), + Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Standalone") os.Exit(1) diff --git a/internal/controller/clustermanager_controller.go b/internal/controller/clustermanager_controller.go index 2a844bd0a..42ee15b7b 100644 --- a/internal/controller/clustermanager_controller.go +++ b/internal/controller/clustermanager_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -47,6 +49,7 @@ type ClusterManagerReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=clustermanagers,verbs=get;list;watch;create;update;patch;delete @@ -79,6 +82,8 @@ func (r *ClusterManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterManager") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("clustermanager", req.NamespacedName) @@ -124,6 +129,7 @@ var ApplyClusterManager = func(ctx context.Context, client client.Client, instan } func (r *ClusterManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "ClusterManager") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.ClusterManager{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/clustermaster_controller.go b/internal/controller/clustermaster_controller.go index 4e2b5b94a..46015861d 100644 --- a/internal/controller/clustermaster_controller.go +++ b/internal/controller/clustermaster_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -47,6 +49,7 @@ type ClusterMasterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=clustermasters,verbs=get;list;watch;create;update;patch;delete @@ -79,6 +82,8 @@ func (r *ClusterMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterMaster") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("clustermaster", req.NamespacedName) @@ -125,6 +130,7 @@ var ApplyClusterMaster = func(ctx context.Context, client client.Client, instanc // SetupWithManager sets up the controller with the Manager. func (r *ClusterMasterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "ClusterMaster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApiV3.ClusterMaster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/indexercluster_controller.go b/internal/controller/indexercluster_controller.go index ddc8b17c9..589e82f30 100644 --- a/internal/controller/indexercluster_controller.go +++ b/internal/controller/indexercluster_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" "github.com/pkg/errors" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" @@ -48,6 +50,7 @@ type IndexerClusterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=indexerclusters,verbs=get;list;watch;create;update;patch;delete @@ -79,6 +82,8 @@ func (r *IndexerClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "IndexerCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "IndexerCluster") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("indexercluster", req.NamespacedName) @@ -129,6 +134,7 @@ var ApplyIndexerCluster = func(ctx context.Context, client client.Client, instan // SetupWithManager sets up the controller with the Manager. func (r *IndexerClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "IndexerCluster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.IndexerCluster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/licensemanager_controller.go b/internal/controller/licensemanager_controller.go index 27e39a7f5..c79f6b305 100644 --- a/internal/controller/licensemanager_controller.go +++ b/internal/controller/licensemanager_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -46,6 +48,7 @@ type LicenseManagerReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=licensemanagers,verbs=get;list;watch;create;update;patch;delete @@ -77,6 +80,8 @@ func (r *LicenseManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseManager") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("licensemanager", req.NamespacedName) @@ -123,6 +128,7 @@ var ApplyLicenseManager = func(ctx context.Context, client client.Client, instan // SetupWithManager sets up the controller with the Manager. func (r *LicenseManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "LicenseManager") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.LicenseManager{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/licensemaster_controller.go b/internal/controller/licensemaster_controller.go index c413cab50..28dd84617 100644 --- a/internal/controller/licensemaster_controller.go +++ b/internal/controller/licensemaster_controller.go @@ -18,9 +18,11 @@ package controller import ( "context" + "log/slog" "time" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -47,6 +49,7 @@ type LicenseMasterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=licensemasters,verbs=get;list;watch;create;update;patch;delete @@ -78,6 +81,8 @@ func (r *LicenseMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseMaster") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("licensemaster", req.NamespacedName) @@ -124,6 +129,7 @@ var ApplyLicenseMaster = func(ctx context.Context, client client.Client, instanc // SetupWithManager sets up the controller with the Manager. func (r *LicenseMasterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "LicenseMaster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApiV3.LicenseMaster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/monitoringconsole_controller.go b/internal/controller/monitoringconsole_controller.go index 571e0f9f8..7ebc51904 100644 --- a/internal/controller/monitoringconsole_controller.go +++ b/internal/controller/monitoringconsole_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -47,6 +49,7 @@ type MonitoringConsoleReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=monitoringconsoles,verbs=get;list;watch;create;update;patch;delete @@ -77,6 +80,9 @@ type MonitoringConsoleReconciler struct { func (r *MonitoringConsoleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "MonitoringConsole")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "MonitoringConsole") + + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("monitoringconsole", req.NamespacedName) @@ -123,6 +129,7 @@ var ApplyMonitoringConsole = func(ctx context.Context, client client.Client, ins // SetupWithManager sets up the controller with the Manager. func (r *MonitoringConsoleReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "MonitoringConsole") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.MonitoringConsole{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/searchheadcluster_controller.go b/internal/controller/searchheadcluster_controller.go index 9ff2eca36..3637cda8f 100644 --- a/internal/controller/searchheadcluster_controller.go +++ b/internal/controller/searchheadcluster_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "github.com/pkg/errors" @@ -46,6 +48,7 @@ type SearchHeadClusterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=searchheadclusters,verbs=get;list;watch;create;update;patch;delete @@ -77,6 +80,8 @@ func (r *SearchHeadClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "SearchHeadCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "SearchHeadCluster") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("searchheadcluster", req.NamespacedName) @@ -123,6 +128,7 @@ var ApplySearchHeadCluster = func(ctx context.Context, client client.Client, ins // SetupWithManager sets up the controller with the Manager. func (r *SearchHeadClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "SearchHeadCluster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.SearchHeadCluster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/standalone_controller.go b/internal/controller/standalone_controller.go index bb7106f05..443af71ce 100644 --- a/internal/controller/standalone_controller.go +++ b/internal/controller/standalone_controller.go @@ -18,10 +18,12 @@ package controller import ( "context" + "log/slog" "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" appsv1 "k8s.io/api/apps/v1" @@ -51,6 +53,7 @@ type StandaloneReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder + Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=standalones,verbs=get;list;watch;create;update;patch;delete @@ -82,6 +85,8 @@ func (r *StandaloneReconciler) Reconcile(ctx context.Context, req ctrl.Request) metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "Standalone")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "Standalone") + ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) + reqLogger := log.FromContext(ctx) reqLogger = reqLogger.WithValues("standalone", req.NamespacedName) @@ -128,6 +133,7 @@ var ApplyStandalone = func(ctx context.Context, client client.Client, instance * // SetupWithManager sets up the controller with the Manager. func (r *StandaloneReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.Logger = r.Logger.With("controller", "Standalone") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.Standalone{}). WithEventFilter(predicate.Or( diff --git a/pkg/splunk/enterprise/configuration.go b/pkg/splunk/enterprise/configuration.go index cb9322f0a..f3d52bf14 100644 --- a/pkg/splunk/enterprise/configuration.go +++ b/pkg/splunk/enterprise/configuration.go @@ -33,14 +33,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/log" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" - "sigs.k8s.io/controller-runtime/pkg/log" ) var defaultLivenessProbe corev1.Probe = corev1.Probe{ @@ -423,8 +424,7 @@ func validateCommonSplunkSpec(ctx context.Context, c splcommon.ControllerClient, // ValidateImagePullSecrets sets default values for imagePullSecrets if not provided func ValidateImagePullSecrets(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, spec *enterpriseApi.CommonSplunkSpec) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ValidateImagePullSecrets").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "ValidateImagePullSecrets") // If no imagePullSecrets are configured var nilImagePullSecrets []corev1.LocalObjectReference @@ -437,7 +437,7 @@ func ValidateImagePullSecrets(ctx context.Context, c splcommon.ControllerClient, for _, secret := range spec.ImagePullSecrets { _, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), secret.Name) if err != nil { - scopedLog.Error(err, "Couldn't get secret in the imagePullSecrets config", "Secret", secret.Name) + logger.ErrorContext(ctx, "Couldn't get secret in the imagePullSecrets config", "Secret", secret.Name, "error", err) } } @@ -578,8 +578,7 @@ func addEphemeralVolumes(statefulSet *appsv1.StatefulSet, volumeType string) err // addStorageVolumes adds storage volumes to the StatefulSet func addStorageVolumes(ctx context.Context, cr splcommon.MetaObject, client splcommon.ControllerClient, spec *enterpriseApi.CommonSplunkSpec, statefulSet *appsv1.StatefulSet, labels map[string]string) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("addStorageVolumes").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "addStorageVolumes") // configure storage for mount path /opt/splunk/etc if spec.EtcVolumeStorageConfig.EphemeralStorage { @@ -608,7 +607,7 @@ func addStorageVolumes(ctx context.Context, cr splcommon.MetaObject, client splc // Add Splunk Probe config map probeConfigMap, err := getProbeConfigMap(ctx, client, cr) if err != nil { - scopedLog.Error(err, "Unable to get probeConfigMap") + logger.ErrorContext(ctx, "Unable to get probeConfigMap", "error", err) return err } addProbeConfigMapVolume(probeConfigMap, statefulSet) @@ -617,28 +616,27 @@ func addStorageVolumes(ctx context.Context, cr splcommon.MetaObject, client splc func getProbeConfigMap(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject) (*corev1.ConfigMap, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getProbeConfigMap").WithValues("namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getProbeConfigMap") configMapName := GetProbeConfigMapName(cr.GetNamespace()) configMapNamespace := cr.GetNamespace() namespacedName := types.NamespacedName{Namespace: configMapNamespace, Name: configMapName} // Check if the config map already exists - scopedLog.Info("Checking for existing config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) + logger.DebugContext(ctx, "Checking for existing config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) var configMap corev1.ConfigMap err := client.Get(ctx, namespacedName, &configMap) if err == nil { - scopedLog.Info("Retrieved existing config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) + logger.DebugContext(ctx, "Retrieved existing config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) return &configMap, nil } else if !k8serrors.IsNotFound(err) { - scopedLog.Error(err, "Error retrieving config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) + logger.ErrorContext(ctx, "Error retrieving config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace, "error", err) return nil, err } // Existing config map not found, create one for the probes - scopedLog.Info("Creating new config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) + logger.InfoContext(ctx, "Creating new config map", "configMapName", configMapName, "configMapNamespace", configMapNamespace) configMap = corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: configMapName, @@ -813,8 +811,7 @@ func getSmartstoreConfigMap(ctx context.Context, client splcommon.ControllerClie // updateSplunkPodTemplateWithConfig modifies the podTemplateSpec object based on configuration of the Splunk Enterprise resource. func updateSplunkPodTemplateWithConfig(ctx context.Context, client splcommon.ControllerClient, podTemplateSpec *corev1.PodTemplateSpec, cr splcommon.MetaObject, spec *enterpriseApi.CommonSplunkSpec, instanceType InstanceType, extraEnv []corev1.EnvVar, secretToMount string) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateSplunkPodTemplateWithConfig").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "updateSplunkPodTemplateWithConfig") // Add custom ports to splunk containers if spec.ServiceTemplate.Spec.Ports != nil { for idx := range podTemplateSpec.Spec.Containers { @@ -874,7 +871,7 @@ func updateSplunkPodTemplateWithConfig(ctx context.Context, client splcommon.Con if err == nil { podTemplateSpec.ObjectMeta.Annotations["defaultConfigRev"] = configMapResourceVersion } else { - scopedLog.Error(err, "Updation of default configMap annotation failed") + logger.ErrorContext(ctx, "Updation of default configMap annotation failed", "error", err) } } @@ -995,7 +992,7 @@ func updateSplunkPodTemplateWithConfig(ctx context.Context, client splcommon.Con managerIdxCluster := &enterpriseApi.ClusterManager{} err := client.Get(ctx, namespacedName, managerIdxCluster) if err != nil { - scopedLog.Error(err, "Unable to get ClusterManager") + logger.ErrorContext(ctx, "Unable to get ClusterManager", "error", err) } if managerIdxCluster.Spec.LicenseManagerRef.Name != "" { @@ -1032,7 +1029,7 @@ func updateSplunkPodTemplateWithConfig(ctx context.Context, client splcommon.Con managerIdxCluster := &enterpriseApiV3.ClusterMaster{} err := client.Get(ctx, namespacedName, managerIdxCluster) if err != nil { - scopedLog.Error(err, "Unable to get ClusterManager") + logger.ErrorContext(ctx, "Unable to get ClusterMaster", "error", err) } if managerIdxCluster.Spec.LicenseManagerRef.Name != "" { @@ -1129,28 +1126,25 @@ func removeDuplicateEnvVars(sliceList []corev1.EnvVar) []corev1.EnvVar { // getLivenessProbe the probe for checking the liveness of the Pod func getLivenessProbe(ctx context.Context, cr splcommon.MetaObject, instanceType InstanceType, spec *enterpriseApi.CommonSplunkSpec) *corev1.Probe { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getLivenessProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx) livenessProbe := getProbeWithConfigUpdates(&defaultLivenessProbe, spec.LivenessProbe, spec.LivenessInitialDelaySeconds) - scopedLog.Info("LivenessProbe", "Configured", livenessProbe) + logger.DebugContext(ctx, "LivenessProbe", "Configured", livenessProbe) return livenessProbe } // getReadinessProbe the probe for checking the readiness of the Pod func getReadinessProbe(ctx context.Context, cr splcommon.MetaObject, instanceType InstanceType, spec *enterpriseApi.CommonSplunkSpec) *corev1.Probe { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getReadinessProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx) readinessProbe := getProbeWithConfigUpdates(&defaultReadinessProbe, spec.ReadinessProbe, spec.ReadinessInitialDelaySeconds) - scopedLog.Info("ReadinessProbe", "Configured", readinessProbe) + logger.DebugContext(ctx, "ReadinessProbe", "Configured", readinessProbe) return readinessProbe } // getStartupProbe the probe for checking the first start of splunk on the Pod func getStartupProbe(ctx context.Context, cr splcommon.MetaObject, instanceType InstanceType, spec *enterpriseApi.CommonSplunkSpec) *corev1.Probe { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getStartupProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx) startupProbe := getProbeWithConfigUpdates(&defaultStartupProbe, spec.StartupProbe, 0) - scopedLog.Info("StartupProbe", "Configured", startupProbe) + logger.DebugContext(ctx, "StartupProbe", "Configured", startupProbe) return startupProbe } @@ -1251,8 +1245,7 @@ func AreRemoteVolumeKeysChanged(ctx context.Context, client splcommon.Controller return false } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("AreRemoteVolumeKeysChanged").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "AreRemoteVolumeKeysChanged") volList := smartstore.VolList for _, volume := range volList { @@ -1267,7 +1260,7 @@ func AreRemoteVolumeKeysChanged(ctx context.Context, client splcommon.Controller // Check if the secret version is already tracked, and if there is a change in it if existingSecretVersion, ok := ResourceRev[volume.SecretRef]; ok { if existingSecretVersion != namespaceScopedSecret.ResourceVersion { - scopedLog.Info("secret keys changed", "previous resource version", existingSecretVersion, "current version", namespaceScopedSecret.ResourceVersion) + logger.InfoContext(ctx, "secret keys changed", "previous resource version", existingSecretVersion, "current version", namespaceScopedSecret.ResourceVersion) ResourceRev[volume.SecretRef] = namespaceScopedSecret.ResourceVersion return true } @@ -1277,7 +1270,7 @@ func AreRemoteVolumeKeysChanged(ctx context.Context, client splcommon.Controller // First time adding to track the secret resource version ResourceRev[volume.SecretRef] = namespaceScopedSecret.ResourceVersion } else { - scopedLog.Info("no valid SecretRef for volume. No secret to track.", "volumeName", volume.Name) + logger.DebugContext(ctx, "no valid SecretRef for volume. No secret to track.", "volumeName", volume.Name) } } @@ -1287,8 +1280,7 @@ func AreRemoteVolumeKeysChanged(ctx context.Context, client splcommon.Controller // ApplyManualAppUpdateConfigMap applies the manual app update config map func ApplyManualAppUpdateConfigMap(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, crKindMap map[string]string) (*corev1.ConfigMap, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyManualAppUpdateConfigMap").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "ApplyManualAppUpdateConfigMap") configMapName := GetSplunkManualAppUpdateConfigMapName(cr.GetNamespace()) namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} @@ -1308,17 +1300,17 @@ func ApplyManualAppUpdateConfigMap(ctx context.Context, client splcommon.Control configMap.SetOwnerReferences(append(configMap.GetOwnerReferences(), splcommon.AsOwner(cr, false))) if newConfigMap { - scopedLog.Info("creating manual app update configMap") + logger.InfoContext(ctx, "creating manual app update configMap") err = splutil.CreateResource(ctx, client, configMap) if err != nil { - scopedLog.Error(err, "Unable to create the configMap", "name", configMapName) + logger.ErrorContext(ctx, "Unable to create the configMap", "name", configMapName, "error", err) return configMap, err } } else { - scopedLog.Info("updating manual app update configMap") + logger.InfoContext(ctx, "updating manual app update configMap") err = splutil.UpdateResource(ctx, client, configMap) if err != nil { - scopedLog.Error(err, "unable to update the configMap", "name", configMapName) + logger.ErrorContext(ctx, "unable to update the configMap", "name", configMapName, "error", err) return configMap, err } } @@ -1327,8 +1319,7 @@ func ApplyManualAppUpdateConfigMap(ctx context.Context, client splcommon.Control // getManualUpdateStatus extracts the status field from the configMap data func getManualUpdateStatus(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, configMapName string) string { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getManualUpdateStatus").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getManualUpdateStatus") namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} configMap, err := splctrl.GetConfigMap(ctx, client, namespacedName) @@ -1338,11 +1329,11 @@ func getManualUpdateStatus(ctx context.Context, client splcommon.ControllerClien data := configMap.Data[cr.GetObjectKind().GroupVersionKind().Kind] result = extractFieldFromConfigMapData(statusRegex, data) if result == "on" { - scopedLog.Info("namespace configMap value is set to", "name", configMapName, "data", result) + logger.InfoContext(ctx, "namespace configMap value is set to", "name", configMapName, "data", result) return result } } else { - scopedLog.Error(err, "Unable to get namespace specific configMap", "name", configMapName) + logger.ErrorContext(ctx, "Unable to get namespace specific configMap", "name", configMapName, "error", err) } return "off" @@ -1350,17 +1341,16 @@ func getManualUpdateStatus(ctx context.Context, client splcommon.ControllerClien // getManualUpdatePerCrStatus extracts the status field from the configMap data func getManualUpdatePerCrStatus(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, configMapName string) string { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getManualUpdatePerCrStatus").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getManualUpdatePerCrStatus") namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: fmt.Sprintf(perCrConfigMapNameStr, KindToInstanceString(cr.GroupVersionKind().Kind), cr.GetName())} crconfigMap, err := splctrl.GetConfigMap(ctx, client, namespacedName) if err == nil { - scopedLog.Info("custom configMap value is set to", "name", configMapName, "data", crconfigMap.Data) + logger.InfoContext(ctx, "custom configMap value is set to", "name", configMapName, "data", crconfigMap.Data) data := crconfigMap.Data["manualUpdate"] return data } else { - scopedLog.Error(err, "unable to get custom specific configMap", "name", configMapName) + logger.ErrorContext(ctx, "unable to get custom specific configMap", "name", configMapName, "error", err) } return "off" @@ -1368,13 +1358,12 @@ func getManualUpdatePerCrStatus(ctx context.Context, client splcommon.Controller // getManualUpdateRefCount extracts the refCount field from the configMap data func getManualUpdateRefCount(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, configMapName string) int { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getManualUpdateRefCount").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getManualUpdateRefCount") var refCount int namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} configMap, err := splctrl.GetConfigMap(ctx, client, namespacedName) if err != nil { - scopedLog.Error(err, "unable to get the configMap", "name", configMapName) + logger.ErrorContext(ctx, "unable to get the configMap", "name", configMapName, "error", err) return refCount } diff --git a/pkg/splunk/enterprise/finalizers.go b/pkg/splunk/enterprise/finalizers.go index 9ecbd0136..d9858602d 100644 --- a/pkg/splunk/enterprise/finalizers.go +++ b/pkg/splunk/enterprise/finalizers.go @@ -19,12 +19,11 @@ import ( "context" "fmt" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) func init() { @@ -35,8 +34,7 @@ func init() { func DeleteSplunkPvc(ctx context.Context, cr splcommon.MetaObject, c splcommon.ControllerClient) error { objectKind := cr.GetObjectKind().GroupVersionKind().Kind - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DeleteSplunkPvc") + logger := logging.FromContext(ctx).With("func", "DeleteSplunkPvc") var components []string switch objectKind { @@ -59,7 +57,7 @@ func DeleteSplunkPvc(ctx context.Context, cr splcommon.MetaObject, c splcommon.C case "IngestorCluster": components = append(components, "ingestor") default: - scopedLog.Info("Skipping PVC removal") + logger.DebugContext(ctx, "Skipping PVC removal") return nil } @@ -79,7 +77,7 @@ func DeleteSplunkPvc(ctx context.Context, cr splcommon.MetaObject, c splcommon.C // delete each PVC for _, pvc := range pvclist.Items { - scopedLog.Info("Deleting PVC", "name", pvc.ObjectMeta.Name) + logger.InfoContext(ctx, "Deleting PVC", "name", pvc.ObjectMeta.Name) if err := c.Delete(context.Background(), &pvc); err != nil { return err } diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index f6b9cbcfb..713ded930 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -196,7 +196,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie return result, err } - mgr := newSearchHeadClusterPodManager(client, scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient) + mgr := newSearchHeadClusterPodManager(client, cr, namespaceScopedSecret, splclient.NewSplunkClient) // handle SHC upgrade process phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) diff --git a/pkg/splunk/enterprise/searchheadcluster_test.go b/pkg/splunk/enterprise/searchheadcluster_test.go index 3d11a539e..5f1b1abfd 100644 --- a/pkg/splunk/enterprise/searchheadcluster_test.go +++ b/pkg/splunk/enterprise/searchheadcluster_test.go @@ -41,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/go-logr/logr" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -184,7 +183,6 @@ func searchHeadClusterPodManagerTester(t *testing.T, method string, mockHandlers os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") // test for updating - scopedLog := logt.WithName(method) cr := enterpriseApi.SearchHeadCluster{ TypeMeta: metav1.TypeMeta{ Kind: "SearchHeadCluster", @@ -213,7 +211,6 @@ func searchHeadClusterPodManagerTester(t *testing.T, method string, mockHandlers mockSplunkClient.AddHandlers(mockHandlers...) mgr := &searchHeadClusterPodManager{ - log: scopedLog, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -419,7 +416,6 @@ func TestApplyShcSecret(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") ctx := context.TODO() method := "ApplyShcSecret" - scopedLog := logt.WithName(method) var initObjectList []client.Object c := spltest.NewMockClient() @@ -513,7 +509,6 @@ func TestApplyShcSecret(t *testing.T) { mockSplunkClient.AddHandlers(mockHandlers...) mgr := &searchHeadClusterPodManager{ c: c, - log: scopedLog, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -709,9 +704,8 @@ func TestShcPasswordSyncCompleted(t *testing.T) { // Initialize a minimal pod manager for ApplyShcSecret mgr := &searchHeadClusterPodManager{ - c: client, - log: logt.WithName("TestShcPasswordSyncCompleted"), - cr: &shc, + c: client, + cr: &shc, } // Use a mock PodExec client; replicas will be 0 so it won't be exercised @@ -1647,9 +1641,8 @@ func TestSearchHeadClusterWithReadyState(t *testing.T) { } // mock new search pod manager - newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { + newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { return searchHeadClusterPodManager{ - log: log, cr: cr, secrets: secret, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -2227,9 +2220,8 @@ func TestShcPasswordSyncFailedEvent(t *testing.T) { } mgr := &searchHeadClusterPodManager{ - c: c, - log: logt.WithName("TestShcPasswordSyncFailedEvent"), - cr: &shc, + c: c, + cr: &shc, } // Configure mock pod exec client to return an error on shcluster-config command diff --git a/pkg/splunk/enterprise/searchheadclusterpodmanager.go b/pkg/splunk/enterprise/searchheadclusterpodmanager.go index 7b3a19d30..956a5fd60 100644 --- a/pkg/splunk/enterprise/searchheadclusterpodmanager.go +++ b/pkg/splunk/enterprise/searchheadclusterpodmanager.go @@ -5,9 +5,9 @@ import ( "fmt" "time" - "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" metrics "github.com/splunk/splunk-operator/pkg/splunk/client/metrics" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -15,22 +15,19 @@ import ( splutil "github.com/splunk/splunk-operator/pkg/splunk/util" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/log" ) // searchHeadClusterPodManager is used to manage the pods within a search head cluster type searchHeadClusterPodManager struct { c splcommon.ControllerClient - log logr.Logger cr *enterpriseApi.SearchHeadCluster secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient } // newSerachHeadClusterPodManager function to create pod manager this is added to write unit test case -var newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log logr.Logger, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { +var newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, cr *enterpriseApi.SearchHeadCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc) searchHeadClusterPodManager { return searchHeadClusterPodManager{ - log: log, cr: cr, secrets: secret, newSplunkClient: newSplunkClient, @@ -40,6 +37,8 @@ var newSearchHeadClusterPodManager = func(client splcommon.ControllerClient, log // Update for searchHeadClusterPodManager handles all updates for a statefulset of search heads func (mgr *searchHeadClusterPodManager) Update(ctx context.Context, c splcommon.ControllerClient, statefulSet *appsv1.StatefulSet, desiredReplicas int32) (enterpriseApi.Phase, error) { + logger := logging.FromContext(ctx).With("func", "searchHeadClusterPodManager.Update") + // Assign client if mgr.c == nil { mgr.c = c @@ -69,7 +68,7 @@ func (mgr *searchHeadClusterPodManager) Update(ctx context.Context, c splcommon. // update CR status with SHC information err = mgr.updateStatus(ctx, statefulSet) if err != nil || mgr.cr.Status.ReadyReplicas == 0 || !mgr.cr.Status.Initialized || !mgr.cr.Status.CaptainReady { - mgr.log.Info("Search head cluster is not ready", "reason ", err) + logger.InfoContext(ctx, "Search head cluster is not ready", "error", err) return enterpriseApi.PhasePending, nil } @@ -99,6 +98,7 @@ func (mgr *searchHeadClusterPodManager) Update(ctx context.Context, c splcommon. // PrepareScaleDown for searchHeadClusterPodManager prepares search head pod to be removed via scale down event; it returns true when ready func (mgr *searchHeadClusterPodManager) PrepareScaleDown(ctx context.Context, n int32) (bool, error) { + logger := logging.FromContext(ctx).With("func", "PrepareScaleDown") // start by quarantining the pod result, err := mgr.PrepareRecycle(ctx, n) if err != nil || !result { @@ -107,7 +107,10 @@ func (mgr *searchHeadClusterPodManager) PrepareScaleDown(ctx context.Context, n // pod is quarantined; decommission it memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) - mgr.log.Info("Removing member from search head cluster", "memberName", memberName) + logger.WarnContext(ctx, "Member leaving search head cluster", + "member", memberName, + "remaining_count", len(mgr.cr.Status.Members)-1) + c := mgr.getClient(ctx, n) err = c.RemoveSearchHeadClusterMember() if err != nil { @@ -120,12 +123,13 @@ func (mgr *searchHeadClusterPodManager) PrepareScaleDown(ctx context.Context, n // PrepareRecycle for searchHeadClusterPodManager prepares search head pod to be recycled for updates; it returns true when ready func (mgr *searchHeadClusterPodManager) PrepareRecycle(ctx context.Context, n int32) (bool, error) { + logger := logging.FromContext(ctx).With("func", "PrepareRecycle") memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) switch mgr.cr.Status.Members[n].Status { case "Up": // Detain search head - mgr.log.Info("Detaining search head cluster member", "memberName", memberName) + logger.InfoContext(ctx, "Detaining search head cluster member", "memberName", memberName) c := mgr.getClient(ctx, n) podExecClient := splutil.GetPodExecClient(mgr.c, mgr.cr, getApplicablePodNameForK8Probes(mgr.cr, n)) @@ -136,14 +140,14 @@ func (mgr *searchHeadClusterPodManager) PrepareRecycle(ctx context.Context, n in // During the Recycle, our reconcile loop is entered multiple times. If the Pod is already down, // there is a chance of readiness probe failing, in which case, even the podExec will not be successful. // So, just log the message, and ignore the error. - mgr.log.Info("Setting Probe level failed. Probably, the Pod is already down", "memberName", memberName) + logger.WarnContext(ctx, "Setting Probe level failed. Probably, the Pod is already down", "memberName", memberName) } - mgr.log.Info("Initializes rolling upgrade process") + logger.InfoContext(ctx, "Initializes rolling upgrade process") err = c.InitiateUpgrade() if err != nil { - mgr.log.Info("Initialization of rolling upgrade failed.") + logger.ErrorContext(ctx, "Initialization of rolling upgrade failed", "error", err) return false, err } @@ -174,14 +178,14 @@ func (mgr *searchHeadClusterPodManager) PrepareRecycle(ctx context.Context, n in // Wait until active searches have drained searchesComplete := mgr.cr.Status.Members[n].ActiveHistoricalSearchCount+mgr.cr.Status.Members[n].ActiveRealtimeSearchCount == 0 if searchesComplete { - mgr.log.Info("Detention complete", "memberName", memberName) + logger.InfoContext(ctx, "Detention complete", "memberName", memberName) } else { - mgr.log.Info("Waiting for active searches to complete", "memberName", memberName) + logger.InfoContext(ctx, "Waiting for active searches to complete", "memberName", memberName) } return searchesComplete, nil case "": // this can happen after the member has already been recycled and we're just waiting for state to update - mgr.log.Info("Member has empty Status", "memberName", memberName) + logger.InfoContext(ctx, "Member has empty Status", "memberName", memberName) return false, nil } @@ -191,6 +195,7 @@ func (mgr *searchHeadClusterPodManager) PrepareRecycle(ctx context.Context, n in // FinishRecycle for searchHeadClusterPodManager completes recycle event for search head pod; it returns true when complete func (mgr *searchHeadClusterPodManager) FinishRecycle(ctx context.Context, n int32) (bool, error) { + logger := logging.FromContext(ctx).With("func", "FinishRecycle") memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) switch mgr.cr.Status.Members[n].Status { @@ -200,7 +205,7 @@ func (mgr *searchHeadClusterPodManager) FinishRecycle(ctx context.Context, n int case "ManualDetention": // release from detention - mgr.log.Info("Releasing search head cluster member from detention", "memberName", memberName) + logger.InfoContext(ctx, "Releasing search head cluster member from detention", "memberName", memberName) c := mgr.getClient(ctx, n) return false, c.SetSearchHeadDetention(false) } @@ -210,11 +215,9 @@ func (mgr *searchHeadClusterPodManager) FinishRecycle(ctx context.Context, n int } func (mgr *searchHeadClusterPodManager) FinishUpgrade(ctx context.Context, n int32) error { - - reqLogger := log.FromContext(ctx) - // check if shc is in an upgrade process if mgr.cr.Status.UpgradePhase == enterpriseApi.UpgradePhaseUpgrading { + logger := logging.FromContext(ctx).With("func", "FinishUpgrade") c := mgr.getClient(ctx, n) // stop gathering metrics @@ -226,7 +229,7 @@ func (mgr *searchHeadClusterPodManager) FinishUpgrade(ctx context.Context, n int // revert upgrade state status mgr.cr.Status.UpgradePhase = enterpriseApi.UpgradePhaseUpgraded - reqLogger.Info("Finalize Upgrade") + logger.InfoContext(ctx, "Finalize Upgrade") return c.FinalizeUpgrade() } @@ -235,9 +238,7 @@ func (mgr *searchHeadClusterPodManager) FinishUpgrade(ctx context.Context, n int // getClient for searchHeadClusterPodManager returns a SplunkClient for the member n func (mgr *searchHeadClusterPodManager) getClient(ctx context.Context, n int32) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("searchHeadClusterPodManager.getClient").WithValues("name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) - + logger := logging.FromContext(ctx).With("func", "searchHeadClusterPodManager.getClient") // Get Pod Name memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) @@ -248,7 +249,7 @@ func (mgr *searchHeadClusterPodManager) getClient(ctx context.Context, n int32) // Retrieve admin password from Pod adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, memberName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from Pod") + logger.ErrorContext(ctx, "Couldn't retrieve the admin password from Pod", "member", memberName, "error", err) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -269,12 +270,18 @@ var GetSearchHeadCaptainInfo = func(ctx context.Context, mgr *searchHeadClusterP // updateStatus for searchHeadClusterPodManager uses the REST API to update the status for a SearcHead custom resource func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statefulSet *appsv1.StatefulSet) error { // populate members status using REST API to get search head cluster member info + previousCaptain := mgr.cr.Status.Captain + previousMemberCount := int32(len(mgr.cr.Status.Members)) + mgr.cr.Status.Captain = "" mgr.cr.Status.CaptainReady = false mgr.cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas if mgr.cr.Status.ReadyReplicas == 0 { return nil } + + shcLogger := logging.FromContext(ctx) + gotCaptainInfo := false for n := int32(0); n < statefulSet.Status.Replicas; n++ { memberName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), n) @@ -287,7 +294,7 @@ func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statef memberStatus.ActiveHistoricalSearchCount = memberInfo.ActiveHistoricalSearchCount memberStatus.ActiveRealtimeSearchCount = memberInfo.ActiveRealtimeSearchCount } else { - mgr.log.Error(err, "Unable to retrieve search head cluster member info", "memberName", memberName) + shcLogger.ErrorContext(ctx, "Unable to retrieve search head cluster member info", "memberName", memberName, "error", err) } if err == nil && !gotCaptainInfo { @@ -300,9 +307,17 @@ func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statef mgr.cr.Status.MinPeersJoined = captainInfo.MinPeersJoined mgr.cr.Status.MaintenanceMode = captainInfo.MaintenanceMode gotCaptainInfo = true + + if previousCaptain != "" && previousCaptain != captainInfo.Label { + shcLogger.InfoContext(ctx, "Captain election completed", + "old_captain", previousCaptain, + "new_captain", captainInfo.Label) + } } else { mgr.cr.Status.CaptainReady = false - mgr.log.Error(err, "Unable to retrieve captain info", "memberName", memberName) + shcLogger.ErrorContext(ctx, "Captain election failed", + "member", memberName, + "error", err) } } @@ -318,5 +333,16 @@ func (mgr *searchHeadClusterPodManager) updateStatus(ctx context.Context, statef mgr.cr.Status.Members = mgr.cr.Status.Members[:statefulSet.Status.Replicas] } + newMemberCount := int32(len(mgr.cr.Status.Members)) + if newMemberCount > previousMemberCount { + shcLogger.InfoContext(ctx, "Member joined search head cluster", + "total_members", newMemberCount, + "previous_members", previousMemberCount) + } else if newMemberCount < previousMemberCount { + shcLogger.WarnContext(ctx, "Member left search head cluster", + "total_members", newMemberCount, + "previous_members", previousMemberCount) + } + return nil } diff --git a/pkg/splunk/enterprise/standalone.go b/pkg/splunk/enterprise/standalone.go index dd498ce33..288f383be 100644 --- a/pkg/splunk/enterprise/standalone.go +++ b/pkg/splunk/enterprise/standalone.go @@ -23,6 +23,7 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" @@ -30,7 +31,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -43,8 +43,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyStandalone") + logger := logging.FromContext(ctx).With("func", "ApplyStandalone") if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) } @@ -64,7 +63,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr err = validateStandaloneSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateStandaloneSpec", fmt.Sprintf("validate standalone spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate standalone spec") + logger.ErrorContext(ctx, "Failed to validate standalone spec", "error", err) return result, err } @@ -110,7 +109,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr // create or update general config resources _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkStandalone) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) + logger.ErrorContext(ctx, "create or update general config failed", "error", err) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) return result, err } @@ -259,7 +258,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { eventPublisher.Warning(ctx, "DeleteReferencesToAutomatedMCIfExists", fmt.Sprintf("delete reference to automated MC if exists failed %s", err.Error())) - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } finalResult := handleAppFrameworkActivity(ctx, client, cr, &cr.Status.AppContext, &cr.Spec.AppFrameworkConfig) @@ -334,14 +333,12 @@ func validateStandaloneSpec(ctx context.Context, c splcommon.ControllerClient, c // helper function to get the list of Standalone types in the current namespace func getStandaloneList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []client.ListOption) (enterpriseApi.StandaloneList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getStandaloneList") - + logger := logging.FromContext(ctx).With("func", "getStandaloneList") objectList := enterpriseApi.StandaloneList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "Standalone types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "Standalone types not found in namespace", "namespace", cr.GetNamespace(), "error", err) return objectList, err } From 611d2c760f0189b391770aa17a72134e93210cae Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Mon, 2 Mar 2026 12:53:14 +0100 Subject: [PATCH 05/19] Use slog.default --- cmd/main.go | 10 +--------- internal/controller/clustermanager_controller.go | 13 ++++--------- internal/controller/clustermaster_controller.go | 13 ++++--------- internal/controller/indexercluster_controller.go | 13 ++++--------- internal/controller/licensemanager_controller.go | 13 ++++--------- internal/controller/licensemaster_controller.go | 13 ++++--------- internal/controller/monitoringconsole_controller.go | 13 ++++--------- internal/controller/searchheadcluster_controller.go | 13 ++++--------- internal/controller/standalone_controller.go | 13 ++++--------- 9 files changed, 33 insertions(+), 81 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d3e2a716e..5f00ea5f9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -173,7 +173,7 @@ func main() { addSourcePtr = &logAddSource } logCfg := logging.LoadConfigWithFlags(logLevel, logFormat, addSourcePtr) - logger := logging.SetupLogger(logCfg, + _ = logging.SetupLogger(logCfg, slog.String("component", "splunk-operator"), slog.String("version", version), slog.String("build", gitCommit)) @@ -235,7 +235,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("clustermanager-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { fmt.Printf(" error - %v", err) setupLog.Error(err, "unable to create controller", "controller", "ClusterManager ") @@ -246,7 +245,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("clustermaster-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterMaster") os.Exit(1) @@ -255,7 +253,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("indexercluster-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "IndexerCluster") os.Exit(1) @@ -264,7 +261,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("licensemaster-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LicenseMaster") os.Exit(1) @@ -273,7 +269,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("licensemanager-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LicenseManager") os.Exit(1) @@ -282,7 +277,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("monitoringconsole-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "MonitoringConsole") os.Exit(1) @@ -291,7 +285,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("searchheadcluster-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "SearchHeadCluster") os.Exit(1) @@ -300,7 +293,6 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("standalone-controller"), - Logger: logger, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Standalone") os.Exit(1) diff --git a/internal/controller/clustermanager_controller.go b/internal/controller/clustermanager_controller.go index 42ee15b7b..ec3626f05 100644 --- a/internal/controller/clustermanager_controller.go +++ b/internal/controller/clustermanager_controller.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -49,7 +48,6 @@ type ClusterManagerReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=clustermanagers,verbs=get;list;watch;create;update;patch;delete @@ -82,10 +80,8 @@ func (r *ClusterManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterManager") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("clustermanager", req.NamespacedName) + logger := slog.Default().With("controller", "ClusterManager", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the ClusterManager instance := &enterpriseApi.ClusterManager{} @@ -110,14 +106,14 @@ func (r *ClusterManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyClusterManager(ctx, r.Client, instance, nil) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -129,7 +125,6 @@ var ApplyClusterManager = func(ctx context.Context, client client.Client, instan } func (r *ClusterManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "ClusterManager") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.ClusterManager{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/clustermaster_controller.go b/internal/controller/clustermaster_controller.go index 46015861d..4b14eede1 100644 --- a/internal/controller/clustermaster_controller.go +++ b/internal/controller/clustermaster_controller.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -49,7 +48,6 @@ type ClusterMasterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=clustermasters,verbs=get;list;watch;create;update;patch;delete @@ -82,10 +80,8 @@ func (r *ClusterMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterMaster") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("clustermaster", req.NamespacedName) + logger := slog.Default().With("controller", "ClusterMaster", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the ClusterMaster instance := &enterpriseApiV3.ClusterMaster{} @@ -110,14 +106,14 @@ func (r *ClusterMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyClusterMaster(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -130,7 +126,6 @@ var ApplyClusterMaster = func(ctx context.Context, client client.Client, instanc // SetupWithManager sets up the controller with the Manager. func (r *ClusterMasterReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "ClusterMaster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApiV3.ClusterMaster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/indexercluster_controller.go b/internal/controller/indexercluster_controller.go index 589e82f30..975e31114 100644 --- a/internal/controller/indexercluster_controller.go +++ b/internal/controller/indexercluster_controller.go @@ -40,7 +40,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -50,7 +49,6 @@ type IndexerClusterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=indexerclusters,verbs=get;list;watch;create;update;patch;delete @@ -82,10 +80,8 @@ func (r *IndexerClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "IndexerCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "IndexerCluster") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("indexercluster", req.NamespacedName) + logger := slog.Default().With("controller", "IndexerCluster", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the IndexerCluster instance := &enterpriseApi.IndexerCluster{} @@ -110,14 +106,14 @@ func (r *IndexerClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyIndexerCluster(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -134,7 +130,6 @@ var ApplyIndexerCluster = func(ctx context.Context, client client.Client, instan // SetupWithManager sets up the controller with the Manager. func (r *IndexerClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "IndexerCluster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.IndexerCluster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/licensemanager_controller.go b/internal/controller/licensemanager_controller.go index c79f6b305..41f759864 100644 --- a/internal/controller/licensemanager_controller.go +++ b/internal/controller/licensemanager_controller.go @@ -38,7 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -48,7 +47,6 @@ type LicenseManagerReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=licensemanagers,verbs=get;list;watch;create;update;patch;delete @@ -80,10 +78,8 @@ func (r *LicenseManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseManager") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("licensemanager", req.NamespacedName) + logger := slog.Default().With("controller", "LicenseManager", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the LicenseManager instance := &enterpriseApi.LicenseManager{} @@ -108,14 +104,14 @@ func (r *LicenseManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyLicenseManager(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -128,7 +124,6 @@ var ApplyLicenseManager = func(ctx context.Context, client client.Client, instan // SetupWithManager sets up the controller with the Manager. func (r *LicenseManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "LicenseManager") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.LicenseManager{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/licensemaster_controller.go b/internal/controller/licensemaster_controller.go index 28dd84617..545c344e1 100644 --- a/internal/controller/licensemaster_controller.go +++ b/internal/controller/licensemaster_controller.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -49,7 +48,6 @@ type LicenseMasterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=licensemasters,verbs=get;list;watch;create;update;patch;delete @@ -81,10 +79,8 @@ func (r *LicenseMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseMaster") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("licensemaster", req.NamespacedName) + logger := slog.Default().With("controller", "LicenseMaster", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the LicenseMaster instance := &enterpriseApiV3.LicenseMaster{} @@ -109,14 +105,14 @@ func (r *LicenseMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyLicenseMaster(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -129,7 +125,6 @@ var ApplyLicenseMaster = func(ctx context.Context, client client.Client, instanc // SetupWithManager sets up the controller with the Manager. func (r *LicenseMasterReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "LicenseMaster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApiV3.LicenseMaster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/monitoringconsole_controller.go b/internal/controller/monitoringconsole_controller.go index 7ebc51904..5507d9802 100644 --- a/internal/controller/monitoringconsole_controller.go +++ b/internal/controller/monitoringconsole_controller.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -49,7 +48,6 @@ type MonitoringConsoleReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=monitoringconsoles,verbs=get;list;watch;create;update;patch;delete @@ -81,10 +79,8 @@ func (r *MonitoringConsoleReconciler) Reconcile(ctx context.Context, req ctrl.Re metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "MonitoringConsole")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "MonitoringConsole") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("monitoringconsole", req.NamespacedName) + logger := slog.Default().With("controller", "MonitoringConsole", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the MonitoringConsole instance := &enterpriseApi.MonitoringConsole{} @@ -109,14 +105,14 @@ func (r *MonitoringConsoleReconciler) Reconcile(ctx context.Context, req ctrl.Re } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyMonitoringConsole(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -129,7 +125,6 @@ var ApplyMonitoringConsole = func(ctx context.Context, client client.Client, ins // SetupWithManager sets up the controller with the Manager. func (r *MonitoringConsoleReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "MonitoringConsole") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.MonitoringConsole{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/searchheadcluster_controller.go b/internal/controller/searchheadcluster_controller.go index 3637cda8f..439389e77 100644 --- a/internal/controller/searchheadcluster_controller.go +++ b/internal/controller/searchheadcluster_controller.go @@ -38,7 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -48,7 +47,6 @@ type SearchHeadClusterReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=searchheadclusters,verbs=get;list;watch;create;update;patch;delete @@ -80,10 +78,8 @@ func (r *SearchHeadClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "SearchHeadCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "SearchHeadCluster") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("searchheadcluster", req.NamespacedName) + logger := slog.Default().With("controller", "SearchHeadCluster", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the SearchHeadCluster instance := &enterpriseApi.SearchHeadCluster{} @@ -108,14 +104,14 @@ func (r *SearchHeadClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplySearchHeadCluster(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -128,7 +124,6 @@ var ApplySearchHeadCluster = func(ctx context.Context, client client.Client, ins // SetupWithManager sets up the controller with the Manager. func (r *SearchHeadClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "SearchHeadCluster") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.SearchHeadCluster{}). WithEventFilter(predicate.Or( diff --git a/internal/controller/standalone_controller.go b/internal/controller/standalone_controller.go index 443af71ce..2c69730ae 100644 --- a/internal/controller/standalone_controller.go +++ b/internal/controller/standalone_controller.go @@ -41,7 +41,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/log" ) const ( @@ -53,7 +52,6 @@ type StandaloneReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - Logger *slog.Logger } //+kubebuilder:rbac:groups=enterprise.splunk.com,resources=standalones,verbs=get;list;watch;create;update;patch;delete @@ -85,10 +83,8 @@ func (r *StandaloneReconciler) Reconcile(ctx context.Context, req ctrl.Request) metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "Standalone")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "Standalone") - ctx = logging.WithLogger(ctx, r.Logger.With("name", req.Name, "namespace", req.Namespace)) - - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("standalone", req.NamespacedName) + logger := slog.Default().With("controller", "Standalone", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the Standalone instance := &enterpriseApi.Standalone{} @@ -113,14 +109,14 @@ func (r *StandaloneReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyStandalone(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err @@ -133,7 +129,6 @@ var ApplyStandalone = func(ctx context.Context, client client.Client, instance * // SetupWithManager sets up the controller with the Manager. func (r *StandaloneReconciler) SetupWithManager(mgr ctrl.Manager) error { - r.Logger = r.Logger.With("controller", "Standalone") return ctrl.NewControllerManagedBy(mgr). For(&enterpriseApi.Standalone{}). WithEventFilter(predicate.Or( From 570e625616e97b787452f6fab6702522da60e95d Mon Sep 17 00:00:00 2001 From: Kasia Koziol Date: Wed, 4 Mar 2026 09:13:27 +0100 Subject: [PATCH 06/19] CSPL-4513 Events for Index & Ingestion separation (#1706) * CSPL-3551 Init IngestorCluster CR implementation * CSPL-3551 Enhancing Ingestor inputs * CSPL-3551 Adding tests * CSPL-3551 Mound defaults and update them with no Splunk restart * CSPL-3551 Fixing code after tests * CSPL-3895-indexercluster (#1562) * Use endpoint to update conf file * CSPL-3895 Matching changes from Ingestor to Indexer --------- Co-authored-by: Kasia Koziol Co-authored-by: igor.grzankowski <@splunk.com> * CSPL-3551 Applying fixes do Indexer integration * CSPL-3551 Fixes * CSPL-3551 Fixes * CSPL-3560 Initial docs for I&I separation * CSPL-3551 Update documentation to reflect on Grafana * CSPL-3556 Unit tests * CSPL-3972 Addressing TODOs * CSPL-3972 Addressing TODOs * CSPL-3551 Moving default from types to controller code * CSPL-3551 Applying previous changes to IndexerCluster * CSPL-3551 Moving validations to separate function and adding validation for empty values * CSPL-3551 Making sure all inputs are put into status * CSPL-3551 Update of conf files when queue name or type change * CSPL-3551 Unit tests * CSPL-4003 Helm charts support for bus and pipeline configs * CSPL-4003 Docs update * CSPL-4003 Addressing comments * CSPL-3558 Integ tests init * CSPL-3558 Adding extra validation for integraion test * CSPL-3558 Refactoring * CSPL-3558 Adding scenario for update and delete * CSPL-3558 Adding helm test * CSPL-3558 Update of k8s version * CSPL-3558 Fix metrics-server installation issues after k8s version upgrade * CSPL-3558 Adding additional tests * CSPL-3558 Fixing helm tests * CSPL-3558 Fix tests after merge * CSPL-3558 Fix helm tests * CSPL-4022 Remove pipeline config from inputs * CSPL-4022 Remove bus inputs * CSPL-4022 Introduce BusConfiguration CR * CSPL-4022 Update docs and tests * CSPL-4022 Update ns reference for BusConfiguration * CSPL-4022 Fixing tests and adding bus config to ingestor controller * CSPL-4022 Fix update behaviour * CSPL-4022 Docs update * CSPL-4022 Fix failing tests * CSPL-4022 Fix tests * CSPL-4022 Addressing PR comments * CSPL-4022 Address comments * CSPL-4022 Fix helm tests * CSPL-4358 Splitting BusConfiguration into Bus and LargeMessageStore * CSPL-4358 Update docs * CSPL-4358 Addressing comments * CSPL-4358 Adding more validations * CSPL-4360 Secret reference added for Bus CR * CSPL-4360 Fix failing tests * CSPL-4360 Add Splunk restart * CSPL-4360 Fix failing tests * CSPL-4360 Fix failing tests * CSPL-4360 Fix errors with failing validation on status * CSPL-4358 Rename Bus to Queue * CSPL-4358 Rename LargeMessageStore to ObjectStorage * CSPL-4358 Making region authRegion and optional, simplifying endpoint * CSPL-4360 Fixing tests after merge * CSPL-4360 Fix validation that fails for status * CSPL-4360 Fix failing to get k8s secret * CSPL-4360 Fix failing integ and helm tests * CSPL-4360 Fixing failing tests due to incorrect secret ref * CSPL-4360 Addressing comments * CSPL-4360 Addressing secret value change and removing redundant controllers * CSPL-4360 Update of docs, helm tests and validations * CSPL-4360 Add secret watch and fix controller tests * CSPL-4360 Update docs * CSPL-4360 Restart Splunk when SA changed * Addressing comments * Addressing changes from recent PRs * Formatting fix * Addressing PR comments * Address comments * Addressing comments * Fix pipeline issues * Reverting * Refactoring according to comments * CSPL-4513 Refactoring of existing events * CSPL-4513 Adding events and tests for them for Index & Ingestion separation use case * Fix formatting issues * Logs * Fmt fix * CSPL-4513 Extra logs * CSPL-4513 Moving to slog and extra tests * CSPL-4513 Fixing ScaledUp/Down events * CSPL-4002 Update Ansible role to ingestor (from standalone) for IngestorCluster (#1606) * CSPL-4002 Update role for ingestor to ingestor from standalone * CSPL-4212 Address changes from unit tests from develop branch * Addressing comments * Run tests against Splunk 10.2 * Fix * Address changes from develop * CSPL-4002 Revert 10.2 changes * CSPL-4513 Address comments * CSPL-4354 logs for CM and IC * CSPL-4354 Refactoring * CSPL-4513 Addressing Igor's changes * Fix * CSPL-4354 Addressing comments * CSPL-4354 Address comments * Feature/cspl 4354 cluster manager reconciler logging (#1721) * CSPL-4354 logs for CM and IC * CSPL-4354 Refactoring * CSPL-4354 Addressing comments * CSPL-4354 Address comments * CSPL-4354 Address comments * CSPL-4513 Address comments --------- Co-authored-by: Igor Grzankowski --- cmd/main.go | 8 +- pkg/splunk/client/enterprise.go | 23 +- pkg/splunk/client/enterprise_test.go | 10 +- pkg/splunk/enterprise/indexercluster.go | 290 +++++++----- pkg/splunk/enterprise/indexercluster_test.go | 367 ++++++++++++++- pkg/splunk/enterprise/ingestorcluster.go | 105 +++-- pkg/splunk/enterprise/ingestorcluster_test.go | 319 +++++++++++++- pkg/splunk/enterprise/standalone.go | 26 +- pkg/splunk/enterprise/util.go | 1 - pkg/splunk/enterprise/util_test.go | 417 +++++++++++++++++- pkg/splunk/util/secrets.go | 2 +- 11 files changed, 1362 insertions(+), 206 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5f00ea5f9..c82a68e95 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -26,14 +26,11 @@ import ( "path/filepath" "time" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" - intController "github.com/splunk/splunk-operator/internal/controller" "github.com/splunk/splunk-operator/internal/controller/debug" "github.com/splunk/splunk-operator/pkg/config" "github.com/splunk/splunk-operator/pkg/logging" "github.com/splunk/splunk-operator/pkg/splunk/enterprise/validation" - "sigs.k8s.io/controller-runtime/pkg/certwatcher" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -47,14 +44,15 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/splunk/splunk-operator/internal/controller" //+kubebuilder:scaffold:imports //extapi "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) @@ -297,7 +295,7 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Standalone") os.Exit(1) } - if err := (&controller.IngestorClusterReconciler{ + if err := (&intController.IngestorClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Recorder: mgr.GetEventRecorderFor("ingestorcluster-controller"), diff --git a/pkg/splunk/client/enterprise.go b/pkg/splunk/client/enterprise.go index 9291484cb..224da7cf3 100644 --- a/pkg/splunk/client/enterprise.go +++ b/pkg/splunk/client/enterprise.go @@ -17,6 +17,7 @@ package client import ( "bytes" + "context" "crypto/tls" "encoding/json" "fmt" @@ -27,7 +28,7 @@ import ( "strings" "time" - "github.com/go-logr/logr" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" ) @@ -1035,23 +1036,25 @@ func (c *SplunkClient) RestartSplunk() error { // Updates conf files and their properties // See https://help.splunk.com/en/splunk-enterprise/leverage-rest-apis/rest-api-reference/10.0/configuration-endpoints/configuration-endpoint-descriptions -func (c *SplunkClient) UpdateConfFile(scopedLog logr.Logger, fileName, property string, propertyKVList [][]string) error { +func (c *SplunkClient) UpdateConfFile(ctx context.Context, fileName, property string, propertyKVList [][]string) error { + logger := logging.FromContext(ctx).With("func", "UpdateConfFile") + // Creates an object in a conf file if it doesn't exist endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/configs/conf-%s", c.ManagementURI, fileName) body := fmt.Sprintf("name=%s", property) - scopedLog.Info("Creating conf file object if it does not exist", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Creating conf file object if it does not exist", "fileName", fileName, "property", property) request, err := http.NewRequest("POST", endpoint, strings.NewReader(body)) if err != nil { - scopedLog.Error(err, "Failed to create conf file object if it does not exist", "fileName", fileName, "property", property) + logger.ErrorContext(ctx, "Failed to create conf file object if it does not exist", "fileName", fileName, "property", property, "error", err.Error()) return err } - scopedLog.Info("Validating conf file object creation", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Validating conf file object creation", "fileName", fileName, "property", property) expectedStatus := []int{200, 201, 409} err = c.Do(request, expectedStatus, nil) if err != nil { - scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object creation", expectedStatus), "fileName", fileName, "property", property) + logger.ErrorContext(ctx, fmt.Sprintf("Status not in %v for conf file object creation", expectedStatus), "fileName", fileName, "property", property, "error", err.Error()) return err } @@ -1065,18 +1068,18 @@ func (c *SplunkClient) UpdateConfFile(scopedLog logr.Logger, fileName, property body = body[:len(body)-1] } - scopedLog.Info("Updating conf file object", "fileName", fileName, "property", property, "body", body) + logger.InfoContext(ctx, "Updating conf file object", "fileName", fileName, "property", property) request, err = http.NewRequest("POST", endpoint, strings.NewReader(body)) if err != nil { - scopedLog.Error(err, "Failed to update conf file object", "fileName", fileName, "property", property, "body", body) + logger.ErrorContext(ctx, "Failed to update conf file object", "fileName", fileName, "property", property, "error", err.Error()) return err } - scopedLog.Info("Validating conf file object update", "fileName", fileName, "property", property) + logger.InfoContext(ctx, "Validating conf file object update", "fileName", fileName, "property", property) expectedStatus = []int{200, 201} err = c.Do(request, expectedStatus, nil) if err != nil { - scopedLog.Error(err, fmt.Sprintf("Status not in %v for conf file object update", expectedStatus), "fileName", fileName, "property", property, "body", body) + logger.ErrorContext(ctx, fmt.Sprintf("Status not in %v for conf file object update", expectedStatus), "fileName", fileName, "property", property, "error", err.Error()) } return err } diff --git a/pkg/splunk/client/enterprise_test.go b/pkg/splunk/client/enterprise_test.go index 2c902d537..37224a35f 100644 --- a/pkg/splunk/client/enterprise_test.go +++ b/pkg/splunk/client/enterprise_test.go @@ -25,7 +25,6 @@ import ( "testing" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "sigs.k8s.io/controller-runtime/pkg/log" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" ) @@ -704,8 +703,7 @@ func TestUpdateConfFile(t *testing.T) { value := "myvalue" fileName := "outputs" - reqLogger := log.FromContext(context.TODO()) - scopedLog := reqLogger.WithName("TestUpdateConfFile") + ctx := context.TODO() // First request: create the property (object) if it doesn't exist createBody := strings.NewReader(fmt.Sprintf("name=%s", property)) @@ -722,7 +720,7 @@ func TestUpdateConfFile(t *testing.T) { c := NewSplunkClient("https://localhost:8089", "admin", "p@ssw0rd") c.Client = mockSplunkClient - err := c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err := c.UpdateConfFile(ctx, fileName, property, [][]string{{key, value}}) if err != nil { t.Errorf("UpdateConfFile err = %v", err) } @@ -732,7 +730,7 @@ func TestUpdateConfFile(t *testing.T) { mockSplunkClient = &spltest.MockHTTPClient{} mockSplunkClient.AddHandler(wantCreateRequest, 500, "", nil) c.Client = mockSplunkClient - err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err = c.UpdateConfFile(ctx, fileName, property, [][]string{{key, value}}) if err == nil { t.Errorf("UpdateConfFile expected error on create, got nil") } @@ -742,7 +740,7 @@ func TestUpdateConfFile(t *testing.T) { mockSplunkClient.AddHandler(wantCreateRequest, 201, "", nil) mockSplunkClient.AddHandler(wantUpdateRequest, 500, "", nil) c.Client = mockSplunkClient - err = c.UpdateConfFile(scopedLog, fileName, property, [][]string{{key, value}}) + err = c.UpdateConfFile(ctx, fileName, property, [][]string{{key, value}}) if err == nil { t.Errorf("UpdateConfFile expected error on update, got nil") } diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index f84756067..3af847603 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -19,6 +19,7 @@ import ( "context" "errors" "fmt" + "log/slog" "regexp" "sort" "strconv" @@ -27,8 +28,8 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/go-logr/logr" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -37,7 +38,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" rclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -52,8 +52,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIndexerClusterManager").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + logger := logging.FromContext(ctx).With("func", "ApplyIndexerClusterManager", "name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -69,14 +69,15 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // validate and updates defaults for CR err = validateIndexerClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateIndexerClusterSpec", fmt.Sprintf("validate indexercluster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate indexercluster spec") + eventPublisher.Warning(ctx, "IndexerClusterSpecValidationFailed", "Validation of Indexer Cluster spec failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) return result, err } // updates status after function completes cr.Status.ClusterManagerPhase = enterpriseApi.PhaseError if cr.Status.Replicas < cr.Spec.Replicas { + logger.InfoContext(ctx, "Scaling up Indexer Cluster", "previousReplicas", cr.Status.Replicas, "newReplicas", cr.Spec.Replicas) cr.Status.CredentialSecretVersion = "0" cr.Status.ServiceAccount = "" } @@ -95,8 +96,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create or update of general config failed", "error", err.Error()) + eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update of general config failed. Check operator logs for details.") return result, err } @@ -115,16 +116,17 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller cr.Status.ClusterManagerPhase = managerIdxCluster.Status.Phase } } else { - scopedLog.Error(nil, "The configured clusterMasterRef doesn't exist", "clusterManagerRef", cr.Spec.ClusterManagerRef.Name) + logger.WarnContext(ctx, "The configured clusterMasterRef doesn't exist", "clusterManagerRef", cr.Spec.ClusterManagerRef.Name) cr.Status.ClusterManagerPhase = enterpriseApi.PhaseError } - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) // Check if we have configured enough number(<= RF) of replicas if mgr.cr.Status.ClusterManagerPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "verifyRFPeers", fmt.Sprintf("verify RF peer failed %s", err.Error())) + eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verification of RF peer failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Verification of RF peer failed", "error", err.Error()) return result, err } } @@ -141,28 +143,32 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, "DeletionFailed", "Deletion of custom resource failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Deletion of custom resource failed", "error", err.Error()) } return result, err } // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Create or update of headless service for Indexer Cluster failed", "error", err.Error()) return result, err } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Create or update of service for Indexer Cluster failed", "error", err.Error()) return result, err } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getIndexerStatefulSet", fmt.Sprintf("get indexer stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -221,15 +227,16 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mismatch for indexer cluster and indexer container, delete statefulset failed. Error=%s", err.Error())) - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("%s-%s, %s-%s", "indexer-image", cr.Spec.Image, "container-image", statefulSet.Spec.Template.Spec.Containers[0].Image)) + eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } time.Sleep(1 * time.Second) @@ -237,7 +244,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } @@ -247,35 +255,47 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") + logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) return result, err } + logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + // If queue is updated if cr.Spec.QueueRef.Name != "" { if secretChanged || serviceAccountChanged { - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "ApplyIndexerClusterManager", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") + eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) return result, err } + eventPublisher.Normal(ctx, "QueueConfigUpdated", + fmt.Sprintf("Queue/Pipeline configuration updated for %d indexers", cr.Spec.Replicas)) + logger.InfoContext(ctx, "Queue/Pipeline configuration updated", "readyReplicas", cr.Status.ReadyReplicas) + for i := int32(0); i < cr.Spec.Replicas; i++ { idxcClient := mgr.getClient(ctx, i) err = idxcClient.RestartSplunk() if err != nil { return result, err } - scopedLog.Info("Restarted splunk", "indexer", i) + logger.DebugContext(ctx, "Restarted splunk", "indexer", i) } + eventPublisher.Normal(ctx, "IndexersRestarted", + fmt.Sprintf("Restarted Splunk on %d indexer pods", cr.Spec.Replicas)) + cr.Status.CredentialSecretVersion = qosCfg.Version cr.Status.ServiceAccount = cr.Spec.ServiceAccount + + logger.InfoContext(ctx, "Updated status", "credentialSecretVersion", cr.Status.CredentialSecretVersion, "serviceAccount", cr.Status.ServiceAccount) } } @@ -283,7 +303,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpec", fmt.Sprintf("retrieve cluster manager spec failed %s", err.Error())) + eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieval of Cluster Manager spec failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Retrieval of Cluster Manager spec failed", "error", err.Error()) return result, err } if cmMonitoringConsoleConfigRef != "" { @@ -294,12 +315,13 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChanges", fmt.Sprintf("get monitoring console client failed %s", err.Error())) + eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Get Monitoring Console client failed. Check operator logs for details.") + logger.ErrorContext(ctx, "get Monitoring Console client failed", "error", err.Error()) return result, err } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { - scopedLog.Info("Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to cluster manager spec") + logger.WarnContext(ctx, "Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to Cluster Manager spec") } } if len(cr.Status.IndexerSecretChanged) > 0 { @@ -314,7 +336,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "SetClusterMaintenanceMode", fmt.Sprintf("set cluster maintenance mode failed %s", err.Error())) + eventPublisher.Warning(ctx, "ClusterMaintenanceModeFailed", "Set Cluster maintenance mode failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Set Cluster maintenance mode failed", "error", err.Error()) return result, err } } @@ -326,13 +349,14 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller result.Requeue = false // Set indexer cluster CR as owner reference for clustermanager - scopedLog.Info("Setting indexer cluster as owner for cluster manager") + logger.DebugContext(ctx, "Setting Indexer Cluster as owner for Cluster Manager") if len(cr.Spec.ClusterManagerRef.Name) > 0 { namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkClusterManager, cr.Spec.ClusterManagerRef.Name)} } err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRef", fmt.Sprintf("set stateful set owner reference failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err } @@ -353,8 +377,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIndexerCluster") + logger := logging.FromContext(ctx).With("func", "ApplyIndexerCluster", "name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher := GetEventPublisher(ctx, cr) cr.Kind = "IndexerCluster" @@ -362,6 +385,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err := validateIndexerClusterSpec(ctx, client, cr) if err != nil { + eventPublisher.Warning(ctx, "ValidateIndexerClusterSpecFailed", "Validate Indexer Cluster spec failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) return result, err } @@ -369,6 +394,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, cr.Status.Phase = enterpriseApi.PhaseError cr.Status.ClusterMasterPhase = enterpriseApi.PhaseError if cr.Status.Replicas < cr.Spec.Replicas { + logger.InfoContext(ctx, "Scaling up Indexer Cluster", "previousReplicas", cr.Status.Replicas, "newReplicas", cr.Spec.Replicas) cr.Status.CredentialSecretVersion = "0" cr.Status.ServiceAccount = "" } @@ -390,8 +416,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + logger.ErrorContext(ctx, "Create or update general config failed", "error", err.Error()) + eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update general config failed. Check operator logs for details.") return result, err } @@ -413,12 +439,13 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, cr.Status.ClusterMasterPhase = enterpriseApi.PhaseError } - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) // Check if we have configured enough number(<= RF) of replicas if mgr.cr.Status.ClusterMasterPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "verifyRFPeers", fmt.Sprintf("verify RF peer failed %s", err.Error())) + eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verify RF peer failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Verify RF peer failed", "error", err.Error()) return result, err } } @@ -435,7 +462,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, "DeleteFailed", "Delete custom resource failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Delete custom resource failed", "error", err.Error()) } return result, err } @@ -443,21 +471,24 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Create or update of headless service failed", "error", err.Error()) return result, err } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update service for indexer cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Create or update of service failed", "error", err.Error()) return result, err } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getIndexerStatefulSet", fmt.Sprintf("get indexer stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -516,15 +547,16 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("version mitmatch for indexer clustre and indexer container, delete statefulset failed %s", err.Error())) - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("%s-%s, %s-%s", "indexer-image", cr.Spec.Image, "container-image", statefulSet.Spec.Template.Spec.Containers[0].Image)) + eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } time.Sleep(1 * time.Second) @@ -532,7 +564,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateManager", fmt.Sprintf("update statefulset failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } } @@ -542,34 +575,46 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") + logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) return result, err } + logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + if cr.Spec.QueueRef.Name != "" { if secretChanged || serviceAccountChanged { - mgr := newIndexerClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "ApplyIndexerClusterManager", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") + eventPublisher.Warning(ctx, "UpdateIndexerConfFileFailed", "Failed to update conf file for Queue/Pipeline config change after pod creation. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config change after pod creation", "error", err.Error()) return result, err } + eventPublisher.Normal(ctx, "QueueConfigUpdated", + fmt.Sprintf("Queue/Pipeline configuration updated for %d indexers", cr.Spec.Replicas)) + logger.InfoContext(ctx, "Queue/Pipeline configuration updated", "readyReplicas", cr.Status.ReadyReplicas) + for i := int32(0); i < cr.Spec.Replicas; i++ { idxcClient := mgr.getClient(ctx, i) err = idxcClient.RestartSplunk() if err != nil { return result, err } - scopedLog.Info("Restarted splunk", "indexer", i) + logger.DebugContext(ctx, "Restarted splunk", "indexer", i) } + eventPublisher.Normal(ctx, "IndexersRestarted", + fmt.Sprintf("Restarted Splunk on %d indexer pods", cr.Spec.Replicas)) + cr.Status.CredentialSecretVersion = qosCfg.Version cr.Status.ServiceAccount = cr.Spec.ServiceAccount + + logger.InfoContext(ctx, "Updated status", "credentialSecretVersion", cr.Status.CredentialSecretVersion, "serviceAccount", cr.Status.ServiceAccount) } } @@ -577,7 +622,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpec", fmt.Sprintf("retrieve cluster master spec failed %s", err.Error())) + eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieve Cluster Master spec failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Retrieve Cluster Master spec failed", "error", err.Error()) return result, err } if cmMonitoringConsoleConfigRef != "" { @@ -588,12 +634,13 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChanges", fmt.Sprintf("get monitoring console client failed %s", err.Error())) + eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Automate MC Apply Changes failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Automate MC Apply Changes failed", "error", err.Error()) return result, err } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { - scopedLog.Info("Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to cluster master spec") + logger.WarnContext(ctx, "Indexer Cluster CR should not specify monitoringConsoleRef and if specified, should be similar to Cluster Master spec") } } if len(cr.Status.IndexerSecretChanged) > 0 { @@ -601,14 +648,15 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.ClusterMasterRef.Name) > 0 { managerIdxcName = cr.Spec.ClusterMasterRef.Name } else { - return result, errors.New("empty cluster master reference") + return result, errors.New("empty Cluster Master reference") } cmPodName := fmt.Sprintf("splunk-%s-cluster-master-%s", managerIdxcName, "0") podExecClient := splutil.GetPodExecClient(client, cr, cmPodName) // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "SetClusterMaintenanceMode", fmt.Sprintf("set cluster maintenance mode failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetClusterMaintenanceModeFailed", "Set Cluster Master maintenance mode failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Set Cluster Master maintenance mode failed", "error", err.Error()) return result, err } } @@ -620,11 +668,12 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false // Set indexer cluster CR as owner reference for clustermaster - scopedLog.Info("Setting indexer cluster as owner for cluster master") + logger.DebugContext(ctx, "Setting Indexer Cluster as owner for Cluster Master") namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkClusterMaster, cr.Spec.ClusterMasterRef.Name)} err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRef", fmt.Sprintf("set stateful set owner reference failed %s", err.Error())) + eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err } @@ -645,14 +694,14 @@ var VerifyRFPeers = func(ctx context.Context, mgr indexerClusterPodManager, clie // indexerClusterPodManager is used to manage the pods within an indexer cluster type indexerClusterPodManager struct { c splcommon.ControllerClient - log logr.Logger + log *slog.Logger cr *enterpriseApi.IndexerCluster secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient } // newIndexerClusterPodManager function to create pod manager this is added to write unit test case -var newIndexerClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { +var newIndexerClusterPodManager = func(log *slog.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { return indexerClusterPodManager{ log: log, cr: cr, @@ -712,25 +761,29 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica return err } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIdxcSecret").WithValues("Desired replicas", replicas, "IdxcSecretChanged", mgr.cr.Status.IndexerSecretChanged, "CrStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "NamespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) + logger := slog.With("func", "ApplyIdxcSecret", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) + logger.InfoContext(ctx, "Applying idxc secret to indexers", "desiredReplicas", replicas, "idxcSecretChanged", mgr.cr.Status.IndexerSecretChanged, "crStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "namespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) // If namespace scoped secret revision is the same ignore if len(mgr.cr.Status.NamespaceSecretResourceVersion) == 0 { // First time, set resource version in CR mgr.cr.Status.NamespaceSecretResourceVersion = namespaceSecret.ObjectMeta.ResourceVersion - scopedLog.Info("Setting CrStatusNamespaceSecretResourceVersion for the first time") + logger.DebugContext(ctx, "Setting CrStatusNamespaceSecretResourceVersion for the first time") return nil } else if mgr.cr.Status.NamespaceSecretResourceVersion == namespaceSecret.ObjectMeta.ResourceVersion { // If resource version hasn't changed don't return return nil } - scopedLog.Info("Namespaced scoped secret revision has changed") + logger.InfoContext(ctx, "Namespaced scoped secret revision has changed") // Retrieve idxc_secret password from secret data nsIdxcSecret := string(namespaceSecret.Data[splcommon.IdxcSecret]) + // Log configuration push start + pushStartTime := time.Now() + logger.InfoContext(ctx, "Starting configuration push to peers", "peerCount", replicas, "configVersion", namespaceSecret.ObjectMeta.ResourceVersion) + // Loop over all indexer pods and get individual pod's idxc password howManyPodsHaveSecretChanged := 0 for i := int32(0); i <= replicas-1; i++ { @@ -740,10 +793,10 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // Check if pod exists before updating secrets pod := &corev1.Pod{} namespacedName := types.NamespacedName{Namespace: mgr.cr.GetNamespace(), Name: indexerPodName} - scopedLog.Info("Check if pod is created before updating its secrets") + logger.DebugContext(ctx, "Check if pod is created before updating its secrets") err := mgr.c.Get(ctx, namespacedName, pod) if err != nil { - mgr.log.Info("Peer doesn't exists", "peerName", indexerPodName) + logger.WarnContext(ctx, "Peer doesn't exists", "peerName", indexerPodName) continue } @@ -762,7 +815,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // If idxc secret is different from namespace scoped secret change it if indIdxcSecret != nsIdxcSecret { - scopedLog.Info("idxc Secret different from namespace scoped secret") + logger.InfoContext(ctx, "IDXC Secret is different from namespace scoped secret") // Enable maintenance mode if len(mgr.cr.Status.IndexerSecretChanged) == 0 && !mgr.cr.Status.MaintenanceMode { @@ -782,7 +835,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { return err } - scopedLog.Info("Set CM in maintenance mode") + logger.InfoContext(ctx, "Set CM in maintenance mode") } // If idxc secret already changed, ignore @@ -801,11 +854,12 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // Emit event for password sync failure if eventPublisher != nil { eventPublisher.Warning(ctx, "PasswordSyncFailed", - fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) + fmt.Sprintf("Password sync failed for pod '%s'. Check operator logs for details.", indexerPodName)) } + mgr.log.ErrorContext(ctx, "Configuration push failed", "failedPeer", indexerPodName, "error", err.Error()) return err } - scopedLog.Info("Changed idxc secret") + logger.InfoContext(ctx, "Changed idxc secret") howManyPodsHaveSecretChanged += 1 @@ -815,11 +869,12 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // Emit event for password sync failure if eventPublisher != nil { eventPublisher.Warning(ctx, "PasswordSyncFailed", - fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) + fmt.Sprintf("Password sync failed for pod '%s'. Check operator logs for details.", indexerPodName)) } + logger.ErrorContext(ctx, "Configuration push failed during restart", "failedPeer", indexerPodName, "error", err.Error()) return err } - scopedLog.Info("Restarted splunk") + logger.InfoContext(ctx, "Restarted splunk") // Keep a track of all the secrets on pods to change their idxc secret below mgr.cr.Status.IdxcPasswordChangedSecrets[podSecret.GetName()] = true @@ -862,7 +917,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { return err } - scopedLog.Info("idxc password changed on the secret mounted on pod", "Secret on Pod:", podSecretName) + logger.InfoContext(ctx, "IDXC password changed on the secret mounted on pod", "podSecretName", podSecretName) // Set to false marking the idxc password change in the secret mgr.cr.Status.IdxcPasswordChangedSecrets[podSecretName] = false @@ -876,6 +931,9 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica fmt.Sprintf("Password synchronized for %d pods", howManyPodsHaveSecretChanged)) } + // Log configuration push completion + logger.InfoContext(ctx, "Configuration push completed", "successCount", howManyPodsHaveSecretChanged, "duration", time.Since(pushStartTime)) + return nil } @@ -887,8 +945,8 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con // Get event publisher from context eventPublisher := GetEventPublisher(ctx, mgr.cr) - // Track last successful replica count to emit scale events after completion - previousReplicas := mgr.cr.Status.Replicas + // Track previous ready replicas for scaling events + previousReadyReplicas := mgr.cr.Status.ReadyReplicas // Assign client if mgr.c == nil { @@ -901,7 +959,7 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con return enterpriseApi.PhaseError, err } } else { - mgr.log.Info("Cluster Manager is not ready yet", "reason ", err) + mgr.log.InfoContext(ctx, "Cluster Manager is not ready yet", "error", err) return enterpriseApi.PhaseError, err } @@ -917,7 +975,7 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con // update CR status with IDXC information err = mgr.updateStatus(ctx, statefulSet) if err != nil || mgr.cr.Status.ReadyReplicas == 0 || !mgr.cr.Status.Initialized || !mgr.cr.Status.IndexingReady || !mgr.cr.Status.ServiceReady { - mgr.log.Info("Indexer cluster is not ready", "reason ", err) + mgr.log.InfoContext(ctx, "Indexer Cluster is not ready", "error ", err) return enterpriseApi.PhasePending, nil } @@ -927,17 +985,19 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con return phase, err } - // Emit ScaledUp event only after a successful scale-up has completed + // Emit scale events when phase is ready and ready replicas changed to match desired if phase == enterpriseApi.PhaseReady { - if desiredReplicas > previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", - fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) - } - } else if desiredReplicas < previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", - fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) + if mgr.cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { + if desiredReplicas > previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledUp", + fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) + } + } else if desiredReplicas < previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledDown", + fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) + } } } } @@ -958,6 +1018,9 @@ func (mgr *indexerClusterPodManager) PrepareScaleDown(ctx context.Context, n int // next, remove the peer c := mgr.getClusterManagerClient(ctx) + peerName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) + remainingPeers := int32(len(mgr.cr.Status.Peers)) - 1 + mgr.log.InfoContext(ctx, "Deregistering peer from cluster manager", "peerName", peerName, "remainingPeers", remainingPeers) return true, c.RemoveIndexerClusterPeer(mgr.cr.Status.Peers[n].ID) } @@ -990,31 +1053,31 @@ func (mgr *indexerClusterPodManager) decommission(ctx context.Context, n int32, // Don't return error here. We may be reconciling several times, and the actual Pod status is down, but // not yet reflecting on the Cluster Master, in which case, the podExec fails, though the decommission is // going fine. - mgr.log.Info("Unable to lower the liveness probe level", "peerName", peerName, "enforceCounts", enforceCounts) + mgr.log.WarnContext(ctx, "Unable to lower the liveness probe level", "peerName", peerName, "enforceCounts", enforceCounts) } - mgr.log.Info("Decommissioning indexer cluster peer", "peerName", peerName, "enforceCounts", enforceCounts) + mgr.log.InfoContext(ctx, "Decommissioning indexer cluster peer", "peerName", peerName, "enforceCounts", enforceCounts) c := mgr.getClient(ctx, n) return false, c.DecommissionIndexerClusterPeer(enforceCounts) case "Decommissioning": - mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + mgr.log.InfoContext(ctx, "Waiting for decommission to complete", "peerName", peerName) return false, nil case "ReassigningPrimaries": - mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + mgr.log.InfoContext(ctx, "Waiting for decommission to complete", "peerName", peerName) return false, nil case "GracefulShutdown": - mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + mgr.log.InfoContext(ctx, "Decommission complete", "peerName", peerName, "status", mgr.cr.Status.Peers[n].Status) return true, nil case "Down": - mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + mgr.log.InfoContext(ctx, "Decommission complete", "peerName", peerName, "status", mgr.cr.Status.Peers[n].Status) return true, nil case "": // this can happen after the peer has been removed from the indexer cluster - mgr.log.Info("Peer has empty ID", "peerName", peerName) + mgr.log.InfoContext(ctx, "Peer has empty ID", "peerName", peerName) return false, nil } @@ -1024,8 +1087,7 @@ func (mgr *indexerClusterPodManager) decommission(ctx context.Context, n int32, // getClient for indexerClusterPodManager returns a SplunkClient for the member n func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("indexerClusterPodManager.getClient").WithValues("name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) + logger := slog.With("func", "indexerClusterPodManager.getClient", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) // Get Pod Name memberName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) @@ -1037,7 +1099,7 @@ func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *sp // Retrieve admin password from Pod adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, memberName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") + logger.WarnContext(ctx, "Couldn't retrieve the admin password from pod", "error", err) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -1045,8 +1107,7 @@ func (mgr *indexerClusterPodManager) getClient(ctx context.Context, n int32) *sp // getClusterManagerClient for indexerClusterPodManager returns a SplunkClient for cluster manager func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("indexerClusterPodManager.getClusterManagerClient") + logger := slog.With("func", "indexerClusterPodManager.getClusterManagerClient", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) // Retrieve admin password from Pod var managerIdxcName string @@ -1058,7 +1119,7 @@ func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context managerIdxcName = mgr.cr.Spec.ClusterMasterRef.Name cm = SplunkClusterMaster } else { - mgr.log.Info("Empty cluster manager reference") + mgr.log.InfoContext(ctx, "Empty cluster manager reference") } // Get Fully Qualified Domain Name @@ -1068,7 +1129,7 @@ func (mgr *indexerClusterPodManager) getClusterManagerClient(ctx context.Context podName := fmt.Sprintf("splunk-%s-%s-%s", managerIdxcName, cm, "0") adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, podName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") + logger.WarnContext(ctx, "Couldn't retrieve the admin password from pod", "error", err.Error()) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -1109,7 +1170,7 @@ func (mgr *indexerClusterPodManager) verifyRFPeers(ctx context.Context, c splcom requestedReplicas := mgr.cr.Spec.Replicas if requestedReplicas < replicationFactor { - mgr.log.Info("Changing number of replicas as it is less than RF number of peers", "replicas", requestedReplicas) + mgr.log.InfoContext(ctx, "Changing number of replicas as it is less than RF number of peers", "replicas", requestedReplicas) // Emit event indicating scaling below RF is blocked/adjusted if eventPublisher != nil { eventPublisher.Warning(ctx, "ScalingBlockedRF", @@ -1160,6 +1221,8 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS if err != nil { return err } + totalPeerCount := len(peers) + clusterName := mgr.cr.GetName() for n := int32(0); n < statefulSet.Status.Replicas; n++ { peerName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), n) peerStatus := enterpriseApi.IndexerClusterMemberStatus{Name: peerName} @@ -1170,8 +1233,12 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS peerStatus.ActiveBundleID = peerInfo.ActiveBundleID peerStatus.BucketCount = peerInfo.BucketCount peerStatus.Searchable = peerInfo.Searchable + slog.InfoContext(ctx, "Peer registered with cluster manager", + "peerName", peerName, + "clusterName", clusterName, + "totalPeerCount", totalPeerCount) } else { - mgr.log.Info("Peer is not known by cluster manager", "peerName", peerName) + mgr.log.InfoContext(ctx, "Peer is not known by Cluster Manager", "peerName", peerName) } if n < int32(len(mgr.cr.Status.Peers)) { mgr.cr.Status.Peers[n] = peerStatus @@ -1255,14 +1322,13 @@ func validateIndexerClusterSpec(ctx context.Context, c splcommon.ControllerClien // helper function to get the list of IndexerCluster types in the current namespace func getIndexerClusterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []rclient.ListOption) (enterpriseApi.IndexerClusterList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getIndexerClusterList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := slog.With("func", "getIndexerClusterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.IndexerClusterList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "IndexerCluster types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, fmt.Sprintf("IndexerCluster types not found in namespace %s", cr.GetNamespace()), "error", err.Error()) return objectList, err } @@ -1331,8 +1397,7 @@ var newSplunkClientForQueuePipeline = splclient.NewSplunkClient // updateIndexerConfFiles checks if Queue or Pipeline inputs are created for the first time and updates the conf file if so func (mgr *indexerClusterPodManager) updateIndexerConfFiles(ctx context.Context, newCR *enterpriseApi.IndexerCluster, queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string, k8s rclient.Client) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateIndexerConfFiles").WithValues("name", newCR.GetName(), "namespace", newCR.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "updateIndexerConfFiles", "name", newCR.GetName(), "namespace", newCR.GetNamespace()) // Only update config for pods that exist readyReplicas := newCR.Status.ReadyReplicas @@ -1351,19 +1416,26 @@ func (mgr *indexerClusterPodManager) updateIndexerConfFiles(ctx context.Context, queueInputs, queueOutputs, pipelineInputs := getQueueAndPipelineInputsForIndexerConfFiles(queue, os, accessKey, secretKey) for _, pbVal := range queueOutputs { - if err := splunkClient.UpdateConfFile(scopedLog, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { + if !strings.Contains(pbVal[0], "access_key") && !strings.Contains(pbVal[0], "secret_key") { + logger.DebugContext(ctx, "Updating queue input in outputs.conf", "input", pbVal) + } + if err := splunkClient.UpdateConfFile(ctx, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { updateErr = err } } for _, pbVal := range queueInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "inputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { + if !strings.Contains(pbVal[0], "access_key") && !strings.Contains(pbVal[0], "secret_key") { + logger.DebugContext(ctx, "Updating queue input in inputs.conf", "input", pbVal) + } + if err := splunkClient.UpdateConfFile(ctx, "inputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{pbVal}); err != nil { updateErr = err } } for _, field := range pipelineInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "default-mode", field[0], [][]string{{field[1], field[2]}}); err != nil { + logger.DebugContext(ctx, "Updating pipeline input in default-mode.conf", "input", field) + if err := splunkClient.UpdateConfFile(ctx, "default-mode", field[0], [][]string{{field[1], field[2]}}); err != nil { updateErr = err } } diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index 0629e27ec..1e1180156 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "net/http" "os" "path/filepath" @@ -44,7 +45,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/go-logr/logr" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" @@ -228,7 +229,7 @@ func TestApplyIndexerCluster(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *spltest.MockClient, cr interface{}) error { - _, err := ApplyIndexerClusterManager(context.Background(), c, cr.(*enterpriseApi.IndexerCluster)) + _, err := ApplyIndexerClusterManager(context.TODO(), c, cr.(*enterpriseApi.IndexerCluster)) return err } spltest.ReconcileTesterWithoutRedundantCheck(t, "TestApplyIndexerClusterManager", ¤t, revised, createCalls, updateCalls, reconcile, true) @@ -238,7 +239,7 @@ func TestApplyIndexerCluster(t *testing.T) { revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr splcommon.MetaObject, c splcommon.ControllerClient) (bool, error) { - _, err := ApplyIndexerClusterManager(context.Background(), c, cr.(*enterpriseApi.IndexerCluster)) + _, err := ApplyIndexerClusterManager(context.TODO(), c, cr.(*enterpriseApi.IndexerCluster)) return true, err } splunkDeletionTester(t, revised, deleteFunc) @@ -289,6 +290,9 @@ func TestApplyIndexerCluster(t *testing.T) { func TestGetMonitoringConsoleClient(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") + + logger := logging.FromContext(context.TODO()).With("func", "TestGetMonitoringConsoleClient", "name", "stack1", "namespace", "test") + current := enterpriseApi.IndexerCluster{ TypeMeta: metav1.TypeMeta{ Kind: "IndexerCluster", @@ -307,7 +311,6 @@ func TestGetMonitoringConsoleClient(t *testing.T) { }, }, } - scopedLog := logt.WithName("TestGetMonitoringConsoleClient") secrets := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -320,7 +323,7 @@ func TestGetMonitoringConsoleClient(t *testing.T) { } mockSplunkClient := &spltest.MockHTTPClient{} mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: ¤t, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -336,7 +339,8 @@ func TestGetClusterManagerClient(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") ctx := context.TODO() - scopedLog := logt.WithName("TestGetClusterManagerClient") + logger := logging.FromContext(ctx).With("func", "TestGetClusterManagerClient", "name", "stack1", "namespace", "test") + cr := enterpriseApi.IndexerCluster{ TypeMeta: metav1.TypeMeta{ Kind: "IndexerCluster", @@ -368,7 +372,7 @@ func TestGetClusterManagerClient(t *testing.T) { } mockSplunkClient := &spltest.MockHTTPClient{} mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -387,7 +391,8 @@ func TestGetClusterManagerClient(t *testing.T) { func getIndexerClusterPodManager(method string, mockHandlers []spltest.MockHTTPHandler, mockSplunkClient *spltest.MockHTTPClient, replicas int32) *indexerClusterPodManager { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") - scopedLog := logt.WithName(method) + logger := logging.FromContext(context.TODO()).With("func", method, "name", "stack1", "namespace", "test") + cr := enterpriseApi.IndexerCluster{ TypeMeta: metav1.TypeMeta{ Kind: "IndexerCluster", @@ -421,7 +426,7 @@ func getIndexerClusterPodManager(method string, mockHandlers []spltest.MockHTTPH } mgr := &indexerClusterPodManager{ - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -1029,7 +1034,8 @@ func TestSetClusterMaintenanceMode(t *testing.T) { func TestApplyIdxcSecret(t *testing.T) { os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") method := "ApplyIdxcSecret" - scopedLog := logt.WithName(method) + logger := logging.FromContext(context.TODO()).With("func", method, "name", "stack1", "namespace", "test") + var initObjectList []client.Object ctx := context.TODO() @@ -1154,7 +1160,7 @@ func TestApplyIdxcSecret(t *testing.T) { mockSplunkClient.AddHandlers(mockHandlers...) mgr := &indexerClusterPodManager{ c: c, - log: scopedLog, + log: logger, cr: &cr, secrets: secrets, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { @@ -1327,19 +1333,19 @@ func TestInvalidIndexerClusterSpec(t *testing.T) { cm.Status.Phase = enterpriseApi.PhaseReady // Empty ClusterManagerRef should return an error cr.Spec.ClusterManagerRef.Name = "" - if _, err := ApplyIndexerClusterManager(context.Background(), c, &cr); err == nil { + if _, err := ApplyIndexerClusterManager(context.TODO(), c, &cr); err == nil { t.Errorf("ApplyIndxerCluster() should have returned error") } cr.Spec.ClusterManagerRef.Name = "manager1" // verifyRFPeers should return err here - if _, err := ApplyIndexerClusterManager(context.Background(), c, &cr); err == nil { + if _, err := ApplyIndexerClusterManager(context.TODO(), c, &cr); err == nil { t.Errorf("ApplyIndxerCluster() should have returned error") } cm.Status.Phase = enterpriseApi.PhaseError cr.Spec.CommonSplunkSpec.EtcVolumeStorageConfig.StorageCapacity = "-abcd" - if _, err := ApplyIndexerClusterManager(context.Background(), c, &cr); err == nil { + if _, err := ApplyIndexerClusterManager(context.TODO(), c, &cr); err == nil { t.Errorf("ApplyIndxerCluster() should have returned error") } } @@ -1596,7 +1602,7 @@ func TestIndexerClusterWithReadyState(t *testing.T) { savedNewIndexerClusterPodManager := newIndexerClusterPodManager defer func() { newIndexerClusterPodManager = savedNewIndexerClusterPodManager }() - newIndexerClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { + newIndexerClusterPodManager = func(log *slog.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { return indexerClusterPodManager{ log: log, cr: cr, @@ -2828,7 +2834,7 @@ func TestPasswordSyncCompleted(t *testing.T) { // Initialize a minimal pod manager for ApplyIdxcSecret mgr := &indexerClusterPodManager{ c: client, - log: logt.WithName("TestPasswordSyncCompleted"), + log: logging.FromContext(ctx).With("func", "TestPasswordSyncCompleted", "name", idxc.GetName(), "namespace", idxc.GetNamespace()), cr: &idxc, } @@ -3360,7 +3366,7 @@ func TestIdxcPasswordSyncFailedEvent(t *testing.T) { mgr := &indexerClusterPodManager{ c: c, - log: logt.WithName("TestIdxcPasswordSyncFailedEvent"), + log: logging.FromContext(ctx).With("func", "TestIdxcPasswordSyncFailedEvent", "name", idxc.GetName(), "namespace", idxc.GetNamespace()), cr: &idxc, newSplunkClient: func(managementURI, username, password string) *splclient.SplunkClient { sc := splclient.NewSplunkClient(managementURI, username, password) @@ -3418,3 +3424,330 @@ func (m *mockEventRecorder) Eventf(object pkgruntime.Object, eventType, reason, func (m *mockEventRecorder) AnnotatedEventf(object pkgruntime.Object, annotations map[string]string, eventType, reason, messageFmt string, args ...interface{}) { m.events = append(m.events, mockEvent{eventType: eventType, reason: reason, message: fmt.Sprintf(messageFmt, args...)}) } + +func TestIdxcQueueConfigUpdatedIndexersRestartedEvents(t *testing.T) { + os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") + + ctx := context.TODO() + recorder := &mockEventRecorder{events: []mockEvent{}} + eventPublisher := &K8EventPublisher{recorder: recorder} + ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) + + oldVerifyRFPeers := VerifyRFPeers + defer func() { VerifyRFPeers = oldVerifyRFPeers }() + VerifyRFPeers = func(ctx context.Context, mgr indexerClusterPodManager, client splcommon.ControllerClient) error { + return nil + } + + oldGetCMInfo := GetClusterManagerInfoCall + oldGetCMPeers := GetClusterManagerPeersCall + defer func() { + GetClusterManagerInfoCall = oldGetCMInfo + GetClusterManagerPeersCall = oldGetCMPeers + }() + GetClusterManagerInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager) (*splclient.ClusterManagerInfo, error) { + return &splclient.ClusterManagerInfo{Initialized: true, IndexingReady: true, ServiceReady: true, MaintenanceMode: false}, nil + } + GetClusterManagerPeersCall = func(ctx context.Context, mgr *indexerClusterPodManager) (map[string]splclient.ClusterManagerPeerInfo, error) { + peers := map[string]splclient.ClusterManagerPeerInfo{} + for i := int32(0); i < 3; i++ { + peerName := GetSplunkStatefulsetPodName(SplunkIndexer, mgr.cr.GetName(), i) + peers[peerName] = splclient.ClusterManagerPeerInfo{ID: fmt.Sprintf("peer-%d", i), Status: "Up", Searchable: true} + } + return peers, nil + } + + sch := pkgruntime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(sch)) + utilruntime.Must(corev1.AddToScheme(sch)) + utilruntime.Must(appsv1.AddToScheme(sch)) + utilruntime.Must(enterpriseApi.AddToScheme(sch)) + + builder := fake.NewClientBuilder(). + WithScheme(sch). + WithStatusSubresource(&enterpriseApi.ClusterManager{}). + WithStatusSubresource(&enterpriseApi.IndexerCluster{}) + c := builder.Build() + + probeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "splunk-test-probe-configmap", + Namespace: "test", + }, + } + _ = c.Create(ctx, probeConfigMap) + + cm := &enterpriseApi.ClusterManager{ + ObjectMeta: metav1.ObjectMeta{Name: "manager1", Namespace: "test"}, + Status: enterpriseApi.ClusterManagerStatus{Phase: enterpriseApi.PhaseReady}, + } + _ = c.Create(ctx, cm) + _ = c.Status().Update(ctx, cm) + + queueSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "queue-secrets", Namespace: "test"}, + Data: map[string][]byte{ + "s3_access_key": []byte("access"), + "s3_secret_key": []byte("secret"), + }, + } + _ = c.Create(ctx, queueSecret) + + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "test-queue", + AuthRegion: "us-west-2", + Endpoint: "https://sqs.us-west-2.amazonaws.com", + DLQ: "sqs-dlq-test", + VolList: []enterpriseApi.VolumeSpec{ + {SecretRef: "queue-secrets"}, + }, + }, + }, + } + _ = c.Create(ctx, queue) + + objStorage := &enterpriseApi.ObjectStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "os", Namespace: "test"}, + Spec: enterpriseApi.ObjectStorageSpec{ + Provider: "s3", + S3: enterpriseApi.S3Spec{ + Endpoint: "https://s3.us-west-2.amazonaws.com", + Path: "bucket/key", + }, + }, + } + _ = c.Create(ctx, objStorage) + + passwordSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-secrets", Namespace: "test"}, + Data: map[string][]byte{"password": []byte("dummy")}, + } + _ = c.Create(ctx, passwordSecret) + + cmPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "splunk-manager1-cluster-manager-0", + Namespace: "test", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "splunk", Image: "splunk/splunk:latest"}}, + Volumes: []corev1.Volume{ + { + Name: "mnt-splunk-secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "test-secrets"}, + }, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{Ready: true}}, + }, + } + _ = c.Create(ctx, cmPod) + + cmReplicas := int32(1) + cmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetSplunkStatefulsetName(SplunkClusterManager, "manager1"), + Namespace: "test", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &cmReplicas, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "splunk", Image: "splunk/splunk:latest"}}, + }, + }, + }, + } + _ = c.Create(ctx, cmSts) + + crName := "stack1" + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: "test"}, + Spec: enterpriseApi.IndexerClusterSpec{ + Replicas: 3, + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Mock: true, + ClusterManagerRef: corev1.ObjectReference{ + Name: "manager1", + }, + ServiceAccount: "", + }, + QueueRef: corev1.ObjectReference{Name: "queue"}, + ObjectStorageRef: corev1.ObjectReference{Name: "os"}, + }, + Status: enterpriseApi.IndexerClusterStatus{ + ReadyReplicas: 0, + CredentialSecretVersion: "0", + ServiceAccount: "", + }, + } + _ = c.Create(ctx, cr) + + threeReplicas := int32(3) + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetSplunkStatefulsetName(SplunkIndexer, cr.GetName()), + Namespace: cr.GetNamespace(), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &threeReplicas, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "splunk", Image: "splunk/splunk:latest"}}, + }, + }, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: threeReplicas, + ReadyReplicas: 0, + CurrentRevision: "v1", + UpdateRevision: "v1", + }, + } + _ = c.Create(ctx, sts) + + basePod := &corev1.Pod{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "splunk", Image: "splunk/splunk:latest"}}, + Volumes: []corev1.Volume{ + { + Name: "mnt-splunk-secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "test-secrets"}, + }, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{Ready: true}}, + }, + } + for i := int32(0); i < threeReplicas; i++ { + pod := basePod.DeepCopy() + pod.ObjectMeta = metav1.ObjectMeta{ + Name: GetSplunkStatefulsetPodName(SplunkIndexer, cr.GetName(), i), + Namespace: cr.GetNamespace(), + Labels: map[string]string{ + "app.kubernetes.io/instance": GetSplunkStatefulsetName(SplunkIndexer, cr.GetName()), + "controller-revision-hash": "v1", + }, + } + _ = c.Create(ctx, pod) + } + + mockHTTPClient := &spltest.MockHTTPClient{} + body := "" + crForHandlers := cr.DeepCopy() + crForHandlers.Status.ReadyReplicas = 3 + addRemoteQueueHandlersForIndexer(mockHTTPClient, crForHandlers, &queue.Spec, "conf-outputs", body) + addRemoteQueueHandlersForIndexer(mockHTTPClient, crForHandlers, &queue.Spec, "conf-inputs", body) + for i := 0; i < int(crForHandlers.Status.ReadyReplicas); i++ { + podName := fmt.Sprintf("splunk-%s-indexer-%d", crForHandlers.GetName(), i) + baseURL := fmt.Sprintf( + "https://%s.splunk-%s-indexer-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/conf-default-mode", + podName, crForHandlers.GetName(), crForHandlers.GetNamespace(), + ) + for _, field := range getPipelineInputsForConfFile(true) { + req, _ := http.NewRequest("POST", baseURL, strings.NewReader(fmt.Sprintf("name=%s", field[0]))) + mockHTTPClient.AddHandler(req, 200, "", nil) + updateURL := fmt.Sprintf("%s/%s", baseURL, field[0]) + req, _ = http.NewRequest("POST", updateURL, strings.NewReader(fmt.Sprintf("%s=%s", field[1], field[2]))) + mockHTTPClient.AddHandler(req, 200, "", nil) + } + req, _ := http.NewRequest( + "POST", + fmt.Sprintf( + "https://%s.splunk-%s-indexer-headless.%s.svc.cluster.local:8089/services/server/control/restart", + podName, crForHandlers.GetName(), crForHandlers.GetNamespace(), + ), + nil, + ) + mockHTTPClient.AddHandler(req, 200, "", nil) + } + + var peersBuilder strings.Builder + peersBuilder.WriteString(`{"entry":[`) + for i := int32(0); i < threeReplicas; i++ { + if i > 0 { + peersBuilder.WriteString(",") + } + peersBuilder.WriteString(fmt.Sprintf(`{"content":{"label":"splunk-%s-indexer-%d"}}`, crForHandlers.GetName(), i)) + } + peersBuilder.WriteString(`]}`) + peersBody := peersBuilder.String() + + mockHTTPClient.AddHandlers( + spltest.MockHTTPHandler{ + Method: "GET", + URL: "https://splunk-manager1-cluster-manager-service.test.svc.cluster.local:8089/services/cluster/manager/info?count=0&output_mode=json", + Status: 200, + Body: splcommon.TestIndexerClusterPodManagerInfo, + }, + spltest.MockHTTPHandler{ + Method: "GET", + URL: "https://splunk-manager1-cluster-manager-service.test.svc.cluster.local:8089/services/cluster/manager/peers?count=0&output_mode=json", + Status: 200, + Body: peersBody, + }, + ) + + oldNewIndexerClusterPodManager := newIndexerClusterPodManager + oldNewSplunkClientForQueuePipeline := newSplunkClientForQueuePipeline + defer func() { + newIndexerClusterPodManager = oldNewIndexerClusterPodManager + newSplunkClientForQueuePipeline = oldNewSplunkClientForQueuePipeline + }() + + newSplunkClientForQueuePipeline = func(uri, user, pass string) *splclient.SplunkClient { + return &splclient.SplunkClient{ManagementURI: uri, Username: user, Password: pass, Client: mockHTTPClient} + } + newIndexerClusterPodManager = func(l *slog.Logger, cr *enterpriseApi.IndexerCluster, secret *corev1.Secret, _ NewSplunkClientFunc, c splcommon.ControllerClient) indexerClusterPodManager { + return indexerClusterPodManager{ + c: c, + log: l, + cr: cr, + secrets: secret, + newSplunkClient: func(uri, user, pass string) *splclient.SplunkClient { + return &splclient.SplunkClient{ManagementURI: uri, Username: user, Password: pass, Client: mockHTTPClient} + }, + } + } + + _, err := ApplyIndexerClusterManager(ctx, c, cr) + assert.NoError(t, err) + + assert.NoError(t, c.Get(ctx, client.ObjectKey{Name: sts.GetName(), Namespace: sts.GetNamespace()}, sts)) + sts.Status.Replicas = threeReplicas + sts.Status.ReadyReplicas = threeReplicas + sts.Status.UpdatedReplicas = threeReplicas + sts.Status.CurrentRevision = "v1" + sts.Status.UpdateRevision = "v1" + assert.NoError(t, c.Status().Update(ctx, sts)) + + cr.Status.ServiceAccount = "" + cr.Status.CredentialSecretVersion = "old" + _, err = ApplyIndexerClusterManager(ctx, c, cr) + assert.NoError(t, err) + + queueUpdated := false + indexersRestarted := false + for _, event := range recorder.events { + if event.reason == "QueueConfigUpdated" { + queueUpdated = true + } + if event.reason == "IndexersRestarted" { + indexersRestarted = true + } + } + assert.True(t, queueUpdated) + assert.True(t, indexersRestarted) +} diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index b0f866413..5a220aabd 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -17,12 +17,13 @@ package enterprise import ( "context" "fmt" + "log/slog" "reflect" "strings" "time" - "github.com/go-logr/logr" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -31,7 +32,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -45,8 +45,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyIngestorCluster") + logger := logging.FromContext(ctx).With("func", "ApplyIngestorCluster", "name", cr.GetName(), "namespace", cr.GetNamespace()) if cr.Status.ResourceRevMap == nil { cr.Status.ResourceRevMap = make(map[string]string) @@ -60,17 +59,21 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Validate and updates defaults for CR err = validateIngestorClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateIngestorClusterSpec", fmt.Sprintf("validate ingestor cluster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate ingestor cluster spec") + eventPublisher.Warning(ctx, "ValidateIngestorClusterSpecFailed", "Validate Ingestor Cluster spec failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to validate Ingestor Cluster spec", "error", err.Error()) return result, err } // Initialize phase cr.Status.Phase = enterpriseApi.PhaseError + // Track previous ready replicas for scaling events + previousReadyReplicas := cr.Status.ReadyReplicas + // Update the CR Status defer updateCRStatus(ctx, client, cr, &err) if cr.Status.Replicas < cr.Spec.Replicas { + logger.InfoContext(ctx, "Scaling up ingestor cluster", "previousReplicas", cr.Status.Replicas, "newReplicas", cr.Spec.Replicas) cr.Status.CredentialSecretVersion = "0" cr.Status.ServiceAccount = "" } @@ -88,7 +91,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err = initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, "AppInfoStatusInitializationFailed", "Init and check app info status failed. Check operator logs for details.") cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -99,8 +102,8 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIngestor) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Apply of general config failed. Check operator logs for details.") + logger.ErrorContext(ctx, "create or update general config failed", "error", err.Error()) return result, err } @@ -109,7 +112,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, make([]corev1.EnvVar, 0), false) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMap", fmt.Sprintf("create/update monitoring console config map failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMapFailed", "Apply of monitoring console config map failed. Check operator logs for details.") return result, err } } @@ -138,14 +141,14 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update a headless service for ingestor cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIngestor, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service for ingestor cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Apply of headless service failed. Check operator logs for details.") return result, err } // Create or update a regular service for ingestor cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIngestor, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update service for ingestor cluster failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyServiceFailed", "Apply of service failed. Check operator logs for details.") return result, err } @@ -187,14 +190,14 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update statefulset for the ingestors statefulSet, err := getIngestorStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getIngestorStatefulSet", fmt.Sprintf("get ingestor stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "GetIngestorStatefulSetFailed", "Get stateful set failed. Check operator logs for details.") return result, err } // Make changes to respective mc configmap when changing/removing mcRef from spec err = validateMonitoringConsoleRef(ctx, client, statefulSet, make([]corev1.EnvVar, 0)) if err != nil { - eventPublisher.Warning(ctx, "validateMonitoringConsoleRef", fmt.Sprintf("validate monitoring console reference failed %s", err.Error())) + eventPublisher.Warning(ctx, "MonitoringConsoleRefValidationFailed", "Monitoring console reference validation failed. Check operator logs for details.") return result, err } @@ -202,56 +205,82 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr phase, err := mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas if err != nil { - eventPublisher.Warning(ctx, "update", fmt.Sprintf("update stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, "UpdateStatefulSetFailed", "Stateful set update failed. Check operator logs for details.") return result, err } cr.Status.Phase = phase + // Emit scaling events when phase is ready and ready replicas changed to match desired + if phase == enterpriseApi.PhaseReady { + desiredReplicas := cr.Spec.Replicas + if cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { + if desiredReplicas > previousReadyReplicas { + eventPublisher.Normal(ctx, "ScaledUp", + fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) + } else if desiredReplicas < previousReadyReplicas { + eventPublisher.Normal(ctx, "ScaledDown", + fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) + } + } + } + // No need to requeue if everything is ready if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - scopedLog.Error(err, "Failed to resolve Queue/ObjectStorage config") + logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) return result, err } + logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) secretChanged := cr.Status.CredentialSecretVersion != qosCfg.Version serviceAccountChanged := cr.Status.ServiceAccount != cr.Spec.ServiceAccount + logger.DebugContext(ctx, "Checking for changes", "previousCredentialSecretVersion", cr.Status.CredentialSecretVersion, "previousServiceAccount", cr.Status.ServiceAccount, "secretChanged", secretChanged, "serviceAccountChanged", serviceAccountChanged) + // If queue is updated if secretChanged || serviceAccountChanged { - mgr := newIngestorClusterPodManager(scopedLog, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) - err = mgr.updateIngestorConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) + ingMgr := newIngestorClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) + err = ingMgr.updateIngestorConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "ApplyIngestorCluster", fmt.Sprintf("Failed to update conf file for Queue/Pipeline config change after pod creation: %s", err.Error())) - scopedLog.Error(err, "Failed to update conf file for Queue/Pipeline config change after pod creation") + eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") + logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) return result, err } + eventPublisher.Normal(ctx, "QueueConfigUpdated", + fmt.Sprintf("Queue/Pipeline configuration updated for %d ingestors", cr.Spec.Replicas)) + logger.InfoContext(ctx, "Queue/Pipeline configuration updated", "readyReplicas", cr.Status.ReadyReplicas) + for i := int32(0); i < cr.Spec.Replicas; i++ { - ingClient := mgr.getClient(ctx, i) + ingClient := ingMgr.getClient(ctx, i) err = ingClient.RestartSplunk() if err != nil { return result, err } - scopedLog.Info("Restarted splunk", "ingestor", i) + logger.DebugContext(ctx, "Restarted splunk", "ingestor", i) } + eventPublisher.Normal(ctx, "IngestorsRestarted", + fmt.Sprintf("Restarted Splunk on %d ingestor pods", cr.Spec.Replicas)) + cr.Status.CredentialSecretVersion = qosCfg.Version cr.Status.ServiceAccount = cr.Spec.ServiceAccount + + logger.InfoContext(ctx, "Updated status", "credentialSecretVersion", cr.Status.CredentialSecretVersion, "serviceAccount", cr.Status.ServiceAccount) } - // Upgrade fron automated MC to MC CRD + // Upgrade from automated MC to MC CRD namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "DeleteReferencesToAutomatedMCIfExists", fmt.Sprintf("delete reference to automated MC if exists failed %s", err.Error())) - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + eventPublisher.Warning(ctx, "MCReferencesDeletionFailed", "Delete of reference to automated MC failed. Check operator logs for details.") + logger.ErrorContext(ctx, "Delete of reference to automated MC failed", "error", err.Error()) } if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, make([]corev1.EnvVar, 0), true) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMap", fmt.Sprintf("apply monitoring console environment config map failed %s", err.Error())) + eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMapFailed", "Apply of monitoring console environment config map failed. Check operator logs for details.") return result, err } } @@ -283,8 +312,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // getClient for ingestorClusterPodManager returns a SplunkClient for the member n func (mgr *ingestorClusterPodManager) getClient(ctx context.Context, n int32) *splclient.SplunkClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ingestorClusterPodManager.getClient").WithValues("name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getClient", "name", mgr.cr.GetName(), "namespace", mgr.cr.GetNamespace()) // Get Pod Name memberName := GetSplunkStatefulsetPodName(SplunkIngestor, mgr.cr.GetName(), n) @@ -296,7 +324,7 @@ func (mgr *ingestorClusterPodManager) getClient(ctx context.Context, n int32) *s // Retrieve admin password from Pod adminPwd, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, memberName, mgr.cr.GetNamespace(), "password") if err != nil { - scopedLog.Error(err, "Couldn't retrieve the admin password from pod") + logger.ErrorContext(ctx, "Couldn't retrieve the admin password from pod", "error", err.Error()) } return mgr.newSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", adminPwd) @@ -334,11 +362,10 @@ func getIngestorStatefulSet(ctx context.Context, client splcommon.ControllerClie // updateIngestorConfFiles checks if Queue or Pipeline inputs are created for the first time and updates the conf file if so func (mgr *ingestorClusterPodManager) updateIngestorConfFiles(ctx context.Context, newCR *enterpriseApi.IngestorCluster, queue *enterpriseApi.QueueSpec, os *enterpriseApi.ObjectStorageSpec, accessKey, secretKey string, k8s client.Client) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateIngestorConfFiles").WithValues("name", newCR.GetName(), "namespace", newCR.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "updateIngestorConfFiles", "name", newCR.GetName(), "namespace", newCR.GetNamespace()) // Only update config for pods that exist - readyReplicas := newCR.Status.Replicas + readyReplicas := newCR.Status.ReadyReplicas // List all pods for this IngestorCluster StatefulSet var updateErr error @@ -354,16 +381,22 @@ func (mgr *ingestorClusterPodManager) updateIngestorConfFiles(ctx context.Contex queueInputs, pipelineInputs := getQueueAndPipelineInputsForIngestorConfFiles(queue, os, accessKey, secretKey) for _, input := range queueInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{input}); err != nil { + if !strings.Contains(input[0], "access_key") && !strings.Contains(input[0], "secret_key") { + logger.DebugContext(ctx, "Updating queue input in outputs.conf", "input", input) + } + if err := splunkClient.UpdateConfFile(ctx, "outputs", fmt.Sprintf("remote_queue:%s", queue.SQS.Name), [][]string{input}); err != nil { updateErr = err } } for _, input := range pipelineInputs { - if err := splunkClient.UpdateConfFile(scopedLog, "default-mode", input[0], [][]string{{input[1], input[2]}}); err != nil { + logger.DebugContext(ctx, "Updating pipeline input in default-mode.conf", "input", input) + if err := splunkClient.UpdateConfFile(ctx, "default-mode", input[0], [][]string{{input[1], input[2]}}); err != nil { updateErr = err } } + + logger.InfoContext(ctx, "Updated conf files for pod", "pod", memberName) } return updateErr @@ -382,14 +415,14 @@ func getQueueAndPipelineInputsForIngestorConfFiles(queue *enterpriseApi.QueueSpe type ingestorClusterPodManager struct { c splcommon.ControllerClient - log logr.Logger + log *slog.Logger cr *enterpriseApi.IngestorCluster secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient } // newIngestorClusterPodManager creates pod manager to handle unit test cases -var newIngestorClusterPodManager = func(log logr.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { +var newIngestorClusterPodManager = func(log *slog.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, newSplunkClient NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { return ingestorClusterPodManager{ log: log, cr: cr, diff --git a/pkg/splunk/enterprise/ingestorcluster_test.go b/pkg/splunk/enterprise/ingestorcluster_test.go index e96002372..207346f82 100644 --- a/pkg/splunk/enterprise/ingestorcluster_test.go +++ b/pkg/splunk/enterprise/ingestorcluster_test.go @@ -17,13 +17,13 @@ package enterprise import ( "context" "fmt" + "log/slog" "net/http" "os" "path/filepath" "strings" "testing" - "github.com/go-logr/logr" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -34,6 +34,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -67,10 +68,6 @@ func TestApplyIngestorCluster(t *testing.T) { provider := "sqs_smartbus" queue := &enterpriseApi.Queue{ - TypeMeta: metav1.TypeMeta{ - Kind: "Queue", - APIVersion: "enterprise.splunk.com/v4", - }, ObjectMeta: metav1.ObjectMeta{ Name: "queue", Namespace: "test", @@ -252,7 +249,7 @@ func TestApplyIngestorCluster(t *testing.T) { // outputs.conf origNew := newIngestorClusterPodManager mockHTTPClient := &spltest.MockHTTPClient{} - newIngestorClusterPodManager = func(l logr.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, _ NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { + newIngestorClusterPodManager = func(l *slog.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, _ NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { return ingestorClusterPodManager{ c: c, log: l, cr: cr, secrets: secret, @@ -746,7 +743,7 @@ func TestUpdateIngestorConfFiles(t *testing.T) { func addRemoteQueueHandlersForIngestor(mockHTTPClient *spltest.MockHTTPClient, cr *enterpriseApi.IngestorCluster, queue *enterpriseApi.QueueSpec, confName, body string) { for i := 0; i < int(cr.Status.ReadyReplicas); i++ { - podName := fmt.Sprintf("splunk-%s-ingestor-%d", cr.GetName(), i) + podName := GetSplunkStatefulsetPodName(SplunkIngestor, cr.GetName(), int32(i)) baseURL := fmt.Sprintf( "https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/%s", podName, cr.GetName(), cr.GetNamespace(), confName, @@ -774,4 +771,312 @@ func newTestIngestorQueuePipelineManager(mockHTTPClient *spltest.MockHTTPClient) return &ingestorClusterPodManager{ newSplunkClient: newSplunkClientForQueuePipeline, } + +} + +func TestIngScaledUpQueueConfigUpdatedIngestorsRestartedScaledDownEvents(t *testing.T) { + os.Setenv("SPLUNK_GENERAL_TERMS", "--accept-sgt-current-at-splunk-com") + + ctx := context.TODO() + recorder := &mockEventRecorder{events: []mockEvent{}} + eventPublisher := &K8EventPublisher{recorder: recorder} + ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) + + scheme := runtime.NewScheme() + _ = enterpriseApi.AddToScheme(scheme) + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + queue := &enterpriseApi.Queue{ + TypeMeta: metav1.TypeMeta{ + Kind: "Queue", + APIVersion: "enterprise.splunk.com/v4", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "queue", + Namespace: "test", + }, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "test-queue", + AuthRegion: "us-west-2", + Endpoint: "https://sqs.us-west-2.amazonaws.com", + DLQ: "sqs-dlq-test", + }, + }, + } + _ = c.Create(ctx, queue) + + objStorage := &enterpriseApi.ObjectStorage{ + ObjectMeta: metav1.ObjectMeta{ + Name: "os", + Namespace: "test", + }, + Spec: enterpriseApi.ObjectStorageSpec{ + Provider: "s3", + S3: enterpriseApi.S3Spec{ + Endpoint: "https://s3.us-west-2.amazonaws.com", + Path: "bucket/key", + }, + }, + } + _ = c.Create(ctx, objStorage) + + probeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "splunk-test-probe-configmap", + Namespace: "test", + }, + } + _ = c.Create(ctx, probeConfigMap) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secrets", + Namespace: "test", + }, + Data: map[string][]byte{"password": []byte("dummy")}, + } + _ = c.Create(ctx, secret) + + cr := &enterpriseApi.IngestorCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingestor", + Namespace: "test", + }, + Spec: enterpriseApi.IngestorClusterSpec{ + Replicas: 1, + CommonSplunkSpec: enterpriseApi.CommonSplunkSpec{ + Mock: true, + ServiceAccount: "sa", + }, + QueueRef: corev1.ObjectReference{ + Name: queue.Name, + Namespace: queue.Namespace, + }, + ObjectStorageRef: corev1.ObjectReference{ + Name: objStorage.Name, + Namespace: objStorage.Namespace, + }, + }, + Status: enterpriseApi.IngestorClusterStatus{ + Replicas: 1, + CredentialSecretVersion: "", + ServiceAccount: "sa", + TelAppInstalled: true, + }, + } + _ = c.Create(ctx, cr) + + oneReplica := int32(1) + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetSplunkStatefulsetName(SplunkIngestor, cr.GetName()), + Namespace: cr.GetNamespace(), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &oneReplica, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "splunk", + Image: "splunk/splunk:latest", + }, + }, + }, + }, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: oneReplica, + ReadyReplicas: oneReplica, + CurrentRevision: "v1", + UpdateRevision: "v1", + }, + } + _ = c.Create(ctx, sts) + + basePod := &corev1.Pod{ + Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: "mnt-splunk-secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "test-secrets", + }, + }, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + {Ready: true}, + }, + }, + } + pod := basePod.DeepCopy() + pod.ObjectMeta = metav1.ObjectMeta{ + Name: GetSplunkStatefulsetPodName(SplunkIngestor, cr.GetName(), 0), + Namespace: cr.GetNamespace(), + Labels: map[string]string{ + "app.kubernetes.io/instance": GetSplunkStatefulsetName(SplunkIngestor, cr.GetName()), + "controller-revision-hash": "v1", + }, + } + _ = c.Create(ctx, pod) + + _, err := ApplyIngestorCluster(ctx, c, cr) + assert.NoError(t, err) + + // ===== Scale up (and trigger queue update + restart) ===== + threeReplicas := int32(3) + cr.Spec.Replicas = threeReplicas + _ = c.Update(ctx, sts) + for i := int32(1); i < threeReplicas; i++ { + pod := basePod.DeepCopy() + pod.ObjectMeta = metav1.ObjectMeta{ + Name: GetSplunkStatefulsetPodName(SplunkIngestor, cr.GetName(), i), + Namespace: cr.GetNamespace(), + Labels: map[string]string{ + "app.kubernetes.io/instance": GetSplunkStatefulsetName(SplunkIngestor, cr.GetName()), + "controller-revision-hash": "v1", + }, + } + _ = c.Create(ctx, pod) + } + + // Reconcile once so ApplyIngestorCluster can update the StatefulSet spec/template as needed. + _, err = ApplyIngestorCluster(ctx, c, cr) + assert.NoError(t, err) + + // Now simulate the StatefulSet controller reporting the desired state as ready. + _ = c.Get(ctx, client.ObjectKey{Name: GetSplunkStatefulsetName(SplunkIngestor, cr.GetName()), Namespace: cr.GetNamespace()}, sts) + sts.Status.Replicas = threeReplicas + sts.Status.ReadyReplicas = threeReplicas + _ = c.Status().Update(ctx, sts) + + mockHTTPClient := &spltest.MockHTTPClient{} + origNew := newIngestorClusterPodManager + newIngestorClusterPodManager = func(l *slog.Logger, cr *enterpriseApi.IngestorCluster, secret *corev1.Secret, _ NewSplunkClientFunc, c splcommon.ControllerClient) ingestorClusterPodManager { + return ingestorClusterPodManager{ + c: c, + log: l, + cr: cr, + secrets: secret, + newSplunkClient: func(uri, user, pass string) *splclient.SplunkClient { + return &splclient.SplunkClient{ + ManagementURI: uri, + Username: user, + Password: pass, + Client: mockHTTPClient, + } + }, + } + } + + propertyKVList := [][]string{ + {"remote_queue.type", "sqs_smartbus"}, + {fmt.Sprintf("remote_queue.%s.encoding_format", "sqs_smartbus"), "s2s"}, + {fmt.Sprintf("remote_queue.%s.auth_region", "sqs_smartbus"), queue.Spec.SQS.AuthRegion}, + {fmt.Sprintf("remote_queue.%s.endpoint", "sqs_smartbus"), queue.Spec.SQS.Endpoint}, + {fmt.Sprintf("remote_queue.%s.large_message_store.endpoint", "sqs_smartbus"), objStorage.Spec.S3.Endpoint}, + {fmt.Sprintf("remote_queue.%s.large_message_store.path", "sqs_smartbus"), objStorage.Spec.S3.Path}, + {fmt.Sprintf("remote_queue.%s.dead_letter_queue.name", "sqs_smartbus"), queue.Spec.SQS.DLQ}, + {fmt.Sprintf("remote_queue.%s.max_count.max_retries_per_part", "sqs_smartbus"), "4"}, + {fmt.Sprintf("remote_queue.%s.retry_policy", "sqs_smartbus"), "max_count"}, + {fmt.Sprintf("remote_queue.%s.send_interval", "sqs_smartbus"), "5s"}, + } + body := buildFormBody(propertyKVList) + + crForHandlers := cr.DeepCopy() + crForHandlers.Status.ReadyReplicas = threeReplicas + addRemoteQueueHandlersForIngestor(mockHTTPClient, crForHandlers, &queue.Spec, "conf-outputs", body) + + propertyKVList = [][]string{ + {"pipeline:remotequeueruleset", "disabled", "false"}, + {"pipeline:ruleset", "disabled", "true"}, + {"pipeline:remotequeuetyping", "disabled", "false"}, + {"pipeline:remotequeueoutput", "disabled", "false"}, + {"pipeline:typing", "disabled", "true"}, + {"pipeline:indexerPipe", "disabled", "true"}, + } + for i := int32(0); i < threeReplicas; i++ { + podName := GetSplunkStatefulsetPodName(SplunkIngestor, cr.GetName(), i) + baseURL := fmt.Sprintf("https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/servicesNS/nobody/system/configs/conf-default-mode", podName, cr.GetName(), cr.GetNamespace()) + for _, field := range propertyKVList { + req, _ := http.NewRequest("POST", baseURL, strings.NewReader(fmt.Sprintf("name=%s", field[0]))) + mockHTTPClient.AddHandler(req, 200, "", nil) + updateURL := fmt.Sprintf("%s/%s", baseURL, field[0]) + req, _ = http.NewRequest("POST", updateURL, strings.NewReader(fmt.Sprintf("%s=%s", field[1], field[2]))) + mockHTTPClient.AddHandler(req, 200, "", nil) + } + + baseURL = fmt.Sprintf("https://%s.splunk-%s-ingestor-headless.%s.svc.cluster.local:8089/services/server/control/restart", podName, cr.GetName(), cr.GetNamespace()) + req, _ := http.NewRequest("POST", baseURL, nil) + mockHTTPClient.AddHandler(req, 200, "", nil) + } + + cr.Status.ServiceAccount = "" + cr.Status.CredentialSecretVersion = "old" + _, err = ApplyIngestorCluster(ctx, c, cr) + assert.NoError(t, err) + assert.Equal(t, enterpriseApi.PhaseReady, cr.Status.Phase) + assert.Equal(t, threeReplicas, cr.Status.ReadyReplicas) + + scaledUp := false + queueUpdated := false + ingestorsRestarted := false + for _, event := range recorder.events { + if event.reason == "ScaledUp" { + scaledUp = true + } + if event.reason == "QueueConfigUpdated" { + queueUpdated = true + } + if event.reason == "IngestorsRestarted" { + ingestorsRestarted = true + } + } + assert.True(t, scaledUp) + assert.True(t, queueUpdated) + assert.True(t, ingestorsRestarted) + + // Stop mocking Splunk API calls before scaling down. + newIngestorClusterPodManager = origNew + // Reset event recorder so scale-down assertion isn't polluted. + recorder.events = []mockEvent{} + + // ===== Scale down ===== + cr.Spec.Replicas = oneReplica + cr.Status.Replicas = threeReplicas + cr.Status.ReadyReplicas = threeReplicas + cr.Status.CredentialSecretVersion = "" + cr.Status.ServiceAccount = "sa" + + sts.Spec.Replicas = &oneReplica + _ = c.Update(ctx, sts) + sts.Status.Replicas = oneReplica + sts.Status.ReadyReplicas = oneReplica + _ = c.Status().Update(ctx, sts) + for i := int32(1); i < threeReplicas; i++ { + _ = c.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: GetSplunkStatefulsetPodName(SplunkIngestor, cr.GetName(), i), Namespace: cr.GetNamespace()}}) + } + + _, err = ApplyIngestorCluster(ctx, c, cr) + assert.NoError(t, err) + assert.Equal(t, enterpriseApi.PhaseReady, cr.Status.Phase) + assert.Equal(t, oneReplica, cr.Status.ReadyReplicas) + + scaledDown := false + for _, event := range recorder.events { + if event.reason == "ScaledDown" { + scaledDown = true + break + } + } + assert.True(t, scaledDown) } diff --git a/pkg/splunk/enterprise/standalone.go b/pkg/splunk/enterprise/standalone.go index 288f383be..36bf92a0c 100644 --- a/pkg/splunk/enterprise/standalone.go +++ b/pkg/splunk/enterprise/standalone.go @@ -214,8 +214,8 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr return result, err } - // Track last successful replica count to emit scale events after completion - previousReplicas := cr.Status.Replicas + // Track previous ready replicas for scaling events + previousReadyReplicas := cr.Status.ReadyReplicas mgr := splctrl.DefaultStatefulSetPodManager{} phase, err := mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) @@ -227,18 +227,20 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr } cr.Status.Phase = phase - // Emit scale events only after a successful scale operation has completed + // Emit scale events when phase is ready and ready replicas changed to match desired if phase == enterpriseApi.PhaseReady { desiredReplicas := cr.Spec.Replicas - if desiredReplicas > previousReplicas && cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", - fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", cr.GetName(), previousReplicas, desiredReplicas)) - } - } else if desiredReplicas < previousReplicas && cr.Status.Replicas == desiredReplicas { - if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", - fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", cr.GetName(), previousReplicas, desiredReplicas)) + if cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { + if desiredReplicas > previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledUp", + fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) + } + } else if desiredReplicas < previousReadyReplicas { + if eventPublisher != nil { + eventPublisher.Normal(ctx, "ScaledDown", + fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) + } } } } diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index 4b5d77e8f..3b525f94a 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -2700,7 +2700,6 @@ func resolveS3Endpoint(ctx context.Context, region string) (string, error) { if err != nil { return "", err } - // Full endpoint URL as string: return ep.URI.String(), nil } diff --git a/pkg/splunk/enterprise/util_test.go b/pkg/splunk/enterprise/util_test.go index 7168e366a..b2ac6ba4d 100644 --- a/pkg/splunk/enterprise/util_test.go +++ b/pkg/splunk/enterprise/util_test.go @@ -3522,8 +3522,8 @@ func TestAppRepositoryConnectionFailedEvent(t *testing.T) { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{Name: "test-s3-secret", Namespace: "test"}, Data: map[string][]byte{ - "s3_access_key": []byte("AKIAIOSFODNN7EXAMPLE"), - "s3_secret_key": []byte("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"), + "s3_access_key": []byte("abc"), + "s3_secret_key": []byte("123"), }, } if err := client.Create(ctx, secret); err != nil { @@ -3573,3 +3573,416 @@ func TestAppRepositoryConnectionFailedEvent(t *testing.T) { t.Errorf("Expected AppRepositoryConnectionFailed event to be published") } } + +func TestResolveSQSEndpoint(t *testing.T) { + ctx := context.TODO() + + tests := []struct { + name string + region string + wantEndpoint string + wantErrContain string + }{ + { + name: "valid us-east-1 region", + region: "us-east-1", + wantEndpoint: "https://sqs.us-east-1.amazonaws.com", + }, + { + name: "valid eu-west-1 region", + region: "eu-west-1", + wantEndpoint: "https://sqs.eu-west-1.amazonaws.com", + }, + { + name: "valid ap-southeast-1 region", + region: "ap-southeast-1", + wantEndpoint: "https://sqs.ap-southeast-1.amazonaws.com", + }, + { + name: "valid cn-north-1 region (China)", + region: "cn-north-1", + wantEndpoint: "https://sqs.cn-north-1.amazonaws.com.cn", + }, + { + name: "valid cn-northwest-1 region (China Ningxia)", + region: "cn-northwest-1", + wantEndpoint: "https://sqs.cn-northwest-1.amazonaws.com.cn", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + endpoint, err := resolveSQSEndpoint(ctx, tt.region) + if tt.wantErrContain != "" { + if err == nil { + t.Errorf("resolveSQSEndpoint() expected error containing %q, got nil", tt.wantErrContain) + } else if !strings.Contains(err.Error(), tt.wantErrContain) { + t.Errorf("resolveSQSEndpoint() error = %v, want error containing %q", err, tt.wantErrContain) + } + return + } + if err != nil { + t.Errorf("resolveSQSEndpoint() unexpected error = %v", err) + return + } + if endpoint != tt.wantEndpoint { + t.Errorf("resolveSQSEndpoint() = %v, want %v", endpoint, tt.wantEndpoint) + } + }) + } +} + +func TestResolveS3Endpoint(t *testing.T) { + ctx := context.TODO() + + tests := []struct { + name string + region string + wantEndpoint string + wantErrContain string + }{ + { + name: "valid us-east-1 region", + region: "us-east-1", + wantEndpoint: "https://s3.us-east-1.amazonaws.com", + }, + { + name: "valid eu-west-1 region", + region: "eu-west-1", + wantEndpoint: "https://s3.eu-west-1.amazonaws.com", + }, + { + name: "valid ap-southeast-1 region", + region: "ap-southeast-1", + wantEndpoint: "https://s3.ap-southeast-1.amazonaws.com", + }, + { + name: "valid cn-north-1 region (China)", + region: "cn-north-1", + wantEndpoint: "https://s3.cn-north-1.amazonaws.com.cn", + }, + { + name: "valid cn-northwest-1 region (China Ningxia)", + region: "cn-northwest-1", + wantEndpoint: "https://s3.cn-northwest-1.amazonaws.com.cn", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + endpoint, err := resolveS3Endpoint(ctx, tt.region) + if tt.wantErrContain != "" { + if err == nil { + t.Errorf("resolveS3Endpoint() expected error containing %q, got nil", tt.wantErrContain) + } else if !strings.Contains(err.Error(), tt.wantErrContain) { + t.Errorf("resolveS3Endpoint() error = %v, want error containing %q", err, tt.wantErrContain) + } + return + } + if err != nil { + t.Errorf("resolveS3Endpoint() unexpected error = %v", err) + return + } + if endpoint != tt.wantEndpoint { + t.Errorf("resolveS3Endpoint() = %v, want %v", endpoint, tt.wantEndpoint) + } + }) + } +} + +func TestResolveQueueAndObjectStorage(t *testing.T) { + ctx := context.TODO() + + sch := pkgruntime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(sch)) + utilruntime.Must(corev1.AddToScheme(sch)) + utilruntime.Must(enterpriseApi.AddToScheme(sch)) + + t.Run("empty refs returns empty config", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(sch).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, corev1.ObjectReference{}, corev1.ObjectReference{}, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg == nil { + t.Fatal("ResolveQueueAndObjectStorage() returned nil config") + } + if cfg.Queue.Provider != "" { + t.Errorf("Expected empty Queue.Provider, got %q", cfg.Queue.Provider) + } + if cfg.OS.Provider != "" { + t.Errorf("Expected empty OS.Provider, got %q", cfg.OS.Provider) + } + }) + + t.Run("queue ref not found returns error", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(sch).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "nonexistent-queue"} + _, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "") + if err == nil { + t.Error("ResolveQueueAndObjectStorage() expected error for nonexistent queue, got nil") + } + }) + + t.Run("objectstorage ref not found returns error", func(t *testing.T) { + client := fake.NewClientBuilder().WithScheme(sch).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + osRef := corev1.ObjectReference{Name: "nonexistent-os"} + _, err := ResolveQueueAndObjectStorage(ctx, client, cr, corev1.ObjectReference{}, osRef, "") + if err == nil { + t.Error("ResolveQueueAndObjectStorage() expected error for nonexistent objectstorage, got nil") + } + }) + + t.Run("valid queue ref returns queue spec", func(t *testing.T) { + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "my-queue", + AuthRegion: "", + DLQ: "my-dlq", + Endpoint: "https://sqs.us-east-1.amazonaws.com", + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg.Queue.Provider != "sqs" { + t.Errorf("Expected Queue.Provider = 'sqs', got %q", cfg.Queue.Provider) + } + if cfg.Queue.SQS.Name != "my-queue" { + t.Errorf("Expected Queue.SQS.Name = 'my-queue', got %q", cfg.Queue.SQS.Name) + } + if cfg.Queue.SQS.Endpoint != "https://sqs.us-east-1.amazonaws.com" { + t.Errorf("Expected Queue.SQS.Endpoint = 'https://sqs.us-east-1.amazonaws.com', got %q", cfg.Queue.SQS.Endpoint) + } + }) + + t.Run("valid objectstorage ref returns os spec", func(t *testing.T) { + os := &enterpriseApi.ObjectStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "test-os", Namespace: "test"}, + Spec: enterpriseApi.ObjectStorageSpec{ + Provider: "s3", + S3: enterpriseApi.S3Spec{ + Endpoint: "https://s3.us-east-1.amazonaws.com", + Path: "my-bucket/prefix", + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(os).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + osRef := corev1.ObjectReference{Name: "test-os"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, corev1.ObjectReference{}, osRef, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg.OS.Provider != "s3" { + t.Errorf("Expected OS.Provider = 's3', got %q", cfg.OS.Provider) + } + if cfg.OS.S3.Path != "my-bucket/prefix" { + t.Errorf("Expected OS.S3.Path = 'my-bucket/prefix', got %q", cfg.OS.S3.Path) + } + }) + + t.Run("queue ref with different namespace", func(t *testing.T) { + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "other-ns"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "cross-ns-queue", + DLQ: "my-dlq", + Endpoint: "https://sqs.eu-west-1.amazonaws.com", + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue", Namespace: "other-ns"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg.Queue.SQS.Name != "cross-ns-queue" { + t.Errorf("Expected Queue.SQS.Name = 'cross-ns-queue', got %q", cfg.Queue.SQS.Name) + } + }) + + t.Run("queue with secret ref extracts credentials", func(t *testing.T) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "aws-creds", Namespace: "test"}, + Data: map[string][]byte{ + "s3_access_key": []byte("abc"), + "s3_secret_key": []byte("123"), + }, + } + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "my-queue", + DLQ: "my-dlq", + Endpoint: "https://sqs.us-east-1.amazonaws.com", + VolList: []enterpriseApi.VolumeSpec{ + { + Name: "vol1", + SecretRef: "aws-creds", + }, + }, + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue, secret).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg.AccessKey != "abc" { + t.Errorf("Expected AccessKey = 'abc', got %q", cfg.AccessKey) + } + if cfg.SecretKey != "123" { + t.Errorf("Expected SecretKey = '123', got %q", cfg.SecretKey) + } + }) + + t.Run("queue with serviceAccount skips secret extraction", func(t *testing.T) { + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "my-queue", + DLQ: "my-dlq", + Endpoint: "https://sqs.us-east-1.amazonaws.com", + VolList: []enterpriseApi.VolumeSpec{ + { + Name: "vol1", + SecretRef: "aws-creds", + }, + }, + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "my-service-account") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + // When serviceAccount is provided, credentials should not be extracted + if cfg.AccessKey != "" { + t.Errorf("Expected empty AccessKey when serviceAccount is provided, got %q", cfg.AccessKey) + } + if cfg.SecretKey != "" { + t.Errorf("Expected empty SecretKey when serviceAccount is provided, got %q", cfg.SecretKey) + } + }) + + t.Run("queue with missing secret returns error", func(t *testing.T) { + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "my-queue", + DLQ: "my-dlq", + Endpoint: "https://sqs.us-east-1.amazonaws.com", + VolList: []enterpriseApi.VolumeSpec{ + { + Name: "vol1", + SecretRef: "nonexistent-secret", + }, + }, + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue"} + _, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, corev1.ObjectReference{}, "") + if err == nil { + t.Error("ResolveQueueAndObjectStorage() expected error for missing secret, got nil") + } + }) + + t.Run("both queue and objectstorage refs", func(t *testing.T) { + queue := &enterpriseApi.Queue{ + ObjectMeta: metav1.ObjectMeta{Name: "test-queue", Namespace: "test"}, + Spec: enterpriseApi.QueueSpec{ + Provider: "sqs", + SQS: enterpriseApi.SQSSpec{ + Name: "my-queue", + DLQ: "my-dlq", + Endpoint: "https://sqs.us-east-1.amazonaws.com", + }, + }, + } + os := &enterpriseApi.ObjectStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "test-os", Namespace: "test"}, + Spec: enterpriseApi.ObjectStorageSpec{ + Provider: "s3", + S3: enterpriseApi.S3Spec{ + Endpoint: "https://s3.us-east-1.amazonaws.com", + Path: "my-bucket", + }, + }, + } + client := fake.NewClientBuilder().WithScheme(sch).WithObjects(queue, os).Build() + cr := &enterpriseApi.IndexerCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test-idxc", Namespace: "test"}, + } + + queueRef := corev1.ObjectReference{Name: "test-queue"} + osRef := corev1.ObjectReference{Name: "test-os"} + cfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, queueRef, osRef, "") + if err != nil { + t.Errorf("ResolveQueueAndObjectStorage() unexpected error = %v", err) + } + if cfg.Queue.Provider != "sqs" { + t.Errorf("Expected Queue.Provider = 'sqs', got %q", cfg.Queue.Provider) + } + if cfg.OS.Provider != "s3" { + t.Errorf("Expected OS.Provider = 's3', got %q", cfg.OS.Provider) + } + }) +} diff --git a/pkg/splunk/util/secrets.go b/pkg/splunk/util/secrets.go index 7240c3916..004266b1b 100644 --- a/pkg/splunk/util/secrets.go +++ b/pkg/splunk/util/secrets.go @@ -197,7 +197,7 @@ func RemoveSecretOwnerRef(ctx context.Context, client splcommon.ControllerClient return refCount, nil } -// RemoveUnwantedSecrets deletes all secrets whose version precedes (latestVersion - MinimumVersionedSecrets) +// RemoveUnwantedSecrets deletes all secrets whose version preceeds (latestVersion - MinimumVersionedSecrets) func RemoveUnwantedSecrets(ctx context.Context, c splcommon.ControllerClient, versionedSecretIdentifier, namespace string) error { logger := slog.With("func", "RemoveUnwantedSecrets") // retrieve the list of versioned namespace scoped secrets From beeec17de7fbefe020ceb76219eb10d56151c506 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Fri, 6 Mar 2026 13:05:20 +0100 Subject: [PATCH 07/19] Remove double logging --- pkg/splunk/enterprise/clustermanager.go | 6 +- pkg/splunk/enterprise/clustermaster.go | 6 +- pkg/splunk/enterprise/indexercluster.go | 102 +++++++-------------- pkg/splunk/enterprise/ingestorcluster.go | 12 +-- pkg/splunk/enterprise/licensemanager.go | 9 +- pkg/splunk/enterprise/licensemaster.go | 6 +- pkg/splunk/enterprise/monitoringconsole.go | 9 +- pkg/splunk/enterprise/searchheadcluster.go | 6 +- pkg/splunk/enterprise/standalone.go | 6 +- 9 files changed, 52 insertions(+), 110 deletions(-) diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index bab4440a9..5125459b9 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -67,8 +67,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, err = validateClusterManagerSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateClusterManagerSpec", fmt.Sprintf("validate clustermanager spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate clustermanager spec") - return result, err + return result, fmt.Errorf("validate clustermanager spec: %w", err) } // updates status after function completes @@ -122,9 +121,8 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // Smart Store secrets get created manually and should not be managed by the Operator diff --git a/pkg/splunk/enterprise/clustermaster.go b/pkg/splunk/enterprise/clustermaster.go index 88e2c3815..1dd78e5e8 100644 --- a/pkg/splunk/enterprise/clustermaster.go +++ b/pkg/splunk/enterprise/clustermaster.go @@ -66,8 +66,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, err = validateClusterMasterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateClusterMasterSpec", fmt.Sprintf("validate clustermaster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate clustermaster spec") - return result, err + return result, fmt.Errorf("validate clustermaster spec: %w", err) } // updates status after function completes @@ -121,9 +120,8 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // Smart Store secrets get created manually and should not be managed by the Operator diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index 3af847603..9b91044c3 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -70,8 +70,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = validateIndexerClusterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "IndexerClusterSpecValidationFailed", "Validation of Indexer Cluster spec failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) - return result, err + return result, fmt.Errorf("validate indexer cluster spec: %w", err) } // updates status after function completes @@ -96,9 +95,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - logger.ErrorContext(ctx, "Create or update of general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update of general config failed. Check operator logs for details.") - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } namespacedName := types.NamespacedName{ @@ -126,8 +124,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = VerifyRFPeers(ctx, mgr, client) if err != nil { eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verification of RF peer failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Verification of RF peer failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("verify RF peers: %w", err) } } @@ -144,7 +141,6 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller } if err != nil { eventPublisher.Warning(ctx, "DeletionFailed", "Deletion of custom resource failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Deletion of custom resource failed", "error", err.Error()) } return result, err } @@ -152,24 +148,21 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Create or update of headless service for Indexer Cluster failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("apply headless service: %w", err) } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Create or update of service for Indexer Cluster failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("apply service: %w", err) } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("get indexer statefulset: %w", err) } // Note: @@ -228,16 +221,14 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("update statefulset: %w", err) } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("delete statefulset: %w", err) } time.Sleep(1 * time.Second) // since we are creating new statefulset, setting resourceVersion to "" @@ -245,8 +236,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("update statefulset: %w", err) } } cr.Status.Phase = phase @@ -255,8 +245,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) - return result, err + return result, fmt.Errorf("resolve queue/object storage config: %w", err) } logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) @@ -272,8 +261,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) - return result, err + return result, fmt.Errorf("update queue/pipeline conf files: %w", err) } eventPublisher.Normal(ctx, "QueueConfigUpdated", @@ -304,8 +292,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieval of Cluster Manager spec failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Retrieval of Cluster Manager spec failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("retrieve CM spec: %w", err) } if cmMonitoringConsoleConfigRef != "" { namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cmMonitoringConsoleConfigRef)} @@ -316,8 +303,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err := c.AutomateMCApplyChanges() if err != nil { eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Get Monitoring Console client failed. Check operator logs for details.") - logger.ErrorContext(ctx, "get Monitoring Console client failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("automate MC apply changes: %w", err) } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { @@ -337,8 +323,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { eventPublisher.Warning(ctx, "ClusterMaintenanceModeFailed", "Set Cluster maintenance mode failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Set Cluster maintenance mode failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("set cluster maintenance mode: %w", err) } } @@ -356,9 +341,8 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true - return result, err + return result, fmt.Errorf("set statefulset owner ref: %w", err) } } // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. @@ -386,8 +370,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err := validateIndexerClusterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "ValidateIndexerClusterSpecFailed", "Validate Indexer Cluster spec failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) - return result, err + return result, fmt.Errorf("validate indexer cluster spec: %w", err) } // updates status after function completes @@ -416,9 +399,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { - logger.ErrorContext(ctx, "Create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update general config failed. Check operator logs for details.") - return result, err + eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update of general config failed. Check operator logs for details.") + return result, fmt.Errorf("apply splunk config: %w", err) } namespacedName := types.NamespacedName{ @@ -445,8 +427,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err = VerifyRFPeers(ctx, mgr, client) if err != nil { eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verify RF peer failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Verify RF peer failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("verify RF peers: %w", err) } } @@ -463,7 +444,6 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, } if err != nil { eventPublisher.Warning(ctx, "DeleteFailed", "Delete custom resource failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Delete custom resource failed", "error", err.Error()) } return result, err } @@ -472,24 +452,21 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Create or update of headless service failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("apply headless service: %w", err) } // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Create or update of service failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("apply service: %w", err) } // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("get indexer statefulset: %w", err) } // Note: @@ -548,16 +525,14 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("update statefulset: %w", err) } } else { // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("delete statefulset: %w", err) } time.Sleep(1 * time.Second) // since we are creating new statefulset, setting resourceVersion to "" @@ -565,8 +540,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("update statefulset: %w", err) } } cr.Status.Phase = phase @@ -575,8 +549,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) - return result, err + return result, fmt.Errorf("resolve queue/object storage config: %w", err) } logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) @@ -591,8 +564,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { eventPublisher.Warning(ctx, "UpdateIndexerConfFileFailed", "Failed to update conf file for Queue/Pipeline config change after pod creation. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config change after pod creation", "error", err.Error()) - return result, err + return result, fmt.Errorf("update queue/pipeline conf files: %w", err) } eventPublisher.Normal(ctx, "QueueConfigUpdated", @@ -623,8 +595,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieve Cluster Master spec failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Retrieve Cluster Master spec failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("retrieve CM spec: %w", err) } if cmMonitoringConsoleConfigRef != "" { namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cmMonitoringConsoleConfigRef)} @@ -635,8 +606,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err := c.AutomateMCApplyChanges() if err != nil { eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Automate MC Apply Changes failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Automate MC Apply Changes failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("automate MC apply changes: %w", err) } } if len(cr.Spec.MonitoringConsoleRef.Name) > 0 && (cr.Spec.MonitoringConsoleRef.Name != cmMonitoringConsoleConfigRef) { @@ -656,8 +626,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { eventPublisher.Warning(ctx, "SetClusterMaintenanceModeFailed", "Set Cluster Master maintenance mode failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Set Cluster Master maintenance mode failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("set cluster maintenance mode: %w", err) } } @@ -673,9 +642,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true - return result, err + return result, fmt.Errorf("set statefulset owner ref: %w", err) } } // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. @@ -871,8 +839,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica eventPublisher.Warning(ctx, "PasswordSyncFailed", fmt.Sprintf("Password sync failed for pod '%s'. Check operator logs for details.", indexerPodName)) } - logger.ErrorContext(ctx, "Configuration push failed during restart", "failedPeer", indexerPodName, "error", err.Error()) - return err + return fmt.Errorf("configuration push failed during restart for peer %s: %w", indexerPodName, err) } logger.InfoContext(ctx, "Restarted splunk") @@ -1322,14 +1289,11 @@ func validateIndexerClusterSpec(ctx context.Context, c splcommon.ControllerClien // helper function to get the list of IndexerCluster types in the current namespace func getIndexerClusterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []rclient.ListOption) (enterpriseApi.IndexerClusterList, error) { - logger := slog.With("func", "getIndexerClusterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) - objectList := enterpriseApi.IndexerClusterList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - logger.ErrorContext(ctx, fmt.Sprintf("IndexerCluster types not found in namespace %s", cr.GetNamespace()), "error", err.Error()) - return objectList, err + return objectList, fmt.Errorf("list IndexerCluster in namespace %s: %w", cr.GetNamespace(), err) } return objectList, nil diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index 5a220aabd..7157eb0cc 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -60,8 +60,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr err = validateIngestorClusterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "ValidateIngestorClusterSpecFailed", "Validate Ingestor Cluster spec failed. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to validate Ingestor Cluster spec", "error", err.Error()) - return result, err + return result, fmt.Errorf("validate ingestor cluster spec: %w", err) } // Initialize phase @@ -103,8 +102,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIngestor) if err != nil { eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Apply of general config failed. Check operator logs for details.") - logger.ErrorContext(ctx, "create or update general config failed", "error", err.Error()) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // Check if deletion has been requested @@ -228,8 +226,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr if cr.Status.Phase == enterpriseApi.PhaseReady { qosCfg, err := ResolveQueueAndObjectStorage(ctx, client, cr, cr.Spec.QueueRef, cr.Spec.ObjectStorageRef, cr.Spec.ServiceAccount) if err != nil { - logger.ErrorContext(ctx, "Failed to resolve Queue/ObjectStorage config", "error", err.Error()) - return result, err + return result, fmt.Errorf("resolve queue/object storage config: %w", err) } logger.DebugContext(ctx, "Resolved Queue/ObjectStorage config", "queue", qosCfg.Queue, "objectStorage", qosCfg.OS, "version", qosCfg.Version, "serviceAccount", cr.Spec.ServiceAccount) @@ -244,8 +241,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr err = ingMgr.updateIngestorConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") - logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) - return result, err + return result, fmt.Errorf("update queue/pipeline conf files: %w", err) } eventPublisher.Normal(ctx, "QueueConfigUpdated", diff --git a/pkg/splunk/enterprise/licensemanager.go b/pkg/splunk/enterprise/licensemanager.go index dd580433b..432760bad 100644 --- a/pkg/splunk/enterprise/licensemanager.go +++ b/pkg/splunk/enterprise/licensemanager.go @@ -65,8 +65,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, err = validateLicenseManagerSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateLicenseManagerSpec", fmt.Sprintf("validate license manager spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate license manager spec") - return result, err + return result, fmt.Errorf("validate license manager spec: %w", err) } // If needed, Migrate the app framework status @@ -90,9 +89,8 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkLicenseManager) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // check if deletion has been requested @@ -148,8 +146,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, // Check for license-related pod failures before updating if err = checkLicenseRelatedPodFailures(ctx, client, cr, statefulSet); err != nil { - scopedLog.Error(err, "License check failed") - return result, err + return result, fmt.Errorf("license check: %w", err) } mgr := splctrl.DefaultStatefulSetPodManager{} diff --git a/pkg/splunk/enterprise/licensemaster.go b/pkg/splunk/enterprise/licensemaster.go index 00adcc9e0..11f954334 100644 --- a/pkg/splunk/enterprise/licensemaster.go +++ b/pkg/splunk/enterprise/licensemaster.go @@ -61,8 +61,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, err = validateLicenseMasterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateLicenseMasterSpec", fmt.Sprintf("validate licensemaster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate license master spec") - return result, err + return result, fmt.Errorf("validate license master spec: %w", err) } // If needed, Migrate the app framework status @@ -86,9 +85,8 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkLicenseMaster) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // check if deletion has been requested diff --git a/pkg/splunk/enterprise/monitoringconsole.go b/pkg/splunk/enterprise/monitoringconsole.go index 0bbc54047..0e0d57c64 100644 --- a/pkg/splunk/enterprise/monitoringconsole.go +++ b/pkg/splunk/enterprise/monitoringconsole.go @@ -46,9 +46,6 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyMonitoringConsole") - eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) cr.Kind = "MonitoringConsole" @@ -68,8 +65,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie err = validateMonitoringConsoleSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateMonitoringConsoleSpec", fmt.Sprintf("validate monitoringconsole spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate monitoring console spec") - return result, err + return result, fmt.Errorf("validate monitoring console spec: %w", err) } // If needed, Migrate the app framework status @@ -95,9 +91,8 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie // create or update general config resources _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkMonitoringConsole) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // check if deletion has been requested diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index 713ded930..6b3d0b7d7 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -64,8 +64,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie err = validateSearchHeadClusterSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateSearchHeadClusterSpec", fmt.Sprintf("validate searchHeadCluster spec failed %s", err.Error())) - scopedLog.Error(err, "Failed to validate searchHeadCluster spec") - return result, err + return result, fmt.Errorf("validate search head cluster spec: %w", err) } // If needed, Migrate the app framework status @@ -77,9 +76,8 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkSearchHead) if err != nil { - scopedLog.Error(err, "create or update general config failed", "error", err.Error()) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // If the app framework is configured then do following things - diff --git a/pkg/splunk/enterprise/standalone.go b/pkg/splunk/enterprise/standalone.go index 36bf92a0c..2bf4aa11b 100644 --- a/pkg/splunk/enterprise/standalone.go +++ b/pkg/splunk/enterprise/standalone.go @@ -63,8 +63,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr err = validateStandaloneSpec(ctx, client, cr) if err != nil { eventPublisher.Warning(ctx, "validateStandaloneSpec", fmt.Sprintf("validate standalone spec failed %s", err.Error())) - logger.ErrorContext(ctx, "Failed to validate standalone spec", "error", err) - return result, err + return result, fmt.Errorf("validate standalone spec: %w", err) } // updates status after function completes @@ -109,9 +108,8 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr // create or update general config resources _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkStandalone) if err != nil { - logger.ErrorContext(ctx, "create or update general config failed", "error", err) eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) - return result, err + return result, fmt.Errorf("apply splunk config: %w", err) } // Smart Store secrets get created manually and should not be managed by the Operator From b03f4464a0ab84ce0a88c0e71f805e43aba8a51b Mon Sep 17 00:00:00 2001 From: Igor Grzankowski Date: Mon, 16 Mar 2026 14:03:31 +0100 Subject: [PATCH 08/19] Add logs (#1755) --- docs/LoggingAndEvents.md | 199 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 docs/LoggingAndEvents.md diff --git a/docs/LoggingAndEvents.md b/docs/LoggingAndEvents.md new file mode 100644 index 000000000..22d712f88 --- /dev/null +++ b/docs/LoggingAndEvents.md @@ -0,0 +1,199 @@ +# Logging & Events + +## Logging + +The operator uses Go's `log/slog` package for structured logging. The global logger is configured once at startup in `cmd/main.go` via `logging.SetupLogger()` and set as the default with `slog.SetDefault()`. + +### Getting a Logger + +| Where | How | +|---|---| +| **Controller `Reconcile`** | `logger := slog.Default().With("controller", "Name", "name", req.Name, "namespace", req.Namespace)` | +| **Business logic (pkg)** | `logger := logging.FromContext(ctx).With("func", "FunctionName")` | + +Controllers inject the logger into context so downstream code can retrieve it: + +```go +logger := slog.Default().With("controller", "Standalone", "name", req.Name, "namespace", req.Namespace) +ctx = logging.WithLogger(ctx, logger) +``` + +### Log Levels + +| Level | Use for | +|---|---| +| `Debug` | Verbose diagnostics (state dumps, intermediate values) | +| `Info` | Normal operations (reconcile start, requeue, status changes) | +| `Warn` | Recoverable issues (deprecated config, fallback behavior) | +| `Error` | Failures that need attention (API errors, broken invariants) | + +Configure at runtime via `LOG_LEVEL` env var or `--log-level` flag. Values: `debug`, `info`, `warn`, `error`. + +### Writing Log Messages + +**Message format:** short, lowercase, past tense or present participle. Describe *what happened*, not the function name. + +```go +// Good +logger.InfoContext(ctx, "statefulset updated", "replicas", replicas) +logger.ErrorContext(ctx, "failed to fetch secret", "error", err, "secret", name) + +// Bad - don't restate the function name or use generic messages +logger.InfoContext(ctx, "ApplyStandalone") +logger.InfoContext(ctx, "error occurred") +``` + +**Rules:** + +1. Always use `*Context(ctx, ...)` variants (`InfoContext`, `ErrorContext`, etc.). +2. Always pass `"error", err` as a key-value pair — never as the message string. +3. Use consistent key names across the codebase: + +| Key | Meaning | +|---|---| +| `"error"` | The `error` value | +| `"name"` | CR or resource name | +| `"namespace"` | Kubernetes namespace | +| `"controller"` | Controller name | +| `"func"` | Function name (in pkg layer) | +| `"replicas"` | Replica count | +| `"phase"` | CR phase | + +4. **Don't log and return the same error.** controller-runtime automatically logs every non-nil error returned from `Reconcile()` via `log.Error(err, "Reconciler error")`. If you also log it in the business logic, the same error appears twice. Instead, wrap the error with context using `fmt.Errorf("description: %w", err)` so the single controller-runtime log line is descriptive: + ```go + // Good — wrap and return, no explicit log needed + return result, fmt.Errorf("validate standalone spec: %w", err) + + // Bad — double-logged: once here, once by controller-runtime + scopedLog.Error(err, "Failed to validate standalone spec") + return result, err + ``` +5. Sensitive data (passwords, tokens, secrets) is automatically redacted by the handler. + +### Configuration + +| Env Var | Flag | Default | Description | +|---|---|---|---| +| `LOG_LEVEL` | `--log-level` | `info` | Minimum log level | +| `LOG_FORMAT` | `--log-format` | `json` | `json` or `text` | +| `LOG_ADD_SOURCE` | `--log-add-source` | `false` | Include source file/line (auto-enabled at debug) | + +Flags take precedence over env vars. + +--- + +## Kubernetes Events + +Events are user-facing signals visible via `kubectl describe`. Use them for significant state changes that an operator user should see — not for internal debugging. + +### How It Works + +1. Controllers pass the event recorder into context: + ```go + ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) + ``` +2. Business logic retrieves a publisher: + ```go + eventPublisher := GetEventPublisher(ctx, cr) + ``` +3. Emit events using constants from `pkg/splunk/enterprise/event_reasons.go`: + ```go + eventPublisher.Normal(ctx, EventReasonScaledUp, + fmt.Sprintf("Successfully scaled %s from %d to %d replicas", cr.GetName(), old, new)) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, + fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) + ``` + +### Event Reason Constants + +All event reasons are defined as constants in `pkg/splunk/enterprise/event_reasons.go`. **Never use string literals for event reasons** — always use the `EventReason*` constants. This ensures consistency across controllers and makes reasons searchable. + +**Normal reasons:** + +| Constant | Value | Use for | +|---|---|---| +| `EventReasonScaledUp` | `ScaledUp` | Successful scale up | +| `EventReasonScaledDown` | `ScaledDown` | Successful scale down | +| `EventReasonClusterInitialized` | `ClusterInitialized` | Cluster first becomes ready | +| `EventReasonClusterQuorumRestored` | `ClusterQuorumRestored` | Quorum recovered | +| `EventReasonPasswordSyncCompleted` | `PasswordSyncCompleted` | Secret sync finished | + +**Warning reasons (common):** + +| Constant | Value | Use for | +|---|---|---| +| `EventReasonValidateSpecFailed` | `ValidateSpecFailed` | CR spec validation errors | +| `EventReasonApplySplunkConfigFailed` | `ApplySplunkConfigFailed` | General config apply failures | +| `EventReasonAppFrameworkInitFailed` | `AppFrameworkInitFailed` | App framework init errors | +| `EventReasonApplyServiceFailed` | `ApplyServiceFailed` | Service create/update failures | +| `EventReasonStatefulSetFailed` | `StatefulSetFailed` | StatefulSet get failures | +| `EventReasonStatefulSetUpdateFailed` | `StatefulSetUpdateFailed` | StatefulSet update failures | +| `EventReasonDeleteFailed` | `DeleteFailed` | CR deletion failures | +| `EventReasonSecretMissing` | `SecretMissing` | Required secret not found | +| `EventReasonUpgradeCheckFailed` | `UpgradeCheckFailed` | Upgrade path validation errors | + +See `event_reasons.go` for the full list. + +To add a new event reason: add a constant to `event_reasons.go`, then use it in the code. + +### When to Use Events vs Logs + +| Scenario | Log | Event | +|---|---|---| +| Reconcile start/requeue | Yes | No | +| API call failed (transient) | Yes | No | +| Spec validation failed | Yes | Yes (Warning) | +| Successful scale up/down | Yes | Yes (Normal) | +| Phase transition | Yes | Yes (Normal) | +| Security-related failure | Yes | Yes (Warning) | + +### Writing Event Messages + +**Reason:** always use an `EventReason*` constant — e.g. `EventReasonScaledUp`, `EventReasonValidateSpecFailed`. + +**Message:** sentence case, user-friendly, include the resource name. **Never include raw `err.Error()` in events** — error details belong in logs only. Events are visible to any user with `kubectl describe` access and may leak internal paths, secret names, or stack traces. Instead, write a user-actionable summary and point to logs for details. + +```go +// Good — uses constant, event gives a user-actionable summary, log has full error +logger.ErrorContext(ctx, "smartstore volume key validation failed", "error", err) +eventPublisher.Warning(ctx, EventReasonRemoteVolumeKeyCheckFailed, + fmt.Sprintf("Remote volume key change check failed for %s — check operator logs", cr.GetName())) + +eventPublisher.Normal(ctx, EventReasonScaledUp, + fmt.Sprintf("Successfully scaled %s from %d to %d replicas", cr.GetName(), oldCount, newCount)) + +// Bad — string literal instead of constant +eventPublisher.Warning(ctx, "ValidationFailed", "...") + +// Bad — leaking err.Error() into events +eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, + fmt.Sprintf("Invalid smartstore config: %s", err.Error())) + +// Bad — too vague, no context +eventPublisher.Warning(ctx, "Error", "something went wrong") +``` + +**Log + Event combo for errors:** + +```go +err = validateStandaloneSpec(ctx, client, cr) +if err != nil { + // Log: full error for operator developers / debugging + logger.ErrorContext(ctx, "spec validation failed", "error", err) + + // Event: user-actionable summary visible in kubectl describe + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, + fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) + + return result, err +} +``` + +**Rules:** + +1. Always use `EventReason*` constants — never string literals for event reasons. +2. Use `Normal` for successful operations the user should know about. +3. Use `Warning` for failures or degraded states that may need user intervention. +4. Always include the resource name in the message. +5. Never pass `err.Error()` into event messages — log the full error, keep events clean. +6. Don't emit events for routine internal operations — keep the event stream actionable. From 81f78103bdc757968849640c69418ae00708ddaf Mon Sep 17 00:00:00 2001 From: Igor Grzankowski Date: Mon, 16 Mar 2026 14:18:51 +0100 Subject: [PATCH 09/19] Refine events (#1751) --- .../controller/ingestorcluster_controller.go | 11 +-- internal/controller/telemetry_controller.go | 29 ++++--- pkg/splunk/enterprise/clustermanager.go | 24 +++--- pkg/splunk/enterprise/clustermaster.go | 20 ++--- pkg/splunk/enterprise/event_reasons.go | 72 +++++++++++++++++ pkg/splunk/enterprise/indexercluster.go | 80 +++++++++---------- pkg/splunk/enterprise/ingestorcluster.go | 24 +++--- pkg/splunk/enterprise/licensemanager.go | 10 +-- pkg/splunk/enterprise/licensemaster.go | 8 +- pkg/splunk/enterprise/monitoringconsole.go | 20 ++--- pkg/splunk/enterprise/searchheadcluster.go | 16 ++-- .../enterprise/searchheadclusterpodmanager.go | 4 +- pkg/splunk/enterprise/standalone.go | 28 +++---- pkg/splunk/enterprise/upgrade.go | 20 ++--- pkg/splunk/enterprise/util.go | 10 +-- 15 files changed, 224 insertions(+), 152 deletions(-) create mode 100644 pkg/splunk/enterprise/event_reasons.go diff --git a/internal/controller/ingestorcluster_controller.go b/internal/controller/ingestorcluster_controller.go index 2725d32a6..7a553c943 100644 --- a/internal/controller/ingestorcluster_controller.go +++ b/internal/controller/ingestorcluster_controller.go @@ -16,6 +16,7 @@ package controller import ( "context" + "log/slog" "time" appsv1 "k8s.io/api/apps/v1" @@ -28,13 +29,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/pkg/errors" enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/splunk/splunk-operator/internal/controller/common" + "github.com/splunk/splunk-operator/pkg/logging" metrics "github.com/splunk/splunk-operator/pkg/splunk/client/metrics" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" enterprise "github.com/splunk/splunk-operator/pkg/splunk/enterprise" @@ -68,8 +69,8 @@ func (r *IngestorClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "IngestorCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "IngestorCluster") - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("ingestorcluster", req.NamespacedName) + logger := slog.Default().With("controller", "IngestorCluster", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) // Fetch the IngestorCluster instance := &enterpriseApi.IngestorCluster{} @@ -94,14 +95,14 @@ func (r *IngestorClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - reqLogger.Info("start", "CR version", instance.GetResourceVersion()) + logger.InfoContext(ctx, "start", "CR version", instance.GetResourceVersion()) // Pass event recorder through context ctx = context.WithValue(ctx, splcommon.EventRecorderKey, r.Recorder) result, err := ApplyIngestorCluster(ctx, r.Client, instance) if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err diff --git a/internal/controller/telemetry_controller.go b/internal/controller/telemetry_controller.go index 7214f67cc..22e9fd43e 100644 --- a/internal/controller/telemetry_controller.go +++ b/internal/controller/telemetry_controller.go @@ -18,19 +18,18 @@ package controller import ( "context" "fmt" - enterprise "github.com/splunk/splunk-operator/pkg/splunk/enterprise" - ctrl "sigs.k8s.io/controller-runtime" + "log/slog" "time" + "github.com/splunk/splunk-operator/pkg/logging" metrics "github.com/splunk/splunk-operator/pkg/splunk/client/metrics" - + enterprise "github.com/splunk/splunk-operator/pkg/splunk/enterprise" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" - + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" ) @@ -55,14 +54,14 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "Telemetry")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "Telemetry") - reqLogger := log.FromContext(ctx) - reqLogger = reqLogger.WithValues("telemetry", req.NamespacedName) + logger := slog.Default().With("controller", "Telemetry", "name", req.Name, "namespace", req.Namespace) + ctx = logging.WithLogger(ctx, logger) - reqLogger.Info("Reconciling telemetry") + logger.InfoContext(ctx, "Reconciling telemetry") defer func() { if rec := recover(); rec != nil { - reqLogger.Error(fmt.Errorf("panic: %v", rec), "Recovered from panic in TelemetryReconciler.Reconcile") + logger.ErrorContext(ctx, "Recovered from panic in TelemetryReconciler.Reconcile", "error", fmt.Errorf("panic: %v", rec)) } }() @@ -71,27 +70,27 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err := r.Get(ctx, req.NamespacedName, cm) if err != nil { if k8serrors.IsNotFound(err) { - reqLogger.Info("telemetry configmap not found; requeueing", "period(seconds)", int(telemetryRetryDelay/time.Second)) + logger.InfoContext(ctx, "telemetry configmap not found; requeueing", "period(seconds)", int(telemetryRetryDelay/time.Second)) return ctrl.Result{Requeue: true, RequeueAfter: telemetryRetryDelay}, nil } - reqLogger.Error(err, "could not load telemetry configmap; requeueing", "period(seconds)", int(telemetryRetryDelay/time.Second)) + logger.ErrorContext(ctx, "could not load telemetry configmap; requeueing", "error", err, "period(seconds)", int(telemetryRetryDelay/time.Second)) return ctrl.Result{Requeue: true, RequeueAfter: telemetryRetryDelay}, nil } if len(cm.Data) == 0 { - reqLogger.Info("telemetry configmap has no data keys") + logger.InfoContext(ctx, "telemetry configmap has no data keys") return ctrl.Result{Requeue: true, RequeueAfter: telemetryRetryDelay}, nil } - reqLogger.Info("start", "Telemetry configmap version", cm.GetResourceVersion()) + logger.InfoContext(ctx, "start", "Telemetry configmap version", cm.GetResourceVersion()) result, err := applyTelemetryFn(ctx, r.Client, cm) if err != nil { - reqLogger.Error(err, "Failed to send telemetry") + logger.ErrorContext(ctx, "Failed to send telemetry", "error", err) return ctrl.Result{Requeue: true, RequeueAfter: telemetryRetryDelay}, nil } if result.Requeue && result.RequeueAfter != 0 { - reqLogger.Info("Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) + logger.InfoContext(ctx, "Requeued", "period(seconds)", int(result.RequeueAfter/time.Second)) } return result, err diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index bab4440a9..8a5245d0f 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -66,7 +66,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err = validateClusterManagerSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateClusterManagerSpec", fmt.Sprintf("validate clustermanager spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate clustermanager spec") return result, err } @@ -78,7 +78,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, AreRemoteVolumeKeysChanged(ctx, client, cr, SplunkClusterManager, &cr.Spec.SmartStore, cr.Status.ResourceRevMap, &err) { if err != nil { - eventPublisher.Warning(ctx, "AreRemoteVolumeKeysChanged", fmt.Sprintf("check remote volume key change failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonRemoteVolumeKeyCheckFailed, fmt.Sprintf("Remote volume key change check failed for %s — check operator logs", cr.GetName())) return result, err } @@ -113,7 +113,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -123,7 +123,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -168,7 +168,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s — check operator logs", cr.GetName())) } return result, err } @@ -331,7 +331,7 @@ func CheckIfsmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.Con stdOut, stdErr, err := podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if err != nil || stdErr != "" { - eventPublisher.Warning(ctx, "PodExecCommand", fmt.Sprintf("Failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err)) + eventPublisher.Warning(ctx, EventReasonPodExecFailed, fmt.Sprintf("Failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err)) return fmt.Errorf("failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err) } @@ -342,12 +342,12 @@ func CheckIfsmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.Con scopedLog.Info("Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) return nil } - eventPublisher.Warning(ctx, "getSmartstoreConfigMap", fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) + eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) return fmt.Errorf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap) } // Somehow the configmap was deleted, ideally this should not happen - eventPublisher.Warning(ctx, "getSmartstoreConfigMap", "smartstore ConfigMap is missing") + eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, "smartstore ConfigMap is missing") return fmt.Errorf("smartstore ConfigMap is missing") } @@ -414,7 +414,7 @@ func PushManagerAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), defaultSecretObjName) if err != nil { - eventPublisher.Warning(ctx, "PushManagerAppsBundle", fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonBundlePushFailed, fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) return fmt.Errorf("could not access default secret object to fetch admin password. Reason %v", err) } @@ -483,7 +483,7 @@ var GetCMMultisiteEnvVarsCall = func(ctx context.Context, cr *enterpriseApi.Clus // on update, and returns error if something is wrong func changeClusterManagerAnnotations(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.LicenseManager) error { reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("changeClusterManagerAnnotations").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := reqLogger.WithName(EventReasonAnnotationUpdateFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -534,13 +534,13 @@ func changeClusterManagerAnnotations(ctx context.Context, c splcommon.Controller image, err := getCurrentImage(ctx, c, cr, SplunkLicenseManager) if err != nil { - eventPublisher.Warning(ctx, "changeClusterManagerAnnotations", fmt.Sprintf("Could not get the LicenseManager Image. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not get the LicenseManager Image. Reason %v", err)) scopedLog.Error(err, "Get LicenseManager Image failed with", "error", err) return err } err = changeAnnotations(ctx, c, image, clusterManagerInstance) if err != nil { - eventPublisher.Warning(ctx, "changeClusterManagerAnnotations", fmt.Sprintf("Could not update annotations. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not update annotations. Reason %v", err)) scopedLog.Error(err, "ClusterManager types update after changing annotations failed with", "error", err) return err } diff --git a/pkg/splunk/enterprise/clustermaster.go b/pkg/splunk/enterprise/clustermaster.go index 88e2c3815..96f1232f0 100644 --- a/pkg/splunk/enterprise/clustermaster.go +++ b/pkg/splunk/enterprise/clustermaster.go @@ -65,7 +65,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err = validateClusterMasterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateClusterMasterSpec", fmt.Sprintf("validate clustermaster spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate clustermaster spec") return result, err } @@ -77,7 +77,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, AreRemoteVolumeKeysChanged(ctx, client, cr, SplunkClusterMaster, &cr.Spec.SmartStore, cr.Status.ResourceRevMap, &err) { if err != nil { - eventPublisher.Warning(ctx, "AreRemoteVolumeKeysChanged", fmt.Sprintf("check remote volume key change failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonRemoteVolumeKeyCheckFailed, fmt.Sprintf("Remote volume key change check failed for %s — check operator logs", cr.GetName())) return result, err } @@ -112,7 +112,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -122,7 +122,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -160,7 +160,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s — check operator logs", cr.GetName())) } return result, err } @@ -313,7 +313,7 @@ func CheckIfMastersmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcomm stdOut, stdErr, err := podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if err != nil || stdErr != "" { - eventPublisher.Warning(ctx, "PodExecCommand", fmt.Sprintf("Failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err)) + eventPublisher.Warning(ctx, EventReasonPodExecFailed, fmt.Sprintf("Failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err)) return fmt.Errorf("failed to check config token value on pod. stdout=%s, stderror=%s, error=%v", stdOut, stdErr, err) } @@ -324,12 +324,12 @@ func CheckIfMastersmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcomm scopedLog.Info("Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) return nil } - eventPublisher.Warning(ctx, "getSmartstoreConfigMap", fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) + eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) return fmt.Errorf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap) } // Somehow the configmap was deleted, ideally this should not happen - eventPublisher.Warning(ctx, "getSmartstoreConfigMap", "smartstore ConfigMap is missing") + eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, "smartstore ConfigMap is missing") return fmt.Errorf("smartstore ConfigMap is missing") } @@ -393,14 +393,14 @@ func PushMasterAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) defaultSecret, err := splutil.GetSecretByName(ctx, c, cr.GetNamespace(), defaultSecretObjName) if err != nil { - eventPublisher.Warning(ctx, "PushMasterAppsBundle", fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonBundlePushFailed, fmt.Sprintf("Could not access default secret object to fetch admin password. Reason %v", err)) return fmt.Errorf("could not access default secret object to fetch admin password. Reason %v", err) } //Get the admin password from the secret object adminPwd, foundSecret := defaultSecret.Data["password"] if !foundSecret { - eventPublisher.Warning(ctx, "PushMasterAppsBundle", "Could not find admin password while trying to push the manager apps bundle") + eventPublisher.Warning(ctx, EventReasonBundlePushFailed, "Could not find admin password while trying to push the manager apps bundle") return fmt.Errorf("could not find admin password while trying to push the manager apps bundle") } diff --git a/pkg/splunk/enterprise/event_reasons.go b/pkg/splunk/enterprise/event_reasons.go new file mode 100644 index 000000000..f8beea0c8 --- /dev/null +++ b/pkg/splunk/enterprise/event_reasons.go @@ -0,0 +1,72 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +const ( + // Normal event reasons + EventReasonScaledUp = "ScaledUp" + EventReasonScaledDown = "ScaledDown" + EventReasonClusterInitialized = "ClusterInitialized" + EventReasonClusterQuorumLost = "ClusterQuorumLost" + EventReasonClusterQuorumRestored = "ClusterQuorumRestored" + EventReasonPasswordSyncCompleted = "PasswordSyncCompleted" + + // Warning event reasons — spec & config + EventReasonValidateSpecFailed = "ValidateSpecFailed" + EventReasonApplySplunkConfigFailed = "ApplySplunkConfigFailed" + EventReasonAppFrameworkInitFailed = "AppFrameworkInitFailed" + EventReasonAppRepoConnFailed = "AppRepositoryConnectionFailed" + EventReasonSmartStoreConfigPending = "SmartStoreConfigPending" + + // Warning event reasons — services & statefulsets + EventReasonApplyServiceFailed = "ApplyServiceFailed" + EventReasonStatefulSetFailed = "StatefulSetFailed" + EventReasonStatefulSetUpdateFailed = "StatefulSetUpdateFailed" + EventReasonStatefulSetDeleteFailed = "StatefulSetDeleteFailed" + EventReasonOwnerRefFailed = "OwnerRefFailed" + + // Warning event reasons — deletion + EventReasonDeleteFailed = "DeleteFailed" + + // Warning event reasons — secrets & credentials + EventReasonSecretMissing = "SecretMissing" + EventReasonSecretInvalid = "SecretInvalid" + EventReasonSecretAccessFailed = "SecretAccessFailed" + EventReasonPasswordSyncFailed = "PasswordSyncFailed" + + // Warning event reasons — monitoring console + EventReasonMonitoringConsoleConfigFailed = "MonitoringConsoleConfigFailed" + EventReasonMonitoringConsoleRefFailed = "MonitoringConsoleRefFailed" + EventReasonMonitoringConsoleCleanupFailed = "MonitoringConsoleCleanupFailed" + EventReasonMonitoringConsoleApplyFailed = "MonitoringConsoleApplyFailed" + EventReasonAnnotationUpdateFailed = "AnnotationUpdateFailed" + EventReasonImageGetFailed = "ImageGetFailed" + + // Warning event reasons — cluster operations + EventReasonRemoteVolumeKeyCheckFailed = "RemoteVolumeKeyCheckFailed" + EventReasonVerifyRFPeersFailed = "VerifyRFPeersFailed" + EventReasonMaintenanceModeFailed = "MaintenanceModeFailed" + EventReasonRetrieveCMSpecFailed = "RetrieveCMSpecFailed" + EventReasonConfFileUpdateFailed = "ConfFileUpdateFailed" + EventReasonBundlePushFailed = "BundlePushFailed" + EventReasonPodExecFailed = "PodExecFailed" + EventReasonScalingBlockedRF = "ScalingBlockedRF" + EventReasonLicenseExpired = "LicenseExpired" + + // Warning event reasons — upgrade + EventReasonUpgradeCheckFailed = "UpgradeCheckFailed" + EventReasonUpgradeBlockedVersionMismatch = "UpgradeBlockedVersionMismatch" +) diff --git a/pkg/splunk/enterprise/indexercluster.go b/pkg/splunk/enterprise/indexercluster.go index 3af847603..57a9e8746 100644 --- a/pkg/splunk/enterprise/indexercluster.go +++ b/pkg/splunk/enterprise/indexercluster.go @@ -69,7 +69,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // validate and updates defaults for CR err = validateIndexerClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "IndexerClusterSpecValidationFailed", "Validation of Indexer Cluster spec failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to validate Indexer Cluster spec", "error", err.Error()) return result, err } @@ -97,7 +97,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { logger.ErrorContext(ctx, "Create or update of general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update of general config failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s \u2014 check operator logs", cr.GetName())) return result, err } @@ -125,7 +125,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if mgr.cr.Status.ClusterManagerPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verification of RF peer failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonVerifyRFPeersFailed, fmt.Sprintf("RF peer verification failed for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Verification of RF peer failed", "error", err.Error()) return result, err } @@ -143,7 +143,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "DeletionFailed", "Deletion of custom resource failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Deletion of custom resource failed", "error", err.Error()) } return result, err @@ -151,7 +151,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply headless service for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Create or update of headless service for Indexer Cluster failed", "error", err.Error()) return result, err } @@ -159,7 +159,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply service for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Create or update of service for Indexer Cluster failed", "error", err.Error()) return result, err } @@ -167,7 +167,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetFailed, fmt.Sprintf("Failed to get indexer statefulset for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -227,7 +227,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } @@ -235,7 +235,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetDeleteFailed, fmt.Sprintf("Version mismatch detected, failed to delete statefulset for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } @@ -244,7 +244,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } @@ -271,7 +271,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonConfFileUpdateFailed, fmt.Sprintf("Failed to update Queue/Pipeline config for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) return result, err } @@ -303,7 +303,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieval of Cluster Manager spec failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonRetrieveCMSpecFailed, fmt.Sprintf("Failed to retrieve cluster manager spec for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Retrieval of Cluster Manager spec failed", "error", err.Error()) return result, err } @@ -315,7 +315,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Get Monitoring Console client failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleApplyFailed, fmt.Sprintf("Failed to apply monitoring console changes for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "get Monitoring Console client failed", "error", err.Error()) return result, err } @@ -336,7 +336,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "ClusterMaintenanceModeFailed", "Set Cluster maintenance mode failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMaintenanceModeFailed, fmt.Sprintf("Failed to disable cluster maintenance mode for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Set Cluster maintenance mode failed", "error", err.Error()) return result, err } @@ -355,7 +355,7 @@ func ApplyIndexerClusterManager(ctx context.Context, client splcommon.Controller } err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonOwnerRefFailed, fmt.Sprintf("Failed to set statefulset owner reference for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err @@ -416,8 +416,8 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIndexer) if err != nil { + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Create or update general config failed. Check operator logs for details.") return result, err } @@ -444,7 +444,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if mgr.cr.Status.ClusterMasterPhase == enterpriseApi.PhaseReady { err = VerifyRFPeers(ctx, mgr, client) if err != nil { - eventPublisher.Warning(ctx, "VerifyRFPeersFailed", "Verify RF peer failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonVerifyRFPeersFailed, fmt.Sprintf("RF peer verification failed for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Verify RF peer failed", "error", err.Error()) return result, err } @@ -462,7 +462,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "DeleteFailed", "Delete custom resource failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Delete custom resource failed", "error", err.Error()) } return result, err @@ -471,7 +471,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update a headless service for indexer cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of headless service for Indexer Cluster failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply headless service for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Create or update of headless service failed", "error", err.Error()) return result, err } @@ -479,7 +479,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update a regular service for indexer cluster (ingestion) err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIndexer, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Create or update of service for Indexer Cluster failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply service for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Create or update of service failed", "error", err.Error()) return result, err } @@ -487,7 +487,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // create or update statefulset for the indexers statefulSet, err := getIndexerStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "GetIndexerStatefulSetFailed", "Get Indexer stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetFailed, fmt.Sprintf("Failed to get indexer statefulset for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Get Indexer stateful set failed", "error", err.Error()) return result, err } @@ -547,7 +547,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, if !versionUpgrade { phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } @@ -555,7 +555,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // Delete the statefulset and recreate new one err = client.Delete(ctx, statefulSet) if err != nil { - eventPublisher.Warning(ctx, "DeleteFailed", "Delete of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetDeleteFailed, fmt.Sprintf("Version mismatch detected, failed to delete statefulset for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Delete of stateful set failed", "error", err.Error()) return result, err } @@ -564,7 +564,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, statefulSet.ResourceVersion = "" phase, err = mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) if err != nil { - eventPublisher.Warning(ctx, "UpdateFailed", "Update of stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Update of stateful set failed", "error", err.Error()) return result, err } @@ -590,7 +590,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, mgr := newIndexerClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = mgr.updateIndexerConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "UpdateIndexerConfFileFailed", "Failed to update conf file for Queue/Pipeline config change after pod creation. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonConfFileUpdateFailed, fmt.Sprintf("Failed to update Queue/Pipeline config for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config change after pod creation", "error", err.Error()) return result, err } @@ -622,7 +622,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, //Retrieve monitoring console ref from CM Spec cmMonitoringConsoleConfigRef, err := RetrieveCMSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "RetrieveCMSpecFailed", "Retrieve Cluster Master spec failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonRetrieveCMSpecFailed, fmt.Sprintf("Failed to retrieve cluster master spec for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Retrieve Cluster Master spec failed", "error", err.Error()) return result, err } @@ -634,7 +634,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, c := mgr.getMonitoringConsoleClient(cr, cmMonitoringConsoleConfigRef) err := c.AutomateMCApplyChanges() if err != nil { - eventPublisher.Warning(ctx, "AutomateMCApplyChangesFailed", "Automate MC Apply Changes failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleApplyFailed, fmt.Sprintf("Failed to apply monitoring console changes for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Automate MC Apply Changes failed", "error", err.Error()) return result, err } @@ -655,7 +655,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, // Disable maintenance mode err = SetClusterMaintenanceMode(ctx, client, cr, false, cmPodName, podExecClient) if err != nil { - eventPublisher.Warning(ctx, "SetClusterMaintenanceModeFailed", "Set Cluster Master maintenance mode failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMaintenanceModeFailed, fmt.Sprintf("Failed to disable cluster maintenance mode for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Set Cluster Master maintenance mode failed", "error", err.Error()) return result, err } @@ -672,7 +672,7 @@ func ApplyIndexerCluster(ctx context.Context, client splcommon.ControllerClient, namespacedName = types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkClusterMaster, cr.Spec.ClusterMasterRef.Name)} err = splctrl.SetStatefulSetOwnerRef(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "SetStatefulSetOwnerRefFailed", "Set stateful set owner reference failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonOwnerRefFailed, fmt.Sprintf("Failed to set statefulset owner reference for %s \u2014 check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Set stateful set owner reference failed", "error", err.Error()) result.Requeue = true return result, err @@ -853,8 +853,8 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { // Emit event for password sync failure if eventPublisher != nil { - eventPublisher.Warning(ctx, "PasswordSyncFailed", - fmt.Sprintf("Password sync failed for pod '%s'. Check operator logs for details.", indexerPodName)) + eventPublisher.Warning(ctx, EventReasonPasswordSyncFailed, + fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) } mgr.log.ErrorContext(ctx, "Configuration push failed", "failedPeer", indexerPodName, "error", err.Error()) return err @@ -868,8 +868,8 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica if err != nil { // Emit event for password sync failure if eventPublisher != nil { - eventPublisher.Warning(ctx, "PasswordSyncFailed", - fmt.Sprintf("Password sync failed for pod '%s'. Check operator logs for details.", indexerPodName)) + eventPublisher.Warning(ctx, EventReasonPasswordSyncFailed, + fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", indexerPodName, err.Error())) } logger.ErrorContext(ctx, "Configuration push failed during restart", "failedPeer", indexerPodName, "error", err.Error()) return err @@ -927,7 +927,7 @@ func ApplyIdxcSecret(ctx context.Context, mgr *indexerClusterPodManager, replica // Emit event for password sync completed if eventPublisher != nil { - eventPublisher.Normal(ctx, "PasswordSyncCompleted", + eventPublisher.Normal(ctx, EventReasonPasswordSyncCompleted, fmt.Sprintf("Password synchronized for %d pods", howManyPodsHaveSecretChanged)) } @@ -990,12 +990,12 @@ func (mgr *indexerClusterPodManager) Update(ctx context.Context, c splcommon.Con if mgr.cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { if desiredReplicas > previousReadyReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", + eventPublisher.Normal(ctx, EventReasonScaledUp, fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) } } else if desiredReplicas < previousReadyReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", + eventPublisher.Normal(ctx, EventReasonScaledDown, fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReadyReplicas, desiredReplicas)) } } @@ -1173,7 +1173,7 @@ func (mgr *indexerClusterPodManager) verifyRFPeers(ctx context.Context, c splcom mgr.log.InfoContext(ctx, "Changing number of replicas as it is less than RF number of peers", "replicas", requestedReplicas) // Emit event indicating scaling below RF is blocked/adjusted if eventPublisher != nil { - eventPublisher.Warning(ctx, "ScalingBlockedRF", + eventPublisher.Warning(ctx, EventReasonScalingBlockedRF, fmt.Sprintf("Cannot scale below replication factor: %d replicas required, %d requested. Adjust replicationFactor or replicas.", replicationFactor, requestedReplicas)) } mgr.cr.Spec.Replicas = replicationFactor @@ -1269,18 +1269,18 @@ func (mgr *indexerClusterPodManager) updateStatus(ctx context.Context, statefulS // Cluster just finished initializing when quorum becomes ready if !oldIndexingReady && mgr.cr.Status.IndexingReady { if !oldInitialized && mgr.cr.Status.Initialized { - eventPublisher.Normal(ctx, "ClusterInitialized", + eventPublisher.Normal(ctx, EventReasonClusterInitialized, fmt.Sprintf("Cluster '%s' initialized with %d peers", mgr.cr.GetName(), totalPeers)) } // Cluster quorum just restored - eventPublisher.Normal(ctx, "ClusterQuorumRestored", + eventPublisher.Normal(ctx, EventReasonClusterQuorumRestored, fmt.Sprintf("Cluster quorum restored: %d/%d peers available", available, totalPeers)) } // Cluster quorum lost (transition out of indexing ready) if oldIndexingReady && !mgr.cr.Status.IndexingReady { - eventPublisher.Warning(ctx, "ClusterQuorumLost", + eventPublisher.Warning(ctx, EventReasonClusterQuorumLost, fmt.Sprintf("Cluster quorum lost: %d/%d peers available. Investigate peer failures immediately.", available, totalPeers)) } } diff --git a/pkg/splunk/enterprise/ingestorcluster.go b/pkg/splunk/enterprise/ingestorcluster.go index 5a220aabd..69eed7fe5 100644 --- a/pkg/splunk/enterprise/ingestorcluster.go +++ b/pkg/splunk/enterprise/ingestorcluster.go @@ -59,7 +59,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Validate and updates defaults for CR err = validateIngestorClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "ValidateIngestorClusterSpecFailed", "Validate Ingestor Cluster spec failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to validate Ingestor Cluster spec", "error", err.Error()) return result, err } @@ -91,7 +91,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err = initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "AppInfoStatusInitializationFailed", "Init and check app info status failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -102,7 +102,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update general config resources namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkIngestor) if err != nil { - eventPublisher.Warning(ctx, "ApplySplunkConfigFailed", "Apply of general config failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "create or update general config failed", "error", err.Error()) return result, err } @@ -112,7 +112,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, make([]corev1.EnvVar, 0), false) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMapFailed", "Apply of monitoring console config map failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleConfigFailed, fmt.Sprintf("Failed to update monitoring console config map for %s — check operator logs", cr.GetName())) return result, err } } @@ -141,14 +141,14 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update a headless service for ingestor cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIngestor, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Apply of headless service failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply headless service for %s — check operator logs", cr.GetName())) return result, err } // Create or update a regular service for ingestor cluster err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkIngestor, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyServiceFailed", "Apply of service failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply service for %s — check operator logs", cr.GetName())) return result, err } @@ -190,14 +190,14 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr // Create or update statefulset for the ingestors statefulSet, err := getIngestorStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "GetIngestorStatefulSetFailed", "Get stateful set failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetFailed, fmt.Sprintf("Failed to get ingestor statefulset for %s — check operator logs", cr.GetName())) return result, err } // Make changes to respective mc configmap when changing/removing mcRef from spec err = validateMonitoringConsoleRef(ctx, client, statefulSet, make([]corev1.EnvVar, 0)) if err != nil { - eventPublisher.Warning(ctx, "MonitoringConsoleRefValidationFailed", "Monitoring console reference validation failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleRefFailed, fmt.Sprintf("Monitoring console reference validation failed for %s — check operator logs", cr.GetName())) return result, err } @@ -205,7 +205,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr phase, err := mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas if err != nil { - eventPublisher.Warning(ctx, "UpdateStatefulSetFailed", "Stateful set update failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s — check operator logs", cr.GetName())) return result, err } cr.Status.Phase = phase @@ -243,7 +243,7 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr ingMgr := newIngestorClusterPodManager(logger, cr, namespaceScopedSecret, splclient.NewSplunkClient, client) err = ingMgr.updateIngestorConfFiles(ctx, cr, &qosCfg.Queue, &qosCfg.OS, qosCfg.AccessKey, qosCfg.SecretKey, client) if err != nil { - eventPublisher.Warning(ctx, "UpdateConfFilesFailed", "Failed to update conf file for Queue/Pipeline config. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonConfFileUpdateFailed, fmt.Sprintf("Failed to update Queue/Pipeline config for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to update conf file for Queue/Pipeline config", "error", err.Error()) return result, err } @@ -274,13 +274,13 @@ func ApplyIngestorCluster(ctx context.Context, client client.Client, cr *enterpr namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "MCReferencesDeletionFailed", "Delete of reference to automated MC failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleCleanupFailed, fmt.Sprintf("Failed to clean up automated monitoring console for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Delete of reference to automated MC failed", "error", err.Error()) } if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, make([]corev1.EnvVar, 0), true) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMapFailed", "Apply of monitoring console environment config map failed. Check operator logs for details.") + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleConfigFailed, fmt.Sprintf("Failed to apply monitoring console config map for %s — check operator logs", cr.GetName())) return result, err } } diff --git a/pkg/splunk/enterprise/licensemanager.go b/pkg/splunk/enterprise/licensemanager.go index dd580433b..c4dc91609 100644 --- a/pkg/splunk/enterprise/licensemanager.go +++ b/pkg/splunk/enterprise/licensemanager.go @@ -64,7 +64,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err = validateLicenseManagerSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateLicenseManagerSpec", fmt.Sprintf("validate license manager spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate license manager spec") return result, err } @@ -81,7 +81,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -91,7 +91,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkLicenseManager) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -123,7 +123,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s — check operator logs", cr.GetName())) } return result, err } @@ -286,7 +286,7 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro // Check for expired licenses for licenseName, licenseInfo := range licenses { if licenseInfo.Status == "EXPIRED" { - eventPublisher.Warning(ctx, "LicenseExpired", + eventPublisher.Warning(ctx, EventReasonLicenseExpired, fmt.Sprintf("License '%s' has expired", licenseName)) scopedLog.Error(nil, "Detected expired license", "licenseName", licenseName, "title", licenseInfo.Title) } diff --git a/pkg/splunk/enterprise/licensemaster.go b/pkg/splunk/enterprise/licensemaster.go index 00adcc9e0..d3611116b 100644 --- a/pkg/splunk/enterprise/licensemaster.go +++ b/pkg/splunk/enterprise/licensemaster.go @@ -60,7 +60,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, // validate and updates defaults for CR err = validateLicenseMasterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateLicenseMasterSpec", fmt.Sprintf("validate licensemaster spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate license master spec") return result, err } @@ -77,7 +77,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -87,7 +87,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkLicenseMaster) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -119,7 +119,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s — check operator logs", cr.GetName())) } return result, err } diff --git a/pkg/splunk/enterprise/monitoringconsole.go b/pkg/splunk/enterprise/monitoringconsole.go index 0bbc54047..1710b90cb 100644 --- a/pkg/splunk/enterprise/monitoringconsole.go +++ b/pkg/splunk/enterprise/monitoringconsole.go @@ -67,7 +67,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie // validate and updates defaults for CR err = validateMonitoringConsoleSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateMonitoringConsoleSpec", fmt.Sprintf("validate monitoringconsole spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate monitoring console spec") return result, err } @@ -84,7 +84,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -96,7 +96,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkMonitoringConsole) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -124,21 +124,21 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie // create or update a headless service err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkMonitoringConsole, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create or update headless service failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply headless service for %s — check operator logs", cr.GetName())) return result, err } // create or update a regular service err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkMonitoringConsole, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create or update regular service failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply regular service for %s — check operator logs", cr.GetName())) return result, err } // create or update statefulset statefulSet, err := getMonitoringConsoleStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getMonitoringConsoleStatefulSet", fmt.Sprintf("get monitoring console stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonStatefulSetFailed, fmt.Sprintf("Failed to get monitoring console statefulset for %s — check operator logs", cr.GetName())) return result, err } @@ -154,7 +154,7 @@ func ApplyMonitoringConsole(ctx context.Context, client splcommon.ControllerClie mgr := splctrl.DefaultStatefulSetPodManager{} phase, err := mgr.Update(ctx, client, statefulSet, 1) if err != nil { - eventPublisher.Warning(ctx, "getMonitoringConsoleStatefulSet", fmt.Sprintf("update to default statefuleset pod manager failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s — check operator logs", cr.GetName())) return result, err } cr.Status.Phase = phase @@ -376,7 +376,7 @@ func DeleteURLsConfigMap(revised *corev1.ConfigMap, crName string, newURLs []cor // on update, and returns error if something is wrong. func changeMonitoringConsoleAnnotations(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) error { reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("changeMonitoringConsoleAnnotations").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := reqLogger.WithName(EventReasonAnnotationUpdateFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -427,13 +427,13 @@ func changeMonitoringConsoleAnnotations(ctx context.Context, client splcommon.Co image, err := getCurrentImage(ctx, client, cr, SplunkClusterManager) if err != nil { - eventPublisher.Warning(ctx, "changeMonitoringConsoleAnnotations", fmt.Sprintf("Could not get the ClusterManager Image. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not get the ClusterManager Image. Reason %v", err)) scopedLog.Error(err, "Get ClusterManager Image failed with", "error", err) return err } err = changeAnnotations(ctx, client, image, monitoringConsoleInstance) if err != nil { - eventPublisher.Warning(ctx, "changeMonitoringConsoleAnnotations", fmt.Sprintf("Could not update annotations. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not update annotations. Reason %v", err)) scopedLog.Error(err, "MonitoringConsole types update after changing annotations failed with", "error", err) return err } diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index 713ded930..6ac1fc1e8 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -63,7 +63,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie // validate and updates defaults for CR err = validateSearchHeadClusterSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateSearchHeadClusterSpec", fmt.Sprintf("validate searchHeadCluster spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) scopedLog.Error(err, "Failed to validate searchHeadCluster spec") return result, err } @@ -78,7 +78,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie namespaceScopedSecret, err := ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkSearchHead) if err != nil { scopedLog.Error(err, "create or update general config failed", "error", err.Error()) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -88,7 +88,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -140,7 +140,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie result.Requeue = false } if err != nil { - eventPublisher.Warning(ctx, "Delete", fmt.Sprintf("delete custom resource failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonDeleteFailed, fmt.Sprintf("Failed to delete custom resource %s — check operator logs", cr.GetName())) } return result, err } @@ -335,7 +335,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli if err != nil { // Emit event for password sync failure if eventPublisher != nil { - eventPublisher.Warning(ctx, "PasswordSyncFailed", + eventPublisher.Warning(ctx, EventReasonPasswordSyncFailed, fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", shPodName, err.Error())) } return err @@ -350,7 +350,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli if err != nil { // Emit event for password sync failure if eventPublisher != nil { - eventPublisher.Warning(ctx, "PasswordSyncFailed", + eventPublisher.Warning(ctx, EventReasonPasswordSyncFailed, fmt.Sprintf("Password sync failed for pod '%s': %s. Check pod logs and secret format.", shPodName, err.Error())) } return err @@ -436,7 +436,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli // Emit event for password sync completed if eventPublisher != nil { - eventPublisher.Normal(ctx, "PasswordSyncCompleted", + eventPublisher.Normal(ctx, EventReasonPasswordSyncCompleted, fmt.Sprintf("Password synchronized for %d pods", howManyPodsHaveSecretChanged)) } @@ -535,7 +535,7 @@ func getSearchHeadClusterList(ctx context.Context, c splcommon.ControllerClient, err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "SearchHeadCluster types not found in namespace", "namsespace", cr.GetNamespace()) + scopedLog.Error(err, "SearchHeadCluster types not found in namespace", "namespace", cr.GetNamespace()) return objectList, err } diff --git a/pkg/splunk/enterprise/searchheadclusterpodmanager.go b/pkg/splunk/enterprise/searchheadclusterpodmanager.go index 956a5fd60..8ca60c7d5 100644 --- a/pkg/splunk/enterprise/searchheadclusterpodmanager.go +++ b/pkg/splunk/enterprise/searchheadclusterpodmanager.go @@ -82,12 +82,12 @@ func (mgr *searchHeadClusterPodManager) Update(ctx context.Context, c splcommon. if phase == enterpriseApi.PhaseReady { if desiredReplicas > previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", + eventPublisher.Normal(ctx, EventReasonScaledUp, fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) } } else if desiredReplicas < previousReplicas && mgr.cr.Status.Replicas == desiredReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", + eventPublisher.Normal(ctx, EventReasonScaledDown, fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", mgr.cr.GetName(), previousReplicas, desiredReplicas)) } } diff --git a/pkg/splunk/enterprise/standalone.go b/pkg/splunk/enterprise/standalone.go index 36bf92a0c..24f0a0c9b 100644 --- a/pkg/splunk/enterprise/standalone.go +++ b/pkg/splunk/enterprise/standalone.go @@ -62,7 +62,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr // validate and updates defaults for CR err = validateStandaloneSpec(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "validateStandaloneSpec", fmt.Sprintf("validate standalone spec failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonValidateSpecFailed, fmt.Sprintf("Spec validation failed for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Failed to validate standalone spec", "error", err) return result, err } @@ -80,7 +80,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr AreRemoteVolumeKeysChanged(ctx, client, cr, SplunkStandalone, &cr.Spec.SmartStore, cr.Status.ResourceRevMap, &err) { if err != nil { - eventPublisher.Warning(ctx, "AreRemoteVolumeKeysChanged", fmt.Sprintf("check remote volume key change failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonRemoteVolumeKeyCheckFailed, fmt.Sprintf("Remote volume key change check failed for %s — check operator logs", cr.GetName())) return result, err } @@ -98,7 +98,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr if len(cr.Spec.AppFrameworkConfig.AppSources) != 0 { err := initAndCheckAppInfoStatus(ctx, client, cr, &cr.Spec.AppFrameworkConfig, &cr.Status.AppContext) if err != nil { - eventPublisher.Warning(ctx, "initAndCheckAppInfoStatus", fmt.Sprintf("init and check app info status failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonAppFrameworkInitFailed, fmt.Sprintf("App framework initialization failed for %s — check operator logs", cr.GetName())) cr.Status.AppContext.IsDeploymentInProgress = false return result, err } @@ -110,7 +110,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr _, err = ApplySplunkConfig(ctx, client, cr, cr.Spec.CommonSplunkSpec, SplunkStandalone) if err != nil { logger.ErrorContext(ctx, "create or update general config failed", "error", err) - eventPublisher.Warning(ctx, "ApplySplunkConfig", fmt.Sprintf("create or update general config failed with error %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplySplunkConfigFailed, fmt.Sprintf("Failed to apply general config for %s — check operator logs", cr.GetName())) return result, err } @@ -124,7 +124,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, getStandaloneExtraEnv(cr, cr.Spec.Replicas), false) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMap", fmt.Sprintf("create/update monitoring console config map failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleConfigFailed, fmt.Sprintf("Failed to update monitoring console config map for %s — check operator logs", cr.GetName())) return result, err } } @@ -154,14 +154,14 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr // create or update a headless service err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkStandalone, true)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update headless service failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply headless service for %s — check operator logs", cr.GetName())) return result, err } // create or update a regular service err = splctrl.ApplyService(ctx, client, getSplunkService(ctx, cr, &cr.Spec.CommonSplunkSpec, SplunkStandalone, false)) if err != nil { - eventPublisher.Warning(ctx, "ApplyService", fmt.Sprintf("create/update regular service failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonApplyServiceFailed, fmt.Sprintf("Failed to apply regular service for %s — check operator logs", cr.GetName())) return result, err } @@ -203,14 +203,14 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr // create or update statefulset statefulSet, err := getStandaloneStatefulSet(ctx, client, cr) if err != nil { - eventPublisher.Warning(ctx, "getStandaloneStatefulSet", fmt.Sprintf("get standalone status set failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonStatefulSetFailed, fmt.Sprintf("Failed to get standalone statefulset for %s — check operator logs", cr.GetName())) return result, err } //make changes to respective mc configmap when changing/removing mcRef from spec err = validateMonitoringConsoleRef(ctx, client, statefulSet, getStandaloneExtraEnv(cr, cr.Spec.Replicas)) if err != nil { - eventPublisher.Warning(ctx, "validateMonitoringConsoleRef", fmt.Sprintf("validate monitoring console reference failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleRefFailed, fmt.Sprintf("Monitoring console reference validation failed for %s — check operator logs", cr.GetName())) return result, err } @@ -221,7 +221,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr phase, err := mgr.Update(ctx, client, statefulSet, cr.Spec.Replicas) cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas if err != nil { - eventPublisher.Warning(ctx, "validateStandaloneSpec", fmt.Sprintf("update stateful set failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonStatefulSetUpdateFailed, fmt.Sprintf("Failed to update statefulset for %s — check operator logs", cr.GetName())) return result, err } @@ -233,12 +233,12 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr if cr.Status.ReadyReplicas == desiredReplicas && previousReadyReplicas != desiredReplicas { if desiredReplicas > previousReadyReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledUp", + eventPublisher.Normal(ctx, EventReasonScaledUp, fmt.Sprintf("Successfully scaled %s up from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) } } else if desiredReplicas < previousReadyReplicas { if eventPublisher != nil { - eventPublisher.Normal(ctx, "ScaledDown", + eventPublisher.Normal(ctx, EventReasonScaledDown, fmt.Sprintf("Successfully scaled %s down from %d to %d replicas", cr.GetName(), previousReadyReplicas, desiredReplicas)) } } @@ -248,7 +248,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr if cr.Spec.MonitoringConsoleRef.Name != "" { _, err = ApplyMonitoringConsoleEnvConfigMap(ctx, client, cr.GetNamespace(), cr.GetName(), cr.Spec.MonitoringConsoleRef.Name, getStandaloneExtraEnv(cr, cr.Spec.Replicas), true) if err != nil { - eventPublisher.Warning(ctx, "ApplyMonitoringConsoleEnvConfigMap", fmt.Sprintf("apply monitoring console environment config map failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleConfigFailed, fmt.Sprintf("Failed to apply monitoring console config map for %s — check operator logs", cr.GetName())) return result, err } } @@ -259,7 +259,7 @@ func ApplyStandalone(ctx context.Context, client splcommon.ControllerClient, cr namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - eventPublisher.Warning(ctx, "DeleteReferencesToAutomatedMCIfExists", fmt.Sprintf("delete reference to automated MC if exists failed %s", err.Error())) + eventPublisher.Warning(ctx, EventReasonMonitoringConsoleCleanupFailed, fmt.Sprintf("Failed to clean up automated monitoring console for %s — check operator logs", cr.GetName())) logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } diff --git a/pkg/splunk/enterprise/upgrade.go b/pkg/splunk/enterprise/upgrade.go index a62fe4f00..737ba5804 100644 --- a/pkg/splunk/enterprise/upgrade.go +++ b/pkg/splunk/enterprise/upgrade.go @@ -34,7 +34,7 @@ var GetClusterInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager // false - exit the reconciliation loop with error func UpgradePathValidation(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, spec enterpriseApi.CommonSplunkSpec, mgr *indexerClusterPodManager) (bool, error) { reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isClusterManagerReadyForUpgrade").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := reqLogger.WithName(EventReasonUpgradeCheckFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -77,7 +77,7 @@ LicenseManager: // get current image of license manager lmImage, err := getCurrentImage(ctx, c, licenseManager, SplunkLicenseManager) if err != nil { - eventPublisher.Warning(ctx, "isClusterManagerReadyForUpgrade", fmt.Sprintf("Could not get the License Manager Image. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the License Manager Image. Reason %v", err)) scopedLog.Error(err, "Unable to get licenseManager current image") return false, err } @@ -124,7 +124,7 @@ ClusterManager: // get the cluster manager referred in custom resource err := c.Get(ctx, namespacedName, clusterManager) if err != nil { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Cluster Manager. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Cluster Manager. Reason %v", err)) scopedLog.Error(err, "Unable to get clusterManager") goto SearchHeadCluster } @@ -132,7 +132,7 @@ ClusterManager: /// get the cluster manager image referred in custom resource cmImage, err := getCurrentImage(ctx, c, clusterManager, SplunkClusterManager) if err != nil { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) scopedLog.Error(err, "Unable to get clusterManager current image") return false, err } @@ -145,7 +145,7 @@ ClusterManager: if cmImage != spec.Image { // Emit event when upgrade is blocked due to ClusterManager / IndexerCluster version mismatch if eventPublisher != nil { - eventPublisher.Warning(ctx, "UpgradeBlockedVersionMismatch", + eventPublisher.Warning(ctx, EventReasonUpgradeBlockedVersionMismatch, fmt.Sprintf("Upgrade blocked: ClusterManager version %s != IndexerCluster version %s. Upgrade ClusterManager first.", cmImage, spec.Image)) } return false, fmt.Errorf("cluster manager %s image (%s) does not match IndexerCluster image (%s). Please upgrade ClusterManager and IndexerCluster together using the operator's RELATED_IMAGE_SPLUNK_ENTERPRISE or upgrade the ClusterManager first", clusterManager.Name, cmImage, spec.Image) @@ -254,7 +254,7 @@ SearchHeadCluster: shcImage, err := getCurrentImage(ctx, c, &searchHeadClusterInstance, SplunkSearchHead) if err != nil { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not get the Search Head Cluster Image. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the Search Head Cluster Image. Reason %v", err)) scopedLog.Error(err, "Unable to get SearchHeadCluster current image") return false, err } @@ -277,7 +277,7 @@ MonitoringConsole: clusterManagerList := &enterpriseApi.ClusterManagerList{} err := c.List(ctx, clusterManagerList, listOpts...) if err != nil && err.Error() != "NotFound" { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Cluster Manager list. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Cluster Manager list. Reason %v", err)) scopedLog.Error(err, "Unable to get clusterManager list") return false, err } @@ -295,7 +295,7 @@ MonitoringConsole: searchHeadClusterList := &enterpriseApi.SearchHeadClusterList{} err = c.List(ctx, searchHeadClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Search Head Cluster list. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Search Head Cluster list. Reason %v", err)) scopedLog.Error(err, "Unable to get Search Head Cluster list") return false, err } @@ -313,7 +313,7 @@ MonitoringConsole: indexerClusterList := &enterpriseApi.IndexerClusterList{} err = c.List(ctx, indexerClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Indexer list. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Indexer list. Reason %v", err)) scopedLog.Error(err, "Unable to get indexer cluster list") return false, err } @@ -331,7 +331,7 @@ MonitoringConsole: standaloneList := &enterpriseApi.IndexerClusterList{} err = c.List(ctx, standaloneList, listOpts...) if err != nil && err.Error() != "NotFound" { - eventPublisher.Warning(ctx, "UpgradePathValidation", fmt.Sprintf("Could not find the Standalone list. Reason %v", err)) + eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Standalone list. Reason %v", err)) scopedLog.Error(err, "Unable to get standalone list") return false, err } diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index 3b525f94a..1dc4af60c 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -165,7 +165,7 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie // Emit event for missing secret if k8serrors.IsNotFound(err) { if eventPublisher != nil { - eventPublisher.Warning(ctx, "SecretMissing", + eventPublisher.Warning(ctx, EventReasonSecretMissing, fmt.Sprintf("Required secret '%s' not found in namespace '%s'. Create secret to proceed.", appSecretRef, cr.GetNamespace())) } } @@ -221,7 +221,7 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie scopedLog.Error(err, "Failed to get the S3 client") // Emit event when operator cannot connect to the remote app repository if eventPublisher != nil { - eventPublisher.Warning(ctx, "AppRepositoryConnectionFailed", + eventPublisher.Warning(ctx, EventReasonAppRepoConnFailed, fmt.Sprintf("Failed to connect to app repository '%s': %s. Check credentials and network.", vol.Name, err.Error())) } return remoteDataClient, err @@ -425,7 +425,7 @@ func GetSmartstoreRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi. // Emit event for missing secret if k8serrors.IsNotFound(err) { if eventPublisher != nil { - eventPublisher.Warning(ctx, "SecretMissing", + eventPublisher.Warning(ctx, EventReasonSecretMissing, fmt.Sprintf("Required secret '%s' not found in namespace '%s'. Create secret to proceed.", volume.SecretRef, cr.GetNamespace())) } } @@ -439,13 +439,13 @@ func GetSmartstoreRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi. if accessKey == "" { if eventPublisher != nil { - eventPublisher.Warning(ctx, "SecretInvalid", + eventPublisher.Warning(ctx, EventReasonSecretInvalid, fmt.Sprintf("Secret '%s' missing required fields: %s. Update secret with required data.", namespaceScopedSecret.GetName(), "accessKey")) } return "", "", "", fmt.Errorf("s3 Access Key is missing") } else if secretKey == "" { if eventPublisher != nil { - eventPublisher.Warning(ctx, "SecretInvalid", + eventPublisher.Warning(ctx, EventReasonSecretInvalid, fmt.Sprintf("Secret '%s' missing required fields: %s. Update secret with required data.", namespaceScopedSecret.GetName(), "s3SecretKey")) } return "", "", "", fmt.Errorf("s3 Secret Key is missing") From f97cd81cd3d3b1d152478380bbde808f27bfb173 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Tue, 31 Mar 2026 14:10:15 +0200 Subject: [PATCH 10/19] Update lefrover calls --- pkg/splunk/enterprise/afwscheduler.go | 404 +++++++++++++------------- pkg/splunk/enterprise/util.go | 353 +++++++++++----------- 2 files changed, 371 insertions(+), 386 deletions(-) diff --git a/pkg/splunk/enterprise/afwscheduler.go b/pkg/splunk/enterprise/afwscheduler.go index 17cfb0ce4..d1dec04e3 100644 --- a/pkg/splunk/enterprise/afwscheduler.go +++ b/pkg/splunk/enterprise/afwscheduler.go @@ -27,12 +27,12 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" "github.com/pkg/errors" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -71,8 +71,8 @@ func isFanOutApplicableToCR(cr splcommon.MetaObject) bool { func (ppln *AppInstallPipeline) createAndAddPipelineWorker(ctx context.Context, phase enterpriseApi.AppPhaseType, appDeployInfo *enterpriseApi.AppDeploymentInfo, appSrcName string, podName string, appFrameworkConfig *enterpriseApi.AppFrameworkSpec, client splcommon.ControllerClient, cr splcommon.MetaObject, statefulSet *appsv1.StatefulSet) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("createAndAddPipelineWorker").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "createAndAddPipelineWorker", "name", cr.GetName(), "namespace", cr.GetNamespace()) worker := &PipelineWorker{ appDeployInfo: appDeployInfo, @@ -85,7 +85,7 @@ func (ppln *AppInstallPipeline) createAndAddPipelineWorker(ctx context.Context, fanOut: isFanOutApplicableToCR(cr), } - scopedLog.Info("Created new worker", "Pod name", worker.targetPodName, "App name", appDeployInfo.AppName, "digest", appDeployInfo.ObjectHash, "phase", appDeployInfo.PhaseInfo.Phase, "fan out", worker.fanOut) + scopedLog.InfoContext(ctx, "Created new worker", "Pod name", worker.targetPodName, "App name", appDeployInfo.AppName, "digest", appDeployInfo.ObjectHash, "phase", appDeployInfo.PhaseInfo.Phase, "fan out", worker.fanOut) ppln.addWorkersToPipelinePhase(ctx, phase, worker) } @@ -171,8 +171,7 @@ func getTelAppNameExtension(crKind string) (string, error) { var addTelApp = func(ctx context.Context, podExecClient splutil.PodExecClientImpl, replicas int32, cr splcommon.MetaObject) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("addTelApp").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "addTelApp", "name", cr.GetObjectMeta().GetName(), "namespace", cr.GetObjectMeta().GetNamespace()) @@ -201,13 +200,13 @@ var addTelApp = func(ctx context.Context, podExecClient splutil.PodExecClientImp // Run the commands on Splunk pods err = runCustomCommandOnSplunkPods(ctx, cr, replicas, command1, podExecClient) if err != nil { - scopedLog.Error(err, "unable to run command on splunk pod") + scopedLog.ErrorContext(ctx, "unable to run command on splunk pod", "error", err) return err } err = runCustomCommandOnSplunkPods(ctx, cr, replicas, command2, podExecClient) if err != nil { - scopedLog.Error(err, "unable to run command on splunk pod") + scopedLog.ErrorContext(ctx, "unable to run command on splunk pod", "error", err) return err } @@ -237,11 +236,11 @@ func canAppScopeHaveInstallWorker(scope string) bool { // addWorkersToPipelinePhase adds a worker to a given pipeline phase func (ppln *AppInstallPipeline) addWorkersToPipelinePhase(ctx context.Context, phaseID enterpriseApi.AppPhaseType, workers ...*PipelineWorker) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("addWorkersToPipelinePhase").WithValues("phase", phaseID) + + scopedLog := logging.FromContext(ctx).With("func", "addWorkersToPipelinePhase", "phase", phaseID) for _, worker := range workers { - scopedLog.Info("Adding worker", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "Pod name", worker.targetPodName, "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash) + scopedLog.InfoContext(ctx, "Adding worker", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "Pod name", worker.targetPodName, "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash) } ppln.pplnPhases[phaseID].mutex.Lock() ppln.pplnPhases[phaseID].q = append(ppln.pplnPhases[phaseID].q, workers...) @@ -250,8 +249,8 @@ func (ppln *AppInstallPipeline) addWorkersToPipelinePhase(ctx context.Context, p // deleteWorkerFromPipelinePhase deletes a given worker from a pipeline phase func (ppln *AppInstallPipeline) deleteWorkerFromPipelinePhase(ctx context.Context, phaseID enterpriseApi.AppPhaseType, worker *PipelineWorker) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("deleteWorkerFromPipelinePhase").WithValues("phase", phaseID) + + scopedLog := logging.FromContext(ctx).With("func", "deleteWorkerFromPipelinePhase", "phase", phaseID) ppln.pplnPhases[phaseID].mutex.Lock() defer ppln.pplnPhases[phaseID].mutex.Unlock() @@ -264,7 +263,7 @@ func (ppln *AppInstallPipeline) deleteWorkerFromPipelinePhase(ctx context.Contex phaseQ = phaseQ[:len(phaseQ)-1] ppln.pplnPhases[phaseID].q = phaseQ - scopedLog.Info("Deleted worker", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "Pod name", worker.targetPodName, "phase", phaseID, "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash) + scopedLog.InfoContext(ctx, "Deleted worker", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "Pod name", worker.targetPodName, "phase", phaseID, "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash) return true } } @@ -305,8 +304,7 @@ func createFanOutWorker(seedWorker *PipelineWorker, ordinalIdx int) *PipelineWor // In the case of Standalone CR with multiple replicas, Fan-out `replicas` number of new workers func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worker *PipelineWorker, currentPhase, nextPhase enterpriseApi.AppPhaseType) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("transitionWorkerPhase").WithValues("name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash, "pod name", worker.targetPodName, "current Phase", currentPhase, "next phase", nextPhase) + scopedLog := logging.FromContext(ctx).With("func", "transitionWorkerPhase", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "App name", worker.appDeployInfo.AppName, "digest", worker.appDeployInfo.ObjectHash, "pod name", worker.targetPodName, "current Phase", currentPhase, "next phase", nextPhase) var replicaCount int32 if worker.sts != nil { @@ -327,14 +325,14 @@ func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worke // without affecting other pods. appDeployInfo := worker.appDeployInfo if worker.fanOut { - scopedLog.Info("Fan-out transition") + scopedLog.InfoContext(ctx, "Fan-out transition") if currentPhase == enterpriseApi.PhaseDownload { // On a reconcile entry, processing the Standalone CR right after loading the appDeployContext from the CR status var podCopyWorkers, installWorkers []*PipelineWorker // Seems like the download just finished. Allocate Phase info if len(appDeployInfo.AuxPhaseInfo) == 0 { - scopedLog.Info("Just finished the download phase") + scopedLog.InfoContext(ctx, "Just finished the download phase") // Create Phase info for all the statefulset Pods. appDeployInfo.AuxPhaseInfo = make([]enterpriseApi.PhaseInfo, replicaCount) @@ -347,10 +345,10 @@ func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worke podCopyWorkers[podID] = createFanOutWorker(worker, podID) setContextForNewPhase(&appDeployInfo.AuxPhaseInfo[podID], enterpriseApi.PhasePodCopy) - scopedLog.Info("Created a new fan-out pod copy worker", "pod name", worker.targetPodName) + scopedLog.InfoContext(ctx, "Created a new fan-out pod copy worker", "pod name", worker.targetPodName) } } else { - scopedLog.Info("Installation was already in progress for replica members") + scopedLog.InfoContext(ctx, "Installation was already in progress for replica members") for podID := range appDeployInfo.AuxPhaseInfo { phaseInfo := &appDeployInfo.AuxPhaseInfo[podID] @@ -366,7 +364,7 @@ func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worke } else if phaseInfo.Phase == enterpriseApi.PhasePodCopy { podCopyWorkers = append(podCopyWorkers, newWorker) } else { - scopedLog.Error(nil, "invalid phase info detected", "phase", phaseInfo.Phase, "phase status", phaseInfo.Status) + scopedLog.ErrorContext(ctx, "invalid phase info detected", "phase", phaseInfo.Phase, "phase status", phaseInfo.Status) } } } @@ -374,11 +372,11 @@ func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worke ppln.addWorkersToPipelinePhase(ctx, enterpriseApi.PhasePodCopy, podCopyWorkers...) ppln.addWorkersToPipelinePhase(ctx, enterpriseApi.PhaseInstall, installWorkers...) } else { - scopedLog.Error(nil, "Invalid phase detected") + scopedLog.ErrorContext(ctx, "Invalid phase detected") } } else { - scopedLog.Info("Simple transition") + scopedLog.InfoContext(ctx, "Simple transition") var phaseInfo *enterpriseApi.PhaseInfo if isFanOutApplicableToCR(worker.cr) { @@ -394,7 +392,7 @@ func (ppln *AppInstallPipeline) transitionWorkerPhase(ctx context.Context, worke // We have already moved the worker(s) to the required queue. // Now, safely delete the worker from the current phase queue - scopedLog.Info("Deleted worker", "phase", currentPhase) + scopedLog.InfoContext(ctx, "Deleted worker", "phase", currentPhase) ppln.deleteWorkerFromPipelinePhase(ctx, currentPhase, worker) } @@ -420,13 +418,13 @@ func needToUseAuxPhaseInfo(worker *PipelineWorker, phaseType enterpriseApi.AppPh // getPhaseInfoByPhaseType gives the phase info suitable for a given phase func getPhaseInfoByPhaseType(ctx context.Context, worker *PipelineWorker, phaseType enterpriseApi.AppPhaseType) *enterpriseApi.PhaseInfo { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getPhaseInfoFromWorker") + + scopedLog := logging.FromContext(ctx).With("func", "getPhaseInfoFromWorker") if needToUseAuxPhaseInfo(worker, phaseType) { podID, err := getOrdinalValFromPodName(worker.targetPodName) if err != nil { - scopedLog.Error(err, "unable to get the pod Id", "pod name", worker.targetPodName) + scopedLog.ErrorContext(ctx, "unable to get the pod Id", "pod name", worker.targetPodName, "error", err) return nil } @@ -438,17 +436,17 @@ func getPhaseInfoByPhaseType(ctx context.Context, worker *PipelineWorker, phaseT // updatePplnWorkerPhaseInfo updates the in-memory PhaseInfo(specifically status and retryCount) func updatePplnWorkerPhaseInfo(ctx context.Context, appDeployInfo *enterpriseApi.AppDeploymentInfo, failCount uint32, statusType enterpriseApi.AppPhaseStatusType) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updatePplnWorkerPhaseInfo").WithValues("appName", appDeployInfo.AppName) - scopedLog.Info("changing the status", "old status", appPhaseStatusAsStr(appDeployInfo.PhaseInfo.Status), "new status", appPhaseStatusAsStr(statusType)) + scopedLog := logging.FromContext(ctx).With("func", "updatePplnWorkerPhaseInfo", "appName", appDeployInfo.AppName) + + scopedLog.InfoContext(ctx, "changing the status", "old status", appPhaseStatusAsStr(appDeployInfo.PhaseInfo.Status), "new status", appPhaseStatusAsStr(statusType)) appDeployInfo.PhaseInfo.FailCount = failCount appDeployInfo.PhaseInfo.Status = statusType } func (downloadWorker *PipelineWorker) createDownloadDirOnOperator(ctx context.Context) (string, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("createDownloadDirOnOperator").WithValues("appSrcName", downloadWorker.appSrcName, "appName", downloadWorker.appDeployInfo.AppName) + + scopedLog := logging.FromContext(ctx).With("func", "createDownloadDirOnOperator", "appSrcName", downloadWorker.appSrcName, "appName", downloadWorker.appDeployInfo.AppName) scope := getAppSrcScope(ctx, downloadWorker.afwConfig, downloadWorker.appSrcName) kind := downloadWorker.cr.GetObjectKind().GroupVersionKind().Kind @@ -462,7 +460,7 @@ func (downloadWorker *PipelineWorker) createDownloadDirOnOperator(ctx context.Co // create the sub-directories on the volume for downloading scoped apps err := createAppDownloadDir(ctx, localPath) if err != nil { - scopedLog.Error(err, "unable to create app download directory on operator") + scopedLog.ErrorContext(ctx, "unable to create app download directory on operator", "error", err) } return localPath, err } @@ -480,8 +478,8 @@ func (downloadWorker *PipelineWorker) download(ctx context.Context, pplnPhase *P splunkCR := downloadWorker.cr appSrcName := downloadWorker.appSrcName - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PipelineWorker.Download()").WithValues("name", splunkCR.GetName(), "namespace", splunkCR.GetNamespace(), "App name", downloadWorker.appDeployInfo.AppName, "objectHash", downloadWorker.appDeployInfo.ObjectHash) + + scopedLog := logging.FromContext(ctx).With("func", "PipelineWorker.Download", "name", splunkCR.GetName(), "namespace", splunkCR.GetNamespace(), "App name", downloadWorker.appDeployInfo.AppName, "objectHash", downloadWorker.appDeployInfo.ObjectHash) appDeployInfo := downloadWorker.appDeployInfo appName := appDeployInfo.AppName @@ -489,7 +487,7 @@ func (downloadWorker *PipelineWorker) download(ctx context.Context, pplnPhase *P localFile := getLocalAppFileName(ctx, localPath, appName, appDeployInfo.ObjectHash) remoteFile, err := getRemoteObjectKey(ctx, splunkCR, downloadWorker.afwConfig, appSrcName, appName) if err != nil { - scopedLog.Error(err, "unable to get remote object key", "appName", appName) + scopedLog.ErrorContext(ctx, "unable to get remote object key", "appName", appName, "error", err) // increment the retry count and mark this app as download pending updatePplnWorkerPhaseInfo(ctx, appDeployInfo, appDeployInfo.PhaseInfo.FailCount+1, enterpriseApi.AppPkgDownloadPending) @@ -499,12 +497,12 @@ func (downloadWorker *PipelineWorker) download(ctx context.Context, pplnPhase *P // download the app from remote storage err = remoteDataClientMgr.DownloadApp(ctx, remoteFile, localFile, appDeployInfo.ObjectHash) if err != nil { - scopedLog.Error(err, "unable to download app", "appName", appName) + scopedLog.ErrorContext(ctx, "unable to download app", "appName", appName, "error", err) // remove the local file err = os.RemoveAll(localFile) if err != nil { - scopedLog.Error(err, "unable to remove local file from operator") + scopedLog.ErrorContext(ctx, "unable to remove local file from operator", "error", err) } // increment the retry count and mark this app as download pending @@ -515,14 +513,13 @@ func (downloadWorker *PipelineWorker) download(ctx context.Context, pplnPhase *P // download is successful, update the state and reset the retry count updatePplnWorkerPhaseInfo(ctx, appDeployInfo, 0, enterpriseApi.AppPkgDownloadComplete) - scopedLog.Info("Finished downloading app") + scopedLog.InfoContext(ctx, "Finished downloading app") } // downloadWorkerHandler schedules the download workers to download app/s func (pplnPhase *PipelinePhase) downloadWorkerHandler(ctx context.Context, ppln *AppInstallPipeline, maxWorkers uint64, scheduleDownloadsWaiter *sync.WaitGroup) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("downloadWorkerHandler") + scopedLog := logging.FromContext(ctx).With("func", "downloadWorkerHandler") // derive a counting semaphore from the channel to represent worker run pool var downloadWorkersRunPool = make(chan struct{}, maxWorkers) @@ -536,13 +533,13 @@ downloadWork: case downloadWorker, ok := <-pplnPhase.msgChannel: // if channel is closed, then just break from here as we have nothing to read if !ok { - scopedLog.Info("msgChannel is closed by downloadPhaseManager, hence nothing to read.") + scopedLog.InfoContext(ctx, "msgChannel is closed by downloadPhaseManager, hence nothing to read.") break downloadWork } // do not redownload the app if it is already downloaded if isAppAlreadyDownloaded(ctx, downloadWorker) { - scopedLog.Info("app is already downloaded on operator pod, hence skipping it.", "appSrcName", downloadWorker.appSrcName, "appName", downloadWorker.appDeployInfo.AppName) + scopedLog.InfoContext(ctx, "app is already downloaded on operator pod, hence skipping it.", "appSrcName", downloadWorker.appSrcName, "appName", downloadWorker.appDeployInfo.AppName) // update the state to be download complete updatePplnWorkerPhaseInfo(ctx, downloadWorker.appDeployInfo, 0, enterpriseApi.AppPkgDownloadComplete) <-downloadWorkersRunPool @@ -553,7 +550,7 @@ downloadWork: err := reserveStorage(downloadWorker.appDeployInfo.Size) if err != nil { - scopedLog.Error(err, "insufficient storage for the app pkg download. appSrcName: %s, app name: %s, app size: %d Bytes", downloadWorker.appSrcName, downloadWorker.appDeployInfo.AppName, downloadWorker.appDeployInfo.Size) + scopedLog.ErrorContext(ctx, "insufficient storage for the app pkg download", "appSrcName", downloadWorker.appSrcName, "appName", downloadWorker.appDeployInfo.AppName, "appSize", downloadWorker.appDeployInfo.Size, "error", err) // setting isActive to false here so that downloadPhaseManager can take care of it. downloadWorker.isActive = false <-downloadWorkersRunPool @@ -568,7 +565,7 @@ downloadWork: // create the sub-directories on the volume for downloading scoped apps localPath, err := downloadWorker.createDownloadDirOnOperator(ctx) if err != nil { - scopedLog.Error(err, "unable to create download directory on operator", "appSrcName", downloadWorker.appSrcName, "appName", appDeployInfo.AppName) + scopedLog.ErrorContext(ctx, "unable to create download directory on operator", "appSrcName", downloadWorker.appSrcName, "appName", appDeployInfo.AppName, "error", err) // increment the retry count and mark this app as download pending updatePplnWorkerPhaseInfo(ctx, appDeployInfo, appDeployInfo.PhaseInfo.FailCount+1, enterpriseApi.AppPkgDownloadPending) @@ -580,7 +577,7 @@ downloadWork: // get the remoteDataClientMgr instance remoteDataClientMgr, err := getRemoteDataClientMgr(ctx, downloadWorker.client, downloadWorker.cr, downloadWorker.afwConfig, downloadWorker.appSrcName) if err != nil { - scopedLog.Error(err, "unable to get remote data client manager") + scopedLog.ErrorContext(ctx, "unable to get remote data client manager", "error", err) // increment the retry count and mark this app as download error updatePplnWorkerPhaseInfo(ctx, appDeployInfo, appDeployInfo.PhaseInfo.FailCount+1, enterpriseApi.AppPkgDownloadError) @@ -599,7 +596,7 @@ downloadWork: } default: // All the workers are busy, check after one second - scopedLog.Info("All the workers are busy, we will check again after one second") + scopedLog.InfoContext(ctx, "All the workers are busy, we will check again after one second") time.Sleep(phaseManagerBusyWaitDuration) } @@ -618,26 +615,26 @@ downloadWork: // 2. wait for the handler to finish all its work // 3. mark the phase as done/complete func (ppln *AppInstallPipeline) shutdownPipelinePhase(ctx context.Context, phaseManager string, pplnPhase *PipelinePhase, perPhaseWaiter *sync.WaitGroup) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName(phaseManager) + + scopedLog := logging.FromContext(ctx).With("func", phaseManager) // close the msgChannel close(pplnPhase.msgChannel) // wait for the handler code to finish its work - scopedLog.Info("Waiting for the workers to finish") + scopedLog.InfoContext(ctx, "Waiting for the workers to finish") perPhaseWaiter.Wait() // mark the phase as done/complete - scopedLog.Info("All the workers finished") + scopedLog.InfoContext(ctx, "All the workers finished") ppln.phaseWaiter.Done() } // downloadPhaseManager creates download phase manager for the install pipeline func (ppln *AppInstallPipeline) downloadPhaseManager(ctx context.Context) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("downloadPhaseManager") - scopedLog.Info("Starting Download phase manager") + + scopedLog := logging.FromContext(ctx).With("func", "downloadPhaseManager") + scopedLog.InfoContext(ctx, "Starting Download phase manager") pplnPhase := ppln.pplnPhases[enterpriseApi.PhaseDownload] @@ -657,7 +654,7 @@ downloadPhase: select { case _, channelOpen := <-ppln.sigTerm: if !channelOpen { - scopedLog.Info("Received the termination request from the scheduler") + scopedLog.InfoContext(ctx, "Received the termination request from the scheduler") break downloadPhase } @@ -674,7 +671,7 @@ downloadPhase: downloadWorker.waiter = &pplnPhase.workerWaiter select { case pplnPhase.msgChannel <- downloadWorker: - scopedLog.Info("Download worker got a run slot", "name", downloadWorker.cr.GetName(), "namespace", downloadWorker.cr.GetNamespace(), "App name", downloadWorker.appDeployInfo.AppName, "digest", downloadWorker.appDeployInfo.ObjectHash) + scopedLog.InfoContext(ctx, "Download worker got a run slot", "name", downloadWorker.cr.GetName(), "namespace", downloadWorker.cr.GetNamespace(), "App name", downloadWorker.appDeployInfo.AppName, "digest", downloadWorker.appDeployInfo.ObjectHash) downloadWorker.isActive = true default: downloadWorker.waiter = nil @@ -689,8 +686,8 @@ downloadPhase: // markWorkerPhaseInstallationComplete marks the worker phase as app package installation complete func markWorkerPhaseInstallationComplete(ctx context.Context, phaseInfo *enterpriseApi.PhaseInfo, worker *PipelineWorker) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("markWorkerPhaseInstallationComplete") + + scopedLog := logging.FromContext(ctx).With("func", "markWorkerPhaseInstallationComplete") // Set auxphase info status for fanout CRs and phaseinfo status for others phaseInfo.Status = enterpriseApi.AppPkgInstallComplete @@ -700,7 +697,7 @@ func markWorkerPhaseInstallationComplete(ctx context.Context, phaseInfo *enterpr // phaseinfo as install complete if isFanOutApplicableToCR(worker.cr) { if isAppInstallationCompleteOnAllReplicas(worker.appDeployInfo.AuxPhaseInfo) { - scopedLog.Info("app pkg installed on all the pods", "app pkg", worker.appDeployInfo.AppName) + scopedLog.InfoContext(ctx, "app pkg installed on all the pods", "app pkg", worker.appDeployInfo.AppName) worker.appDeployInfo.PhaseInfo.Phase = enterpriseApi.PhaseInstall worker.appDeployInfo.PhaseInfo.Status = enterpriseApi.AppPkgInstallComplete @@ -714,8 +711,7 @@ func markWorkerPhaseInstallationComplete(ctx context.Context, phaseInfo *enterpr func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr splcommon.MetaObject, phaseInfo *enterpriseApi.PhaseInfo) error { worker := localCtx.worker - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("installApp").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) + scopedLog := logging.FromContext(rctx).With("func", "installApp", "name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) // if the app name is app1.tgz and hash is "abcd1234", then appPkgFileName is app1.tgz_abcd1234 appPkgFileName := getAppPackageName(worker) @@ -724,7 +720,7 @@ func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp appPkgPathOnPod := filepath.Join(appBktMnt, worker.appSrcName, appPkgFileName) if !checkIfFileExistsOnPod(rctx, cr, appPkgPathOnPod, localCtx.podExecClient) { - scopedLog.Error(nil, "app pkg missing on Pod", "app pkg path", appPkgPathOnPod) + scopedLog.ErrorContext(rctx, "app pkg missing on Pod", "app pkg path", appPkgPathOnPod) phaseInfo.Status = enterpriseApi.AppPkgMissingOnPodError return fmt.Errorf("app pkg missing on Pod. app pkg path: %s", appPkgPathOnPod) @@ -733,10 +729,10 @@ func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp if worker.appDeployInfo.AppPackageTopFolder == "" { appTopFolder, err := getAppTopFolderFromPackage(rctx, cr, appPkgPathOnPod, localCtx.podExecClient) if err != nil { - scopedLog.Error(err, "local scoped app package install failed while getting name of installed app") + scopedLog.ErrorContext(rctx, "local scoped app package install failed while getting name of installed app", "error", err) return err } - scopedLog.Info("app top folder", "name", appTopFolder) + scopedLog.InfoContext(rctx, "app top folder", "name", appTopFolder) worker.appDeployInfo.AppPackageTopFolder = appTopFolder } @@ -750,16 +746,16 @@ func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp // we can come to this block if post installation failed // e.g. es post installation failed but es app was already installed - scopedLog.Info("Check if app is already installed ", "name", worker.appDeployInfo.AppPackageTopFolder) + scopedLog.InfoContext(rctx, "Check if app is already installed ", "name", worker.appDeployInfo.AppPackageTopFolder) appInstalled, err := isAppAlreadyInstalled(rctx, cr, localCtx.podExecClient, worker.appDeployInfo.AppPackageTopFolder) if err != nil { - scopedLog.Error(err, "local scoped app package install failed while checking if app is already installed") + scopedLog.ErrorContext(rctx, "local scoped app package install failed while checking if app is already installed", "error", err) return err } if appInstalled { - scopedLog.Info("Not reinstalling app as it is already installed.") + scopedLog.InfoContext(rctx, "Not reinstalling app as it is already installed.") return nil } @@ -773,13 +769,13 @@ func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp // TODO(patrykw-splunk): remove this once we have confirm that we are not using stderr for error detection at all // Log stderr content for debugging but don't use it for error detection if stdErr != "" { - scopedLog.Info("App install command stderr output (informational only)", "stderr", stdErr) + scopedLog.InfoContext(rctx, "App install command stderr output (informational only)", "stderr", stdErr) } // Check only the actual command execution error, not stderr content if err != nil { phaseInfo.FailCount++ - scopedLog.Error(err, "local scoped app package install failed", "stdout", stdOut, "stderr", stdErr, "app pkg path", appPkgPathOnPod, "failCount", phaseInfo.FailCount) + scopedLog.ErrorContext(rctx, "local scoped app package install failed", "stdout", stdOut, "stderr", stdErr, "app pkg path", appPkgPathOnPod, "failCount", phaseInfo.FailCount, "error", err) return fmt.Errorf("local scoped app package install failed. stdOut: %s, stdErr: %s, app pkg path: %s, failCount: %d", stdOut, stdErr, appPkgPathOnPod, phaseInfo.FailCount) } @@ -789,17 +785,17 @@ func installApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp // check if the given app is already installed and enabled. // the installed app name is supposed to be same as // name of top folder (AppTopFolder) -func isAppAlreadyInstalled(ctx context.Context, cr splcommon.MetaObject, podExecClient splutil.PodExecClientImpl, appTopFolder string) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isAppAlreadyInstalled").WithValues("podName", podExecClient.GetTargetPodName(), "namespace", cr.GetNamespace()).WithValues("AppTopFolder", appTopFolder) +func isAppAlreadyInstalled(rctx context.Context, cr splcommon.MetaObject, podExecClient splutil.PodExecClientImpl, appTopFolder string) (bool, error) { + + scopedLog := logging.FromContext(rctx).With("func", "isAppAlreadyInstalled", "podName", podExecClient.GetTargetPodName(), "namespace", cr.GetNamespace(), "AppTopFolder", appTopFolder) - scopedLog.Info("check app's installation state") + scopedLog.InfoContext(rctx, "check app's installation state") command := fmt.Sprintf("/opt/splunk/bin/splunk list app %s -auth admin:`cat /mnt/splunk-secrets/password`| grep ENABLED", appTopFolder) streamOptions := splutil.NewStreamOptionsObject(command) - stdOut, stdErr, err := podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) + stdOut, stdErr, err := podExecClient.RunPodExecCommand(rctx, streamOptions, []string{"/bin/sh"}) // Handle specific stderr cases first if strings.Contains(stdErr, "Could not find object") { @@ -811,7 +807,7 @@ func isAppAlreadyInstalled(ctx context.Context, cr splcommon.MetaObject, podExec // Log any other stderr content for debugging but don't use it for error detection if stdErr != "" { - scopedLog.Info("Command stderr output (informational only)", "stderr", stdErr) + scopedLog.InfoContext(rctx, "Command stderr output (informational only)", "stderr", stdErr) } // Now check the actual command result @@ -823,7 +819,7 @@ func isAppAlreadyInstalled(ctx context.Context, cr splcommon.MetaObject, podExec // Check for grep exit code 1 (pattern not found) if strings.Contains(errMsg, "exit status 1") || strings.Contains(errMsg, "command terminated with exit code 1") { // grep exit code 1 means "ENABLED" pattern not found - app exists but is not enabled - scopedLog.Info("App not enabled - grep pattern not found", "stdout", stdOut, "stderr", stdErr) + scopedLog.InfoContext(rctx, "App not enabled - grep pattern not found", "stdout", stdOut, "stderr", stdErr) return false, nil } @@ -838,22 +834,21 @@ func isAppAlreadyInstalled(ctx context.Context, cr splcommon.MetaObject, podExec return false, fmt.Errorf("command succeeded but no output received, command: %s", command) } - scopedLog.Info("App installation state check successful - app is enabled", "appStatus", strings.TrimSpace(stdOut)) + scopedLog.InfoContext(rctx, "App installation state check successful - app is enabled", "appStatus", strings.TrimSpace(stdOut)) return true, nil } // get the name of top folder from the package. // this name is later used as installed app name func getAppTopFolderFromPackage(rctx context.Context, cr splcommon.MetaObject, appPkgPathOnPod string, podExecClient splutil.PodExecClientImpl) (string, error) { - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("getAppTopFolderFromPackage").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "appPkgPathOnPod", appPkgPathOnPod) + scopedLog := logging.FromContext(rctx).With("func", "getAppTopFolderFromPackage", "name", cr.GetName(), "namespace", cr.GetNamespace(), "appPkgPathOnPod", appPkgPathOnPod) command := fmt.Sprintf("tar tf %s|head -1|cut -d/ -f1", appPkgPathOnPod) streamOptions := splutil.NewStreamOptionsObject(command) stdOut, stdErr, err := podExecClient.RunPodExecCommand(rctx, streamOptions, []string{"/bin/sh"}) - scopedLog.Info("Pod exec result", "stdOut", stdOut) + scopedLog.InfoContext(rctx, "Pod exec result", "stdOut", stdOut) if stdErr != "" || err != nil { // CSPL-2598 - Log warnings/errors. @@ -864,7 +859,7 @@ func getAppTopFolderFromPackage(rctx context.Context, cr splcommon.MetaObject, a // The onus falls on the user to make sure the app packages are tarred appropriately // to avoid the re-installation cycles as it is prudent to continue // to the install step for harmless warnings - scopedLog.Error(err, "error in tar contents list, but app installation will continue", "stdOut", stdOut, "stdErr", stdErr, "command", command, "appPkgPathOnPod", appPkgPathOnPod) + scopedLog.ErrorContext(rctx, "error in tar contents list, but app installation will continue", "stdOut", stdOut, "stdErr", stdErr, "command", command, "appPkgPathOnPod", appPkgPathOnPod, "error", err) if stdOut == "" { return "Empty app package name, could not get installed app name", err } @@ -879,8 +874,7 @@ func getAppTopFolderFromPackage(rctx context.Context, cr splcommon.MetaObject, a func cleanupApp(rctx context.Context, localCtx *localScopePlaybookContext, cr splcommon.MetaObject, phaseInfo *enterpriseApi.PhaseInfo) error { worker := localCtx.worker - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("cleanupApp").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) + scopedLog := logging.FromContext(rctx).With("func", "cleanupApp", "name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) // if the app name is app1.tgz and hash is "abcd1234", then appPkgFileName is app1.tgz_abcd1234 appPkgFileName := getAppPackageName(worker) @@ -893,10 +887,10 @@ func cleanupApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp streamOptions := splutil.NewStreamOptionsObject(command) stdOut, stdErr, err := localCtx.podExecClient.RunPodExecCommand(rctx, streamOptions, []string{"/bin/sh"}) if stdErr != "" || err != nil { - scopedLog.Error(err, "app pkg deletion failed", "stdout", stdOut, "stderr", stdErr, "app pkg path", appPkgPathOnPod) + scopedLog.ErrorContext(rctx, "app pkg deletion failed", "stdout", stdOut, "stderr", stdErr, "app pkg path", appPkgPathOnPod, "error", err) return fmt.Errorf("app pkg deletion failed. stdOut: %s, stdErr: %s, app pkg path: %s", stdOut, stdErr, appPkgPathOnPod) } - scopedLog.Info("App package deleted from target pod", "command", command) + scopedLog.InfoContext(rctx, "App package deleted from target pod", "command", command) // Try to remove the app package from the Operator Pod tryAppPkgCleanupFromOperatorPod(rctx, worker) @@ -908,8 +902,7 @@ func cleanupApp(rctx context.Context, localCtx *localScopePlaybookContext, cr sp func (localCtx *localScopePlaybookContext) runPlaybook(rctx context.Context) error { worker := localCtx.worker cr := worker.cr - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("localScopePlaybookContext.runPlaybook").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) + scopedLog := logging.FromContext(rctx).With("func", "localScopePlaybookContext.runPlaybook", "name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) defer func() { <-localCtx.sem @@ -923,7 +916,7 @@ func (localCtx *localScopePlaybookContext) runPlaybook(rctx context.Context) err // Call the API to install an app err := installApp(rctx, localCtx, cr, phaseInfo) if err != nil { - scopedLog.Error(err, "app package installation error") + scopedLog.ErrorContext(rctx, "app package installation error", "error", err) return fmt.Errorf("app pkg installation failed. error %s", err.Error()) } @@ -933,7 +926,7 @@ func (localCtx *localScopePlaybookContext) runPlaybook(rctx context.Context) err // Call the API to cleanup the app err = cleanupApp(rctx, localCtx, cr, phaseInfo) if err != nil { - scopedLog.Error(err, "app package cleanup error") + scopedLog.ErrorContext(rctx, "app package cleanup error", "error", err) return fmt.Errorf("app pkg cleanup failed. error %s", err.Error()) } @@ -943,8 +936,8 @@ func (localCtx *localScopePlaybookContext) runPlaybook(rctx context.Context) err // extractClusterScopedAppOnPod untars the given app package to the bundle push location func extractClusterScopedAppOnPod(ctx context.Context, worker *PipelineWorker, appSrcScope string, appPkgPathOnPod, appPkgLocalPath string, podExecClient splutil.PodExecClientImpl) error { cr := worker.cr - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("extractClusterScopedAppOnPod").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "app name", worker.appDeployInfo.AppName) + + scopedLog := logging.FromContext(ctx).With("func", "extractClusterScopedAppOnPod", "name", cr.GetName(), "namespace", cr.GetNamespace(), "app name", worker.appDeployInfo.AppName) var stdOut, stdErr string var err error @@ -952,7 +945,7 @@ func extractClusterScopedAppOnPod(ctx context.Context, worker *PipelineWorker, a clusterAppsPath := getClusterScopedAppsLocOnPod(worker.cr) if clusterAppsPath == "" { // This should never happen - scopedLog.Error(nil, "could not find the cluster scoped apps location on the Pod") + scopedLog.ErrorContext(ctx, "could not find the cluster scoped apps location on the Pod") return err } @@ -979,8 +972,8 @@ func extractClusterScopedAppOnPod(ctx context.Context, worker *PipelineWorker, a // runPodCopyWorker runs one pod copy worker func runPodCopyWorker(ctx context.Context, worker *PipelineWorker, ch chan struct{}) { cr := worker.cr - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("runPodCopyWorker").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "app name", worker.appDeployInfo.AppName, "pod", worker.targetPodName) + + scopedLog := logging.FromContext(ctx).With("func", "runPodCopyWorker", "name", cr.GetName(), "namespace", cr.GetNamespace(), "app name", worker.appDeployInfo.AppName, "pod", worker.targetPodName) defer func() { <-ch worker.isActive = false @@ -999,7 +992,7 @@ func runPodCopyWorker(ctx context.Context, worker *PipelineWorker, ch chan struc _, err := os.Stat(appPkgLocalPath) if err != nil { // Move the worker to download phase - scopedLog.Error(err, "app package is missing", "pod name", worker.targetPodName) + scopedLog.ErrorContext(ctx, "app package is missing", "pod name", worker.targetPodName, "error", err) phaseInfo.Status = enterpriseApi.AppPkgMissingFromOperator return } @@ -1013,7 +1006,7 @@ func runPodCopyWorker(ctx context.Context, worker *PipelineWorker, ch chan struc stdOut, stdErr, err := CopyFileToPod(ctx, worker.client, cr.GetNamespace(), appPkgLocalPath, appPkgPathOnPod, podExecClient) if err != nil { phaseInfo.FailCount++ - scopedLog.Error(err, "app package pod copy failed", "stdout", stdOut, "stderr", stdErr, "failCount", phaseInfo.FailCount) + scopedLog.ErrorContext(ctx, "app package pod copy failed", "stdout", stdOut, "stderr", stdErr, "failCount", phaseInfo.FailCount, "error", err) return } @@ -1021,19 +1014,19 @@ func runPodCopyWorker(ctx context.Context, worker *PipelineWorker, ch chan struc err = extractClusterScopedAppOnPod(ctx, worker, appSrcScope, appPkgPathOnPod, appPkgLocalPath, podExecClient) if err != nil { phaseInfo.FailCount++ - scopedLog.Error(err, "extracting the app package on pod failed", "failCount", phaseInfo.FailCount) + scopedLog.ErrorContext(ctx, "extracting the app package on pod failed", "failCount", phaseInfo.FailCount, "error", err) return } } - scopedLog.Info("podCopy complete", "app pkg path", appPkgPathOnPod) + scopedLog.InfoContext(ctx, "podCopy complete", "app pkg path", appPkgPathOnPod) phaseInfo.Status = enterpriseApi.AppPkgPodCopyComplete } // podCopyWorkerHandler fetches and runs the pod copy workers func (pplnPhase *PipelinePhase) podCopyWorkerHandler(ctx context.Context, handlerWaiter *sync.WaitGroup, numPodCopyRunners int) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("podCopyWorkerHandler") + + scopedLog := logging.FromContext(ctx).With("func", "podCopyWorkerHandler") defer handlerWaiter.Done() // Using the channel, derive a counting semaphore called podCopyRunPool that represents worker run pool @@ -1051,7 +1044,7 @@ podCopyHandler: case worker, channelOpen := <-pplnPhase.msgChannel: if !channelOpen { // Channel is closed, so, do not handle any more workers - scopedLog.Info("worker channel closed") + scopedLog.InfoContext(ctx, "worker channel closed") break podCopyHandler } @@ -1060,7 +1053,7 @@ podCopyHandler: go runPodCopyWorker(ctx, worker, podCopyWorkerPool) } else { /// This should never happen - scopedLog.Error(nil, "invalid worker reference") + scopedLog.ErrorContext(ctx, "invalid worker reference") <-podCopyWorkerPool } @@ -1076,16 +1069,16 @@ podCopyHandler: } // Wait for all the workers to finish - scopedLog.Info("Waiting for all the workers to finish") + scopedLog.InfoContext(ctx, "Waiting for all the workers to finish") pplnPhase.workerWaiter.Wait() - scopedLog.Info("All the workers finished") + scopedLog.InfoContext(ctx, "All the workers finished") } // podCopyPhaseManager creates pod copy phase manager for the install pipeline func (ppln *AppInstallPipeline) podCopyPhaseManager(ctx context.Context) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("podCopyPhaseManager") - scopedLog.Info("Starting Pod copy phase manager") + + scopedLog := logging.FromContext(ctx).With("func", "podCopyPhaseManager") + scopedLog.InfoContext(ctx, "Starting Pod copy phase manager") var handlerWaiter sync.WaitGroup pplnPhase := ppln.pplnPhases[enterpriseApi.PhasePodCopy] @@ -1105,7 +1098,7 @@ podCopyPhase: select { case _, channelOpen := <-ppln.sigTerm: if !channelOpen { - scopedLog.Info("Received the termination request from the scheduler") + scopedLog.InfoContext(ctx, "Received the termination request from the scheduler") break podCopyPhase } @@ -1129,7 +1122,7 @@ podCopyPhase: podCopyWorker.waiter = &pplnPhase.workerWaiter select { case pplnPhase.msgChannel <- podCopyWorker: - scopedLog.Info("Pod copy worker got a run slot", "name", podCopyWorker.cr.GetName(), "namespace", podCopyWorker.cr.GetNamespace(), "pod name", podCopyWorker.targetPodName, "App name", podCopyWorker.appDeployInfo.AppName, "digest", podCopyWorker.appDeployInfo.ObjectHash) + scopedLog.InfoContext(ctx, "Pod copy worker got a run slot", "name", podCopyWorker.cr.GetName(), "namespace", podCopyWorker.cr.GetNamespace(), "pod name", podCopyWorker.targetPodName, "App name", podCopyWorker.appDeployInfo.AppName, "digest", podCopyWorker.appDeployInfo.ObjectHash) podCopyWorker.isActive = true default: podCopyWorker.waiter = nil @@ -1144,11 +1137,11 @@ podCopyPhase: // getInstallSlotForPod tries to allocate a local scoped install slot for a pod func getInstallSlotForPod(ctx context.Context, installTracker []chan struct{}, podName string) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getInstallSlotForPod") + + scopedLog := logging.FromContext(ctx).With("func", "getInstallSlotForPod") podID, err := getOrdinalValFromPodName(podName) if err != nil { - scopedLog.Error(err, "unable to derive podId for podname", podName) + scopedLog.ErrorContext(ctx, "unable to derive podId for podname", "podName", podName, "error", err) return false } @@ -1162,18 +1155,18 @@ func getInstallSlotForPod(ctx context.Context, installTracker []chan struct{}, p // freeInstallSlotForPod frees up an install slot for a pod func freeInstallSlotForPod(ctx context.Context, installTracker []chan struct{}, podName string) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("freeInstallSlotForPod") + + scopedLog := logging.FromContext(ctx).With("func", "freeInstallSlotForPod") podID, err := getOrdinalValFromPodName(podName) if err != nil { - scopedLog.Error(err, "unable to derive podId for podname", podName) + scopedLog.ErrorContext(ctx, "unable to derive podId for podname", "podName", podName, "error", err) return } select { case <-installTracker[podID]: default: - scopedLog.Error(nil, "trying to free an install slot without even allocating it") + scopedLog.ErrorContext(ctx, "trying to free an install slot without even allocating it") } } @@ -1222,8 +1215,8 @@ func tryAppPkgCleanupFromOperatorPod(ctx context.Context, installWorker *Pipelin // installWorkerHandler fetches and runs the install workers // local scope installs are handled first, then the cluster scoped apps are considered for bundle push func (pplnPhase *PipelinePhase) installWorkerHandler(ctx context.Context, ppln *AppInstallPipeline, handlerWaiter *sync.WaitGroup, installTracker []chan struct{}) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("installWorkerHandler") + + scopedLog := logging.FromContext(ctx).With("func", "installWorkerHandler") defer handlerWaiter.Done() installHandler: @@ -1232,7 +1225,7 @@ installHandler: case installWorker, channelOpen := <-pplnPhase.msgChannel: if !channelOpen { // Channel is closed, so, do not handle any more workers - scopedLog.Info("worker channel closed") + scopedLog.InfoContext(ctx, "worker channel closed") break installHandler } @@ -1247,7 +1240,7 @@ installHandler: // Get app source spec appSrcSpec, err := getAppSrcSpec(installWorker.afwConfig.AppSources, installWorker.appSrcName) if err != nil { - scopedLog.Error(err, "getting app source spec failed while installing app app src name %s", installWorker.appSrcName) + scopedLog.ErrorContext(ctx, "getting app source spec failed while installing app", "appSrcName", installWorker.appSrcName, "error", err) } // Get app source scope @@ -1266,11 +1259,11 @@ installHandler: go iwctx.runPlaybook(ctx) } else { <-installTracker[podID] - scopedLog.Error(nil, "unable to get install worker context. app name %s", installWorker.appDeployInfo.AppName) + scopedLog.ErrorContext(ctx, "unable to get install worker context", "appName", installWorker.appDeployInfo.AppName) } } else { // This should never happen - scopedLog.Error(nil, "invalid worker reference") + scopedLog.ErrorContext(ctx, "invalid worker reference") } default: @@ -1290,7 +1283,7 @@ installHandler: if ctxt != nil { ctxt.runPlaybook(ctx) } else { - scopedLog.Error(nil, "unable to get the cluster scoped playbook context, kind: %s, name: %s", ppln.cr.GroupVersionKind().Kind, ppln.cr.GetName()) + scopedLog.ErrorContext(ctx, "unable to get the cluster scoped playbook context", "kind", ppln.cr.GroupVersionKind().Kind, "name", ppln.cr.GetName()) } } else { break @@ -1301,16 +1294,16 @@ installHandler: } // Wait for all the workers to finish - scopedLog.Info("Waiting for all the workers to finish") + scopedLog.InfoContext(ctx, "Waiting for all the workers to finish") pplnPhase.workerWaiter.Wait() - scopedLog.Info("All the workers finished") + scopedLog.InfoContext(ctx, "All the workers finished") } // installPhaseManager creates install phase manager for the afw installation pipeline func (ppln *AppInstallPipeline) installPhaseManager(ctx context.Context) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("installPhaseManager") - scopedLog.Info("Starting Install phase manager") + + scopedLog := logging.FromContext(ctx).With("func", "installPhaseManager") + scopedLog.InfoContext(ctx, "Starting Install phase manager") var handlerWaiter sync.WaitGroup @@ -1343,7 +1336,7 @@ installPhase: select { case _, channelOpen := <-ppln.sigTerm: if !channelOpen { - scopedLog.Info("Received the termination request from the scheduler") + scopedLog.InfoContext(ctx, "Received the termination request from the scheduler") break installPhase } @@ -1353,7 +1346,7 @@ installPhase: // Cluster scope has only bundle push no workers to install appScope := getAppSrcScope(ctx, installWorker.afwConfig, installWorker.appSrcName) if !canAppScopeHaveInstallWorker(appScope) { - scopedLog.Error(nil, "Install worker with incorrect scope", "name", installWorker.cr.GetName(), "namespace", installWorker.cr.GetNamespace(), "pod name", installWorker.targetPodName, "App name", installWorker.appDeployInfo.AppName, "digest", installWorker.appDeployInfo.ObjectHash, "scope", appScope) + scopedLog.ErrorContext(ctx, "Install worker with incorrect scope", "name", installWorker.cr.GetName(), "namespace", installWorker.cr.GetNamespace(), "pod name", installWorker.targetPodName, "App name", installWorker.appDeployInfo.AppName, "digest", installWorker.appDeployInfo.ObjectHash, "scope", appScope) continue } @@ -1363,7 +1356,7 @@ installPhase: // For fanout CRs, also update the main PhaseInfo to reflect the failure if isFanOutApplicableToCR(installWorker.cr) { - scopedLog.Info("Max retries reached for fanout CR - updating main phase info", "app", installWorker.appDeployInfo.AppName, "failCount", phaseInfo.FailCount) + scopedLog.InfoContext(ctx, "Max retries reached for fanout CR - updating main phase info", "app", installWorker.appDeployInfo.AppName, "failCount", phaseInfo.FailCount) installWorker.appDeployInfo.PhaseInfo.Phase = enterpriseApi.PhaseInstall installWorker.appDeployInfo.PhaseInfo.Status = enterpriseApi.AppPkgInstallError installWorker.appDeployInfo.DeployStatus = enterpriseApi.DeployStatusError @@ -1379,7 +1372,7 @@ installPhase: installWorker.waiter = &pplnPhase.workerWaiter select { case pplnPhase.msgChannel <- installWorker: - scopedLog.Info("Install worker got a run slot", "name", installWorker.cr.GetName(), "namespace", installWorker.cr.GetNamespace(), "pod name", installWorker.targetPodName, "App name", installWorker.appDeployInfo.AppName, "digest", installWorker.appDeployInfo.ObjectHash) + scopedLog.InfoContext(ctx, "Install worker got a run slot", "name", installWorker.cr.GetName(), "namespace", installWorker.cr.GetNamespace(), "pod name", installWorker.targetPodName, "App name", installWorker.appDeployInfo.AppName, "digest", installWorker.appDeployInfo.ObjectHash) // Always set the isActive in Phase manager itself, to avoid any delay in the install handler, otherwise it can // cause running the same playbook multiple times. @@ -1425,8 +1418,8 @@ func isPhaseStatusComplete(phaseInfo *enterpriseApi.PhaseInfo) bool { // validatePhaseInfo validates if phase and status in phaseInfo is valid func validatePhaseInfo(ctx context.Context, phaseInfo *enterpriseApi.PhaseInfo) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("validatePhaseInfo").WithValues("phaseInfo", phaseInfo) + + scopedLog := logging.FromContext(ctx).With("func", "validatePhaseInfo", "phaseInfo", phaseInfo) // Check for phase in phaseInfo phases := string( @@ -1435,12 +1428,12 @@ func validatePhaseInfo(ctx context.Context, phaseInfo *enterpriseApi.PhaseInfo) enterpriseApi.PhaseInstall) if !strings.Contains(phases, string(phaseInfo.Phase)) { - scopedLog.Error(nil, "Invalid phase in PhaseInfo", "phase", string(phaseInfo.Phase)) + scopedLog.ErrorContext(ctx, "Invalid phase in PhaseInfo", "phase", string(phaseInfo.Phase)) return false } if ok := appPhaseInfoStatuses[phaseInfo.Status]; !ok { - scopedLog.Error(nil, "Invalid status in PhaseInfo", "phase", string(phaseInfo.Phase), "status", phaseInfo.Status) + scopedLog.ErrorContext(ctx, "Invalid status in PhaseInfo", "phase", string(phaseInfo.Phase), "status", phaseInfo.Status) return false } return true @@ -1523,25 +1516,25 @@ func initAppInstallPipeline(ctx context.Context, appDeployContext *enterpriseApi // deleteAppPkgFromOperator removes the app pkg from the Operator Pod func deleteAppPkgFromOperator(ctx context.Context, worker *PipelineWorker) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("deleteAppPkgFromOperator").WithValues("name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "app pkg", worker.appDeployInfo.AppName) + + scopedLog := logging.FromContext(ctx).With("func", "deleteAppPkgFromOperator", "name", worker.cr.GetName(), "namespace", worker.cr.GetNamespace(), "app pkg", worker.appDeployInfo.AppName) appPkgLocalPath := getAppPackageLocalPath(ctx, worker) err := os.Remove(appPkgLocalPath) if err != nil { // Issue is local, so just log an error msg and return // ToDo: sgontla: For any transient errors, handle the clean-up at the end of the install - scopedLog.Error(err, "failed to delete app pkg from Operator", "app pkg path", appPkgLocalPath) + scopedLog.ErrorContext(ctx, "failed to delete app pkg from Operator", "app pkg path", appPkgLocalPath, "error", err) return } - scopedLog.Info("Deleted app package from the operator", "App package path", appPkgLocalPath) + scopedLog.InfoContext(ctx, "Deleted app package from the operator", "App package path", appPkgLocalPath) releaseStorage(worker.appDeployInfo.Size) } func afwGetReleventStatefulsetByKind(ctx context.Context, cr splcommon.MetaObject, client splcommon.ControllerClient) *appsv1.StatefulSet { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getReleventStatefulsetByKind").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "getReleventStatefulsetByKind", "name", cr.GetName(), "namespace", cr.GetNamespace()) var instanceID InstanceType switch cr.GetObjectKind().GroupVersionKind().Kind { @@ -1569,7 +1562,7 @@ func afwGetReleventStatefulsetByKind(ctx context.Context, cr splcommon.MetaObjec namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: statefulsetName} sts, err := splctrl.GetStatefulSetByName(ctx, client, namespacedName) if err != nil { - scopedLog.Error(err, "Unable to get the stateful set") + scopedLog.ErrorContext(ctx, "Unable to get the stateful set", "error", err) } return sts @@ -1621,8 +1614,8 @@ func getLocalScopePlaybookContext(ctx context.Context, installWorker *PipelineWo // getInsallWorkerPlaybookContext returns the playbook context for install workers i.e either local // or premiumApps scope for now func getInsallWorkerPlaybookContext(ctx context.Context, worker *PipelineWorker, sem chan struct{}, podExecClient splutil.PodExecClientImpl, appSrcSpec *enterpriseApi.AppSourceSpec, appSrcScope string, ppln *AppInstallPipeline) PlaybookImpl { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getInsallWorkerPlaybookContext").WithValues("crName", ppln.cr.GetName(), "namespace", ppln.cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "getInsallWorkerPlaybookContext", "crName", ppln.cr.GetName(), "namespace", ppln.cr.GetNamespace()) // Since local app context is needed for premiumAppContext we retrieve it for both cases localCtx := getLocalScopePlaybookContext(ctx, worker, sem, podExecClient) @@ -1633,7 +1626,7 @@ func getInsallWorkerPlaybookContext(ctx context.Context, worker *PipelineWorker, } // Invalid scope - scopedLog.Error(nil, "Install workers can have only local or premium apps scope", "appSrcScope", appSrcScope) + scopedLog.ErrorContext(ctx, "Install workers can have only local or premium apps scope", "appSrcScope", appSrcScope) return nil } @@ -1668,8 +1661,8 @@ func (shcPlaybookContext *SHCPlaybookContext) removeSHCBundlePushStatusFile(ctx // isBundlePushComplete checks whether the SHC bundle push is complete or still pending func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.Context) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isBundlePushComplete").WithValues("crName", shcPlaybookContext.cr.GetName(), "namespace", shcPlaybookContext.cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "isBundlePushComplete", "crName", shcPlaybookContext.cr.GetName(), "namespace", shcPlaybookContext.cr.GetNamespace()) cmd := fmt.Sprintf("cat %s", shcBundlePushStatusCheckFile) streamOptions := splutil.NewStreamOptionsObject(cmd) @@ -1692,12 +1685,12 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C // 1. stdOut is empty, which means bundle push is still in progress // 2. stdOut has some other string other than the bundle push success message if stdOut == "" { - scopedLog.Info("SHC Bundle Push is still in progress") + scopedLog.InfoContext(ctx, "SHC Bundle Push is still in progress") return false, nil } else if !strings.Contains(stdOut, shcBundlePushCompleteStr) { // this means there was an error in bundle push command err = fmt.Errorf("there was an error in applying SHC Bundle, err=\"%v\"", stdOut) - scopedLog.Error(err, "SHC Bundle push status file reported an error while applying bundle") + scopedLog.ErrorContext(ctx, "SHC Bundle push status file reported an error while applying bundle", "error", err) // reset the bundle push state to Pending, so that we retry again. setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushPending) @@ -1713,7 +1706,7 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C // now that bundle push is complete, remove the status file err = shcPlaybookContext.removeSHCBundlePushStatusFile(ctx) if err != nil { - scopedLog.Error(err, "removing SHC Bundle Push status file failed, will retry again.") + scopedLog.ErrorContext(ctx, "removing SHC Bundle Push status file failed, will retry again.", "error", err) // reset the state to BundlePushInProgress so that we can check the status of file again. setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushInProgress) @@ -1727,8 +1720,8 @@ func (shcPlaybookContext *SHCPlaybookContext) isBundlePushComplete(ctx context.C // triggerBundlePush triggers the bundle push operation for SHC func (shcPlaybookContext *SHCPlaybookContext) triggerBundlePush(ctx context.Context) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("shcPlaybookContext.triggerBundlePush()").WithValues( + + scopedLog := logging.FromContext(ctx).With("func", "shcPlaybookContext.triggerBundlePush", "shcCaptainUrl", shcPlaybookContext.searchHeadCaptainURL, "cr", shcPlaybookContext.cr.GetName()) @@ -1737,7 +1730,7 @@ func (shcPlaybookContext *SHCPlaybookContext) triggerBundlePush(ctx context.Cont // Trigger bundle push cmd := fmt.Sprintf(applySHCBundleCmdStr, shcPlaybookContext.searchHeadCaptainURL, shcBundlePushStatusCheckFile) - scopedLog.Info("Triggering bundle push", "command", cmd) + scopedLog.InfoContext(ctx, "Triggering bundle push", "command", cmd) streamOptions := splutil.NewStreamOptionsObject(cmd) stdOut, stdErr, err := shcPlaybookContext.podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if err != nil || stdErr != "" { @@ -1749,14 +1742,14 @@ func (shcPlaybookContext *SHCPlaybookContext) triggerBundlePush(ctx context.Cont // setLivenessProbeLevel sets the liveness probe level across all the Search Head Pods. func (shcPlaybookContext *SHCPlaybookContext) setLivenessProbeLevel(ctx context.Context, probeLevel int) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("shcPlaybookContext.setLivenessProbeLevel()") + + scopedLog := logging.FromContext(ctx).With("func", "shcPlaybookContext.setLivenessProbeLevel") shcStsName := GetSplunkStatefulsetName(SplunkSearchHead, shcPlaybookContext.cr.GetName()) shcStsNamespaceName := types.NamespacedName{Namespace: shcPlaybookContext.cr.GetNamespace(), Name: shcStsName} shcSts, err := splctrl.GetStatefulSetByName(ctx, shcPlaybookContext.client, shcStsNamespaceName) if err != nil { - scopedLog.Error(err, "Unable to get the stateful set") + scopedLog.ErrorContext(ctx, "Unable to get the stateful set", "error", err) return err } @@ -1774,7 +1767,7 @@ func (shcPlaybookContext *SHCPlaybookContext) setLivenessProbeLevel(ctx context. err = setProbeLevelOnCRPods(ctx, shcPlaybookContext.cr, *shcSts.Spec.Replicas, shcPlaybookContext.podExecClient, probeLevel) if err != nil { - scopedLog.Error(err, "Unable to set the Liveness probe level") + scopedLog.ErrorContext(ctx, "Unable to set the Liveness probe level", "error", err) return err } @@ -1815,8 +1808,8 @@ func adjustClusterAppsFilePermissions(ctx context.Context, podExecClient splutil // runPlaybook will implement the bundle push logic for SHC func (shcPlaybookContext *SHCPlaybookContext) runPlaybook(ctx context.Context) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("runPlaybook").WithValues("crName", shcPlaybookContext.cr.GetName(), "namespace", shcPlaybookContext.cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "runPlaybook", "crName", shcPlaybookContext.cr.GetName(), "namespace", shcPlaybookContext.cr.GetNamespace()) var err error var ok bool @@ -1825,7 +1818,7 @@ func (shcPlaybookContext *SHCPlaybookContext) runPlaybook(ctx context.Context) e return nil } if cr.Status.Phase != enterpriseApi.PhaseReady { - scopedLog.Info("SHC is not ready yet.") + scopedLog.InfoContext(ctx, "SHC is not ready yet.") return nil } @@ -1834,11 +1827,11 @@ func (shcPlaybookContext *SHCPlaybookContext) runPlaybook(ctx context.Context) e switch appDeployContext.BundlePushStatus.BundlePushStage { // if the bundle push is already in progress, check the status case enterpriseApi.BundlePushInProgress: - scopedLog.Info("checking the status of SHC Bundle Push") + scopedLog.InfoContext(ctx, "checking the status of SHC Bundle Push") // check if the bundle push is complete ok, err = shcPlaybookContext.isBundlePushComplete(ctx) if ok { - scopedLog.Info("Bundle push complete, setting bundle push state in CR") + scopedLog.InfoContext(ctx, "Bundle push complete, setting bundle push state in CR") // set the bundle push status to complete setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushComplete) @@ -1852,28 +1845,28 @@ func (shcPlaybookContext *SHCPlaybookContext) runPlaybook(ctx context.Context) e // set the liveness probe to default shcPlaybookContext.setLivenessProbeLevel(ctx, livenessProbeLevelDefault) } else if err != nil { - scopedLog.Error(err, "there was an error in SHC bundle push, will retry again") + scopedLog.ErrorContext(ctx, "there was an error in SHC bundle push, will retry again", "error", err) } else { - scopedLog.Info("SHC Bundle Push is still in progress, will check back again") + scopedLog.InfoContext(ctx, "SHC Bundle Push is still in progress, will check back again") } case enterpriseApi.BundlePushPending: // run the command to apply cluster bundle - scopedLog.Info("running command to apply SHC Bundle") + scopedLog.InfoContext(ctx, "running command to apply SHC Bundle") // Adjust the file permissions err = adjustClusterAppsFilePermissions(ctx, shcPlaybookContext.podExecClient) if err != nil { - scopedLog.Error(err, "failed to adjust the file permissions") + scopedLog.ErrorContext(ctx, "failed to adjust the file permissions", "error", err) return err } err = shcPlaybookContext.triggerBundlePush(ctx) if err != nil { - scopedLog.Error(err, "failed to apply SHC Bundle") + scopedLog.ErrorContext(ctx, "failed to apply SHC Bundle", "error", err) return err } - scopedLog.Info("SHC Bundle Push is in progress") + scopedLog.InfoContext(ctx, "SHC Bundle Push is in progress") // set the state to bundle push complete since SHC bundle push is a sync call setBundlePushState(ctx, shcPlaybookContext.afwPipeline, enterpriseApi.BundlePushInProgress) @@ -1886,29 +1879,29 @@ func (shcPlaybookContext *SHCPlaybookContext) runPlaybook(ctx context.Context) e // isBundlePushComplete checks the status of bundle push func (idxcPlaybookContext *IdxcPlaybookContext) isBundlePushComplete(ctx context.Context) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isBundlePushComplete").WithValues("crName", idxcPlaybookContext.cr.GetName(), "namespace", idxcPlaybookContext.cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "isBundlePushComplete", "crName", idxcPlaybookContext.cr.GetName(), "namespace", idxcPlaybookContext.cr.GetNamespace()) streamOptions := splutil.NewStreamOptionsObject(idxcShowClusterBundleStatusStr) stdOut, stdErr, err := idxcPlaybookContext.podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if err == nil && strings.Contains(stdOut, "cluster_status=None") && !strings.Contains(stdOut, "last_bundle_validation_status=failure") { - scopedLog.Info("IndexerCluster Bundle push complete") + scopedLog.InfoContext(ctx, "IndexerCluster Bundle push complete") return true } if err != nil || stdErr != "" { - scopedLog.Error(err, "show cluster-bundle-status failed", "stdout", stdOut, "stderr", stdErr) + scopedLog.ErrorContext(ctx, "show cluster-bundle-status failed", "stdout", stdOut, "stderr", stdErr, "error", err) return false } - scopedLog.Info("IndexerCluster Bundle push is still in progress") + scopedLog.InfoContext(ctx, "IndexerCluster Bundle push is still in progress") return false } // triggerBundlePush triggers the bundle push for indexer cluster func (idxcPlaybookContext *IdxcPlaybookContext) triggerBundlePush(ctx context.Context) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("idxcPlaybookContext.triggerBundlePush()") + + scopedLog := logging.FromContext(ctx).With("func", "idxcPlaybookContext.triggerBundlePush") // Reduce the liveness probe level idxcPlaybookContext.setLivenessProbeLevel(ctx, livenessProbeLevelOne) @@ -1918,7 +1911,7 @@ func (idxcPlaybookContext *IdxcPlaybookContext) triggerBundlePush(ctx context.Co // If the error is due to a bundle which is already present, don't do anything. // In the next reconcile we will mark it as bundle push complete if strings.Contains(stdErr, idxcBundleAlreadyPresentStr) { - scopedLog.Info("bundle already present on peers") + scopedLog.InfoContext(ctx, "bundle already present on peers") } else if err != nil || !strings.Contains(stdErr, "OK\n") { err = fmt.Errorf("error while applying cluster bundle. stdout: %s, stderr: %s, err: %v", stdOut, stdErr, err) return err @@ -1929,8 +1922,8 @@ func (idxcPlaybookContext *IdxcPlaybookContext) triggerBundlePush(ctx context.Co // setLivenessProbeLevel sets the liveness probe level across all the indexer pods func (idxcPlaybookContext *IdxcPlaybookContext) setLivenessProbeLevel(ctx context.Context, probeLevel int) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("idxcPlaybookContext.setLivenessProbeLevel()") + + scopedLog := logging.FromContext(ctx).With("func", "idxcPlaybookContext.setLivenessProbeLevel") var err error managerSts := afwGetReleventStatefulsetByKind(ctx, idxcPlaybookContext.cr, idxcPlaybookContext.client) @@ -1962,7 +1955,7 @@ func (idxcPlaybookContext *IdxcPlaybookContext) setLivenessProbeLevel(ctx contex err = idxcPlaybookContext.client.Get(ctx, idxcNameSpaceName, &idxcCR) if err != nil { // Probably a dangling owner reference, just ignore and continue - scopedLog.Error(err, "Unable to fetch the CR", "Name", managerOwnerRefs[i].Name, "Namespace", idxcPlaybookContext.cr.GetNamespace()) + scopedLog.ErrorContext(ctx, "Unable to fetch the CR", "Name", managerOwnerRefs[i].Name, "Namespace", idxcPlaybookContext.cr.GetNamespace(), "error", err) continue } @@ -1970,14 +1963,14 @@ func (idxcPlaybookContext *IdxcPlaybookContext) setLivenessProbeLevel(ctx contex idxcStsNamespaceName := types.NamespacedName{Namespace: idxcCR.GetNamespace(), Name: idxcStsName} idxcSts, err := splctrl.GetStatefulSetByName(ctx, idxcPlaybookContext.client, idxcStsNamespaceName) if err != nil { - scopedLog.Error(err, "Unable to get the stateful set") + scopedLog.ErrorContext(ctx, "Unable to get the stateful set", "error", err) // Probably a dangling owner reference, just ignore and continue continue } err = setProbeLevelOnCRPods(ctx, &idxcCR, *idxcSts.Spec.Replicas, idxcPlaybookContext.podExecClient, probeLevel) if err != nil { - scopedLog.Error(err, "Unable to set the Liveness probe level") + scopedLog.ErrorContext(ctx, "Unable to set the Liveness probe level", "error", err) return err } } @@ -1992,15 +1985,14 @@ func (idxcPlaybookContext *IdxcPlaybookContext) setLivenessProbeLevel(ctx contex // 2. OR else, if the bundle push is already in progress, check the status of bundle push func (idxcPlaybookContext *IdxcPlaybookContext) runPlaybook(ctx context.Context) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("RunPlaybook").WithValues("crName", idxcPlaybookContext.cr.GetName(), "namespace", idxcPlaybookContext.cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "RunPlaybook", "crName", idxcPlaybookContext.cr.GetName(), "namespace", idxcPlaybookContext.cr.GetNamespace()) appDeployContext := idxcPlaybookContext.afwPipeline.appDeployContext switch appDeployContext.BundlePushStatus.BundlePushStage { // if the bundle push is already in progress, check the status case enterpriseApi.BundlePushInProgress: - scopedLog.Info("checking the status of IndexerCluster Bundle Push") + scopedLog.InfoContext(ctx, "checking the status of IndexerCluster Bundle Push") // check if the bundle push is complete if idxcPlaybookContext.isBundlePushComplete(ctx) { // set the bundle push status to complete @@ -2013,22 +2005,22 @@ func (idxcPlaybookContext *IdxcPlaybookContext) runPlaybook(ctx context.Context) setInstallStateForClusterScopedApps(ctx, appDeployContext) idxcPlaybookContext.setLivenessProbeLevel(ctx, livenessProbeLevelDefault) } else { - scopedLog.Info("IndexerCluster Bundle Push is still in progress, will check back again in next reconcile..") + scopedLog.InfoContext(ctx, "IndexerCluster Bundle Push is still in progress, will check back again in next reconcile..") } case enterpriseApi.BundlePushPending: // Adjust the file permissions err := adjustClusterAppsFilePermissions(ctx, idxcPlaybookContext.podExecClient) if err != nil { - scopedLog.Error(err, "failed to adjust the file permissions") + scopedLog.ErrorContext(ctx, "failed to adjust the file permissions", "error", err) return err } // run the command to apply cluster bundle - scopedLog.Info("running command to apply IndexerCluster Bundle") + scopedLog.InfoContext(ctx, "running command to apply IndexerCluster Bundle") err = idxcPlaybookContext.triggerBundlePush(ctx) if err != nil { - scopedLog.Error(err, "failed to apply IndexerCluster Bundle") + scopedLog.ErrorContext(ctx, "failed to apply IndexerCluster Bundle", "error", err) return err } @@ -2061,8 +2053,7 @@ func handleEsappPostinstall(rctx context.Context, preCtx *premiumAppScopePlayboo cr := preCtx.cr appSrcSpec := preCtx.appSrcSpec - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("handleEsappPostinstall").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) + scopedLog := logging.FromContext(rctx).With("func", "handleEsappPostinstall", "name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) // For ES app, run post-install commands var command string @@ -2080,7 +2071,7 @@ func handleEsappPostinstall(rctx context.Context, preCtx *premiumAppScopePlayboo stdOut, stdErr, err := preCtx.localCtx.podExecClient.RunPodExecCommand(rctx, streamOptions, []string{"/bin/sh"}) if stdErr != "" || err != nil { phaseInfo.FailCount++ - scopedLog.Error(err, "premium scoped app package install failed", "stdout", stdOut, "stderr", stdErr, "post install command", command, "failCount", phaseInfo.FailCount) + scopedLog.ErrorContext(rctx, "premium scoped app package install failed", "stdout", stdOut, "stderr", stdErr, "post install command", command, "failCount", phaseInfo.FailCount, "error", err) return fmt.Errorf("premium scoped app package install failed. stdOut: %s, stdErr: %s, post install command: %s, failCount: %d", stdOut, stdErr, command, phaseInfo.FailCount) } @@ -2097,8 +2088,7 @@ func (preCtx *premiumAppScopePlaybookContext) runPlaybook(rctx context.Context) worker := preCtx.localCtx.worker appSrcSpec := preCtx.appSrcSpec - reqLogger := log.FromContext(rctx) - scopedLog := reqLogger.WithName("premiumAppScopePlaybookContext.runPlaybook").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) + scopedLog := logging.FromContext(rctx).With("func", "premiumAppScopePlaybookContext.runPlaybook", "name", cr.GetName(), "namespace", cr.GetNamespace(), "pod", worker.targetPodName, "app name", worker.appDeployInfo.AppName) defer func() { <-preCtx.localCtx.sem @@ -2112,7 +2102,7 @@ func (preCtx *premiumAppScopePlaybookContext) runPlaybook(rctx context.Context) // Call the API to install an app err := installApp(rctx, preCtx.localCtx, cr, phaseInfo) if err != nil { - scopedLog.Error(err, "premium app package installation error") + scopedLog.ErrorContext(rctx, "premium app package installation error", "error", err) return fmt.Errorf("app pkg installation failed. error %s", err.Error()) } @@ -2120,7 +2110,7 @@ func (preCtx *premiumAppScopePlaybookContext) runPlaybook(rctx context.Context) if appSrcSpec.PremiumAppsProps.Type == enterpriseApi.PremiumAppsTypeEs { err = handleEsappPostinstall(rctx, preCtx, phaseInfo) if err != nil { - scopedLog.Error(err, "app package post installation error") + scopedLog.ErrorContext(rctx, "app package post installation error", "error", err) return fmt.Errorf("app pkg post installation failed. error %s", err.Error()) } } @@ -2131,7 +2121,7 @@ func (preCtx *premiumAppScopePlaybookContext) runPlaybook(rctx context.Context) // Call the API to clean up app err = cleanupApp(rctx, preCtx.localCtx, cr, phaseInfo) if err != nil { - scopedLog.Error(err, "premium app package installation error") + scopedLog.ErrorContext(rctx, "premium app package installation error", "error", err) return fmt.Errorf("app pkg installation failed. error %s", err.Error()) } @@ -2182,8 +2172,8 @@ func isPhaseInfoEligibleForSchedulerEntry(ctx context.Context, appSrcName string // afwSchedulerEntry Starts the scheduler Pipeline with the required phases func afwSchedulerEntry(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appDeployContext *enterpriseApi.AppDeploymentContext, appFrameworkConfig *enterpriseApi.AppFrameworkSpec) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("afwSchedulerEntry").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "afwSchedulerEntry", "name", cr.GetName(), "namespace", cr.GetNamespace()) // return error, if there is no storage defined for the Operator pod if !isPersistentVolConfigured() { @@ -2211,7 +2201,7 @@ func afwSchedulerEntry(ctx context.Context, client splcommon.ControllerClient, c afwPipeline.phaseWaiter.Add(1) go afwPipeline.installPhaseManager(ctx) - scopedLog.Info("Creating pipeline workers for pending app packages") + scopedLog.InfoContext(ctx, "Creating pipeline workers for pending app packages") for appSrcName, appSrcDeployInfo := range appDeployContext.AppsSrcDeployStatus { @@ -2226,7 +2216,7 @@ func afwSchedulerEntry(ctx context.Context, client splcommon.ControllerClient, c // create the dir on Splunk pod/s where app/s will be copied from operator pod err = createDirOnSplunkPods(ctx, cr, *sts.Spec.Replicas, appsPathOnPod, podExecClient) if err != nil { - scopedLog.Error(err, "unable to create directory on splunk pod") + scopedLog.ErrorContext(ctx, "unable to create directory on splunk pod", "error", err) // break from here and let yield logic take care of everything break } @@ -2247,11 +2237,11 @@ func afwSchedulerEntry(ctx context.Context, client splcommon.ControllerClient, c afwPipeline.phaseWaiter.Add(1) go afwPipeline.afwYieldWatcher(ctx) - scopedLog.Info("Waiting for the phase managers to finish") + scopedLog.InfoContext(ctx, "Waiting for the phase managers to finish") // Wait for all the pipeline managers to finish afwPipeline.phaseWaiter.Wait() - scopedLog.Info("All the phase managers finished") + scopedLog.InfoContext(ctx, "All the phase managers finished") // Finally mark if all the App framework is complete checkAndUpdateAppFrameworkProgressFlag(afwPipeline) @@ -2261,15 +2251,15 @@ func afwSchedulerEntry(ctx context.Context, client splcommon.ControllerClient, c // afwYieldWatcher issues termination request to the scheduler when the yield time expires or the pipelines become empty. func (ppln *AppInstallPipeline) afwYieldWatcher(ctx context.Context) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("afwYieldWatcher").WithValues("name", ppln.cr.GetName(), "namespace", ppln.cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "afwYieldWatcher", "name", ppln.cr.GetName(), "namespace", ppln.cr.GetNamespace()) yieldTrigger := time.After(time.Duration(ppln.appDeployContext.AppFrameworkConfig.SchedulerYieldInterval) * time.Second) yieldScheduler: for { select { case <-yieldTrigger: - scopedLog.Info("Yielding from AFW scheduler", "time elapsed", time.Now().Unix()-ppln.afwEntryTime) + scopedLog.InfoContext(ctx, "Yielding from AFW scheduler", "time elapsed", time.Now().Unix()-ppln.afwEntryTime) break yieldScheduler default: if ppln.isPipelineEmpty() { @@ -2283,5 +2273,5 @@ yieldScheduler: // Trigger the pipeline termination by closing the channel close(ppln.sigTerm) ppln.phaseWaiter.Done() - scopedLog.Info("Termination issued") + scopedLog.InfoContext(ctx, "Termination issued") } diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index 1dc4af60c..576210d6f 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -46,10 +46,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -139,8 +139,7 @@ func updateStorageTracker(ctx context.Context) error { // GetRemoteStorageClient returns the corresponding RemoteDataClient func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appFrameworkRef *enterpriseApi.AppFrameworkSpec, vol *enterpriseApi.VolumeSpec, location string, fn splclient.GetInitFunc) (splclient.SplunkRemoteDataClient, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetRemoteStorageClient").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "GetRemoteStorageClient", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -155,7 +154,7 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie var secretAccessKey string if appSecretRef == "" { // No secretRef means we should try to use the credentials available in the pod already via kube2iam or something similar - scopedLog.Info("No secrectRef provided. Attempt to access remote storage client without access/secret keys") + scopedLog.InfoContext(ctx, "No secrectRef provided. Attempt to access remote storage client without access/secret keys") accessKeyID = "" secretAccessKey = "" } else { @@ -211,14 +210,14 @@ func GetRemoteStorageClient(ctx context.Context, client splcommon.ControllerClie // Ex. ("a/b" + "c"), ("a/b/" + "c"), ("a/b/" + "/c"), ("a/b/" + "/c"), ("a/b//", + "c/././") ("a/b/../b", + "c/../c") all are joined as "a/b/c" prefix := filepath.Join(basePrefix, location) + "/" - scopedLog.Info("Creating the client", "volume", vol.Name, "bucket", bucket, "bucket path", prefix) + scopedLog.InfoContext(ctx, "Creating the client", "volume", vol.Name, "bucket", bucket, "bucket path", prefix) var err error remoteDataClient.Client, err = getClient(ctx, bucket, accessKeyID, secretAccessKey, prefix, prefix /* startAfter*/, vol.Region, vol.Endpoint, fn) if err != nil { - scopedLog.Error(err, "Failed to get the S3 client") + scopedLog.ErrorContext(ctx, "Failed to get the S3 client", "error", err) // Emit event when operator cannot connect to the remote app repository if eventPublisher != nil { eventPublisher.Warning(ctx, EventReasonAppRepoConnFailed, @@ -265,8 +264,8 @@ func ApplySplunkConfig(ctx context.Context, client splcommon.ControllerClient, c // ReconcileCRSpecificConfigMap reconciles CR Specific config map exists and contains the ManualUpdate field set to "off" func ReconcileCRSpecificConfigMap(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ReconcileCRSpecificConfigMap").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "ReconcileCRSpecificConfigMap", "name", cr.GetName(), "namespace", cr.GetNamespace()) a := cr.GetObjectKind() b := a.GroupVersionKind() c := b.Kind @@ -291,13 +290,13 @@ func ReconcileCRSpecificConfigMap(ctx context.Context, client splcommon.Controll configMap.SetOwnerReferences(append(configMap.GetOwnerReferences(), splcommon.AsOwner(cr, true))) err = client.Create(ctx, configMap) if err != nil { - scopedLog.Error(err, "Failed to create config map") + scopedLog.ErrorContext(ctx, "Failed to create config map", "error", err) return err } - scopedLog.Info("Created new config map with ManualUpdate set to 'on'") + scopedLog.InfoContext(ctx, "Created new config map with ManualUpdate set to 'on'") return nil } - scopedLog.Error(err, "Failed to get config map") + scopedLog.ErrorContext(ctx, "Failed to get config map", "error", err) return err } @@ -306,10 +305,10 @@ func ReconcileCRSpecificConfigMap(ctx context.Context, client splcommon.Controll configMap.Data["manualUpdate"] = "off" err = client.Update(ctx, configMap) if err != nil { - scopedLog.Error(err, "Failed to update config map with manualUpdate field") + scopedLog.ErrorContext(ctx, "Failed to update config map with manualUpdate field", "error", err) return err } - scopedLog.Info("Updated config map with manualUpdate set to 'on'") + scopedLog.InfoContext(ctx, "Updated config map with manualUpdate set to 'on'") } return nil @@ -530,10 +529,10 @@ func bundlePushStateAsStr(ctx context.Context, state enterpriseApi.BundlePushSta // setBundlePushState sets the bundle push state to the new state func setBundlePushState(ctx context.Context, afwPipeline *AppInstallPipeline, state enterpriseApi.BundlePushStageType) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("setBundlePushState") - scopedLog.Info("Setting the bundle push state", "old state", bundlePushStateAsStr(ctx, afwPipeline.appDeployContext.BundlePushStatus.BundlePushStage), "new state", bundlePushStateAsStr(ctx, state)) + scopedLog := logging.FromContext(ctx).With("func", "setBundlePushState") + + scopedLog.InfoContext(ctx, "Setting the bundle push state", "old state", bundlePushStateAsStr(ctx, afwPipeline.appDeployContext.BundlePushStatus.BundlePushStage), "new state", bundlePushStateAsStr(ctx, state)) afwPipeline.appDeployContext.BundlePushStatus.BundlePushStage = state } @@ -544,13 +543,13 @@ func getBundlePushState(afwPipeline *AppInstallPipeline) enterpriseApi.BundlePus // createAppDownloadDir creates the app download directory on the operator pod func createAppDownloadDir(ctx context.Context, path string) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("createAppDownloadDir").WithValues("path", path) + + scopedLog := logging.FromContext(ctx).With("func", "createAppDownloadDir", "path", path) _, err := os.Stat(path) if errors.Is(err, os.ErrNotExist) { errDir := os.MkdirAll(path, 0700) if errDir != nil { - scopedLog.Error(errDir, "Unable to create directory at path") + scopedLog.ErrorContext(ctx, "Unable to create directory at path", "error", errDir) return errDir } } @@ -561,16 +560,16 @@ func createAppDownloadDir(ctx context.Context, path string) error { func getAvailableDiskSpace(ctx context.Context) (uint64, error) { var availDiskSpace uint64 var stat syscall.Statfs_t - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getAvailableDiskSpace").WithValues("volume mount", splcommon.AppDownloadVolume) + + scopedLog := logging.FromContext(ctx).With("func", "getAvailableDiskSpace", "volume mount", splcommon.AppDownloadVolume) err := syscall.Statfs(splcommon.AppDownloadVolume, &stat) if err != nil { - scopedLog.Error(err, "There is no default volume configured for the App framework, use the temporary location", "dir", TmpAppDownloadDir) + scopedLog.ErrorContext(ctx, "There is no default volume configured for the App framework, use the temporary location", "dir", TmpAppDownloadDir, "error", err) splcommon.AppDownloadVolume = TmpAppDownloadDir err = os.MkdirAll(splcommon.AppDownloadVolume, 0700) if err != nil { - scopedLog.Error(err, "Unable to create the directory", "dir", splcommon.AppDownloadVolume) + scopedLog.ErrorContext(ctx, "Unable to create the directory", "dir", splcommon.AppDownloadVolume, "error", err) return 0, err } } @@ -581,7 +580,7 @@ func getAvailableDiskSpace(ctx context.Context) (uint64, error) { } availDiskSpace = stat.Bavail * uint64(stat.Bsize) - scopedLog.Info("current available disk space in GB", "availableDiskSpace(GB)", availDiskSpace/1024/1024/1024) + scopedLog.InfoContext(ctx, "current available disk space in GB", "availableDiskSpace(GB)", availDiskSpace/1024/1024/1024) return availDiskSpace, err } @@ -591,18 +590,17 @@ func getRemoteObjectKey(ctx context.Context, cr splcommon.MetaObject, appFramewo var remoteObjectKey string var vol enterpriseApi.VolumeSpec - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getRemoteObjectKey").WithValues("crName", cr.GetName(), "namespace", cr.GetNamespace(), "appSrcName", appSrcName, "appName", appName) + scopedLog := logging.FromContext(ctx).With("func", "getRemoteObjectKey", "crName", cr.GetName(), "namespace", cr.GetNamespace(), "appSrcName", appSrcName, "appName", appName) appSrc, err := getAppSrcSpec(appFrameworkConfig.AppSources, appSrcName) if err != nil { - scopedLog.Error(err, "unable to get appSourceSpec") + scopedLog.ErrorContext(ctx, "unable to get appSourceSpec", "error", err) return remoteObjectKey, err } vol, err = splclient.GetAppSrcVolume(ctx, *appSrc, appFrameworkConfig) if err != nil { - scopedLog.Error(err, "unable to get volume spec") + scopedLog.ErrorContext(ctx, "unable to get volume spec", "error", err) return remoteObjectKey, err } @@ -624,18 +622,18 @@ func getRemoteObjectKey(ctx context.Context, cr splcommon.MetaObject, appFramewo // getRemoteDataClientMgr gets the RemoteDataClientMgr instance to download apps func getRemoteDataClientMgr(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appFrameworkConfig *enterpriseApi.AppFrameworkSpec, appSrcName string) (*RemoteDataClientManager, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("RemoteDataClientMgr").WithValues("crName", cr.GetName(), "namespace", cr.GetNamespace(), "appSrcName", appSrcName) + + scopedLog := logging.FromContext(ctx).With("func", "RemoteDataClientMgr", "crName", cr.GetName(), "namespace", cr.GetNamespace(), "appSrcName", appSrcName) var vol enterpriseApi.VolumeSpec appSrc, err := getAppSrcSpec(appFrameworkConfig.AppSources, appSrcName) if err != nil { - scopedLog.Error(err, "unable to get appSrcSpc") + scopedLog.ErrorContext(ctx, "unable to get appSrcSpc", "error", err) return nil, err } vol, err = splclient.GetAppSrcVolume(ctx, *appSrc, appFrameworkConfig) if err != nil { - scopedLog.Error(err, "unable to get volume spec") + scopedLog.ErrorContext(ctx, "unable to get volume spec", "error", err) return nil, err } @@ -682,8 +680,7 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl var configMapDataChanged bool crKind = cr.GetObjectKind().GroupVersionKind().Kind - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplySmartStoreConfigMap").WithValues("kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "ApplySmartStoreConfigMap", "kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) // 1. Prepare the indexes.conf entries mapSplunkConfDetails := make(map[string]string) @@ -695,14 +692,14 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl } if volumesConfIni == "" { - scopedLog.Info("Volume stanza list is empty") + scopedLog.InfoContext(ctx, "Volume stanza list is empty") } // Get the list of indexes in INI format indexesConfIni := GetSmartstoreIndexesConfig(smartstore.IndexList) if indexesConfIni == "" { - scopedLog.Info("Index stanza list is empty") + scopedLog.InfoContext(ctx, "Index stanza list is empty") } else if volumesConfIni == "" { return nil, configMapDataChanged, fmt.Errorf("indexes without Volume configuration is not allowed") } @@ -733,7 +730,7 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl configMapDataChanged, err = splctrl.ApplyConfigMap(ctx, client, SplunkOperatorAppConfigMap) if err != nil { - scopedLog.Error(err, "config map create/update failed", "error", err.Error()) + scopedLog.ErrorContext(ctx, "config map create/update failed", "error", err) return nil, configMapDataChanged, err } else if configMapDataChanged { // Create a token to check if the config is really populated to the pod @@ -753,7 +750,7 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl } } if err != nil { - scopedLog.Error(err, "config map update failed", "error", err.Error()) + scopedLog.ErrorContext(ctx, "config map update failed", "error", err) return nil, configMapDataChanged, err } } @@ -766,8 +763,8 @@ func ApplySmartstoreConfigMap(ctx context.Context, client splcommon.ControllerCl // are wiped out due to bundle push and hence we need to reinstate it in case of any smartstore changes. var resetSymbolicLinks = func(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, replicas int32, podExecClient splutil.PodExecClientImpl) error { crKind := cr.GetObjectKind().GroupVersionKind().Kind - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ResetSymbolicLinks").WithValues("kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "ResetSymbolicLinks", "kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) // Create command for symbolic link creation var command string @@ -780,11 +777,11 @@ var resetSymbolicLinks = func(ctx context.Context, client splcommon.ControllerCl // Run the commands on Splunk pods err := runCustomCommandOnSplunkPods(ctx, cr, replicas, command, podExecClient) if err != nil { - scopedLog.Error(err, "unable to run command on splunk pod") + scopedLog.ErrorContext(ctx, "unable to run command on splunk pod", "error", err) return err } - scopedLog.Info("Reset symbolic links successfully") + scopedLog.InfoContext(ctx, "Reset symbolic links successfully") // All good return nil @@ -849,14 +846,14 @@ func setupInitContainer(podTemplateSpec *corev1.PodTemplateSpec, Image string, i // Ideally we should be removing the owner reference wherever the CR is not controller for the resource func DeleteOwnerReferencesForResources(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DeleteOwnerReferencesForResources").WithValues("kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "DeleteOwnerReferencesForResources", "kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) // Delete references to Default secret object defaultSecretName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) _, err = splutil.RemoveSecretOwnerRef(ctx, client, defaultSecretName, cr) if err != nil { - scopedLog.Error(err, fmt.Sprintf("Owner reference removal failed for Secret Object %s", defaultSecretName)) + scopedLog.ErrorContext(ctx, fmt.Sprintf("Owner reference removal failed for Secret Object %s", defaultSecretName), "error", err) return err } @@ -875,7 +872,7 @@ func DeleteOwnerReferencesForResources(ctx context.Context, client splcommon.Con namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(instanceType, cr.GetName())} err = splctrl.RemoveUnwantedOwnerRefSs(ctx, client, namespacedName, cr) if err != nil { - scopedLog.Error(err, "Owner Reference removal failed for statefulSet") + scopedLog.ErrorContext(ctx, "Owner Reference removal failed for statefulSet", "error", err) return err } @@ -885,8 +882,8 @@ func DeleteOwnerReferencesForResources(ctx context.Context, client splcommon.Con // DeleteOwnerReferencesForS3SecretObjects deletes owner references for all the secret objects referred by smartstore // remote volume end points func DeleteOwnerReferencesForS3SecretObjects(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, smartstore *enterpriseApi.SmartStoreSpec) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DeleteOwnerReferencesForS3SecretObjects").WithValues("kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "DeleteOwnerReferencesForS3SecretObjects", "kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) var err error = nil if !isSmartstoreConfigured(smartstore) { @@ -898,9 +895,9 @@ func DeleteOwnerReferencesForS3SecretObjects(ctx context.Context, client splcomm if volume.SecretRef != "" && volume.SecretRef != splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) { _, err = splutil.RemoveSecretOwnerRef(ctx, client, volume.SecretRef, cr) if err == nil { - scopedLog.Info("Removed references for Secret Object", "secret", volume.SecretRef) + scopedLog.InfoContext(ctx, "Removed references for Secret Object", "secret", volume.SecretRef) } else { - scopedLog.Error(err, fmt.Sprintf("Owner reference removal failed for Secret Object %s", volume.SecretRef)) + scopedLog.ErrorContext(ctx, fmt.Sprintf("Owner reference removal failed for Secret Object %s", volume.SecretRef), "error", err) } } } @@ -967,12 +964,11 @@ var GetAppsList = func(ctx context.Context, RemoteDataClientMgr RemoteDataClient // GetAppListFromRemoteBucket gets the list of apps from remote storage. func GetAppListFromRemoteBucket(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appFrameworkRef *enterpriseApi.AppFrameworkSpec) (map[string]splclient.RemoteDataListResponse, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetAppListFromRemoteBucket").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "GetAppListFromRemoteBucket", "name", cr.GetName(), "namespace", cr.GetNamespace()) sourceToAppListMap := make(map[string]splclient.RemoteDataListResponse) - scopedLog.Info("Getting the list of apps from remote storage...") + scopedLog.InfoContext(ctx, "Getting the list of apps from remote storage...") var remoteDataListResponse splclient.RemoteDataListResponse var vol enterpriseApi.VolumeSpec @@ -1002,7 +998,7 @@ func GetAppListFromRemoteBucket(ctx context.Context, client splcommon.Controller remoteDataListResponse, err = GetAppsList(ctx, remoteDataClientMgr) if err != nil { // move on to the next appSource if we are not able to get apps list - scopedLog.Error(err, "Unable to get apps list", "appSource", appSource.Name) + scopedLog.ErrorContext(ctx, "Unable to get apps list", "appSource", appSource.Name, "error", err) allSuccess = false continue } @@ -1054,8 +1050,8 @@ func updateAuxPhaseInfo(appDeployInfo *enterpriseApi.AppDeploymentInfo, desiredR // changePhaseInfo changes PhaseInfo and AuxPhaseInfo for each app to desired state func changePhaseInfo(ctx context.Context, desiredReplicas int32, appSrc string, appSrcDeployStatus map[string]enterpriseApi.AppSrcDeployInfo) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("changePhaseInfo") + + scopedLog := logging.FromContext(ctx).With("func", "changePhaseInfo") if appSrcDeploymentInfo, ok := appSrcDeployStatus[appSrc]; ok { appDeployInfoList := appSrcDeploymentInfo.AppDeploymentInfoList @@ -1078,15 +1074,15 @@ func changePhaseInfo(ctx context.Context, desiredReplicas int32, appSrc string, } } else { // Ideally this should never happen, check if the "IsDeploymentInProgress" flag is handled correctly or not - scopedLog.Error(nil, "Could not find the App Source in App context") + scopedLog.ErrorContext(ctx, "Could not find the App Source in App context") } } // checkCmRemainingReferences checks for any stale references of IndexerCluster, LicenseManager, SearchheadCluster, MonitoringConsole // pointing to a particular ClusterManager CR func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClient, cmCr splcommon.MetaObject) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("checkCmRemainingReferences").WithValues("cmCr", cmCr.GetName(), "namespace", cmCr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "checkCmRemainingReferences", "cmCr", cmCr.GetName(), "namespace", cmCr.GetNamespace()) // Filter by namespace listOpts := []client.ListOption{ @@ -1097,13 +1093,13 @@ func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClien idxcList, err := getIndexerClusterList(ctx, c, cmCr, listOpts) if err != nil { if err.Error() != "NotFound" && !k8serrors.IsNotFound(err) { - scopedLog.Error(err, "Couldn't retrieve IndexerCluster list") + scopedLog.ErrorContext(ctx, "Couldn't retrieve IndexerCluster list", "error", err) return err } } for _, item := range idxcList.Items { if item.Spec.ClusterManagerRef.Name == cmCr.GetName() { - scopedLog.Error(nil, fmt.Sprintf(`IndexerCluster %s still has a reference for clusterManager %s, + scopedLog.ErrorContext(ctx, fmt.Sprintf(`IndexerCluster %s still has a reference for clusterManager %s, please backup if needed and delete the IndexerCluster`, item.GetName(), cmCr.GetName())) return fmt.Errorf("ClusterManager has stale references to an indexerCluster") } @@ -1113,13 +1109,13 @@ func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClien shcList, err := getSearchHeadClusterList(ctx, c, cmCr, listOpts) if err != nil { if err.Error() != "NotFound" && !k8serrors.IsNotFound(err) { - scopedLog.Error(err, "Couldn't retrieve SearchHeadCluster list") + scopedLog.ErrorContext(ctx, "Couldn't retrieve SearchHeadCluster list", "error", err) return err } } for _, item := range shcList.Items { if item.Spec.ClusterManagerRef.Name == cmCr.GetName() { - scopedLog.Error(nil, fmt.Sprintf(`SearchHeadCluster %s still has a reference for clusterManager %s, + scopedLog.ErrorContext(ctx, fmt.Sprintf(`SearchHeadCluster %s still has a reference for clusterManager %s, please backup if needed and delete the SearchHeadCluster`, item.GetName(), cmCr.GetName())) return fmt.Errorf("ClusterManager has stale references to a searchHeadCluster") } @@ -1129,13 +1125,13 @@ func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClien lmList, err := getLicenseManagerList(ctx, c, cmCr, listOpts) if err != nil { if err.Error() != "NotFound" && !k8serrors.IsNotFound(err) { - scopedLog.Error(err, "Couldn't retrieve LicenseManager list") + scopedLog.ErrorContext(ctx, "Couldn't retrieve LicenseManager list", "error", err) return err } } for _, item := range lmList.Items { if item.Spec.ClusterManagerRef.Name == cmCr.GetName() { - scopedLog.Error(nil, fmt.Sprintf(`LicenseManager %s still has a reference for clusterManager %s, + scopedLog.ErrorContext(ctx, fmt.Sprintf(`LicenseManager %s still has a reference for clusterManager %s, please backup if needed and delete the LicenseManager`, item.GetName(), cmCr.GetName())) return fmt.Errorf("ClusterManager has stale references to a LicenseManager") } @@ -1145,13 +1141,13 @@ func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClien mcList, err := getMonitoringConsoleList(ctx, c, cmCr, listOpts) if err != nil { if err.Error() != "NotFound" && !k8serrors.IsNotFound(err) { - scopedLog.Error(err, "Couldn't retrieve MonitoringConsole list") + scopedLog.ErrorContext(ctx, "Couldn't retrieve MonitoringConsole list", "error", err) return err } } for _, item := range mcList.Items { if item.Spec.ClusterManagerRef.Name == cmCr.GetName() { - scopedLog.Error(nil, fmt.Sprintf(`MonitoringConsole %s still has a reference for clusterManager %s, + scopedLog.ErrorContext(ctx, fmt.Sprintf(`MonitoringConsole %s still has a reference for clusterManager %s, please backup if needed and delete the MonitoringConsole`, item.GetName(), cmCr.GetName())) return fmt.Errorf("ClusterManager has stale references to a MonitoringConsole") } @@ -1161,8 +1157,8 @@ func checkCmRemainingReferences(ctx context.Context, c splcommon.ControllerClien } func removeStaleEntriesFromAuxPhaseInfo(ctx context.Context, desiredReplicas int32, appSrc string, appSrcDeployStatus map[string]enterpriseApi.AppSrcDeployInfo) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("changePhaseInfo") + + scopedLog := logging.FromContext(ctx).With("func", "changePhaseInfo") if appSrcDeploymentInfo, ok := appSrcDeployStatus[appSrc]; ok { appDeployInfoList := appSrcDeploymentInfo.AppDeploymentInfoList @@ -1175,7 +1171,7 @@ func removeStaleEntriesFromAuxPhaseInfo(ctx context.Context, desiredReplicas int } } else { // Ideally this should never happen, check if the "IsDeploymentInProgress" flag is handled correctly or not - scopedLog.Error(nil, "Could not find the App Source in App context") + scopedLog.ErrorContext(ctx, "Could not find the App Source in App context") } } @@ -1183,8 +1179,8 @@ func removeStaleEntriesFromAuxPhaseInfo(ctx context.Context, desiredReplicas int // changeAppSrcDeployInfoStatus sets the new status to all the apps in an AppSrc if the given repo state and deploy status matches // primarily used in Phase-3 func changeAppSrcDeployInfoStatus(ctx context.Context, appSrc string, appSrcDeployStatus map[string]enterpriseApi.AppSrcDeployInfo, repoState enterpriseApi.AppRepoState, oldDeployStatus enterpriseApi.AppDeploymentStatus, newDeployStatus enterpriseApi.AppDeploymentStatus) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("changeAppSrcDeployInfoStatus").WithValues("Called for AppSource: ", appSrc, "repoState", repoState, "oldDeployStatus", oldDeployStatus, "newDeployStatus", newDeployStatus) + + scopedLog := logging.FromContext(ctx).With("func", "changeAppSrcDeployInfoStatus", "appSource", appSrc, "repoState", repoState, "oldDeployStatus", oldDeployStatus, "newDeployStatus", newDeployStatus) if appSrcDeploymentInfo, ok := appSrcDeployStatus[appSrc]; ok { appDeployInfoList := appSrcDeploymentInfo.AppDeploymentInfoList @@ -1197,10 +1193,10 @@ func changeAppSrcDeployInfoStatus(ctx context.Context, appSrc string, appSrcDepl // Update the Map entry again appSrcDeployStatus[appSrc] = appSrcDeploymentInfo - scopedLog.Info("Complete") + scopedLog.InfoContext(ctx, "Complete") } else { // Ideally this should never happen, check if the "IsDeploymentInProgress" flag is handled correctly or not - scopedLog.Error(nil, "Could not find the App Source in App context") + scopedLog.ErrorContext(ctx, "Could not find the App Source in App context") } } @@ -1230,14 +1226,14 @@ func isAppRepoStateDeleted(appDeployInfo enterpriseApi.AppDeploymentInfo) bool { func handleAppRepoChanges(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appDeployContext *enterpriseApi.AppDeploymentContext, remoteObjListingMap map[string]splclient.RemoteDataListResponse, appFrameworkConfig *enterpriseApi.AppFrameworkSpec) (bool, error) { crKind := cr.GetObjectKind().GroupVersionKind().Kind - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("handleAppRepoChanges").WithValues("kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "handleAppRepoChanges", "kind", crKind, "name", cr.GetName(), "namespace", cr.GetNamespace()) var err error appsModified := false - scopedLog.Info("received App listing", "for App sources", len(remoteObjListingMap)) + scopedLog.InfoContext(ctx, "received App listing", "for App sources", len(remoteObjListingMap)) if len(remoteObjListingMap) == 0 { - scopedLog.Error(nil, "remoteObjectList is empty. Any apps that are already deployed will be disabled") + scopedLog.ErrorContext(ctx, "remoteObjectList is empty. Any apps that are already deployed will be disabled") } // Check if the appSource is still valid in the config @@ -1253,7 +1249,7 @@ func handleAppRepoChanges(ctx context.Context, client splcommon.ControllerClient // If the AppSrc is missing mark all the corresponding apps for deletion if !CheckIfAppSrcExistsInConfig(appFrameworkConfig, appSrc) || !checkIfAppSrcExistsWithRemoteListing(appSrc, remoteObjListingMap) { - scopedLog.Info("App change: App source is missing in config or remote listing, deleting/disabling all the apps", "App source", appSrc) + scopedLog.InfoContext(ctx, "App change: App source is missing in config or remote listing, deleting/disabling all the apps", "App source", appSrc) curAppDeployList := appSrcDeploymentInfo.AppDeploymentInfoList var modified bool @@ -1275,7 +1271,7 @@ func handleAppRepoChanges(ctx context.Context, client splcommon.ControllerClient currentList := appSrcDeploymentInfo.AppDeploymentInfoList for appIdx := range currentList { if !isAppRepoStateDeleted(appSrcDeploymentInfo.AppDeploymentInfoList[appIdx]) && !checkIfAnAppIsActiveOnRemoteStore(currentList[appIdx].AppName, remoteDataListResponse.Objects) { - scopedLog.Info("App change", "deleting/disabling the App: ", currentList[appIdx].AppName, "as it is missing in the remote listing", nil) + scopedLog.InfoContext(ctx, "App change", "deleting/disabling the App: ", currentList[appIdx].AppName, "as it is missing in the remote listing", nil) setStateAndStatusForAppDeployInfo(¤tList[appIdx], enterpriseApi.RepoStateDeleted, enterpriseApi.DeployStatusComplete) } } @@ -1312,8 +1308,8 @@ func isAppExtensionValid(receivedKey string) bool { // AddOrUpdateAppSrcDeploymentInfoList modifies the App deployment status as perceived from the remote object listing func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentInfo *enterpriseApi.AppSrcDeployInfo, remoteS3ObjList []*splclient.RemoteObject) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("AddOrUpdateAppSrcDeploymentInfoList").WithValues("Called with length: ", len(remoteS3ObjList)) + + scopedLog := logging.FromContext(ctx).With("func", "AddOrUpdateAppSrcDeploymentInfoList", "listLength", len(remoteS3ObjList)) var found bool var appName string @@ -1324,7 +1320,7 @@ func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentIn for _, remoteObj := range remoteS3ObjList { receivedKey := *remoteObj.Key if !isAppExtensionValid(receivedKey) { - scopedLog.Error(nil, "App name Parsing: Ignoring the key with invalid extension", "receivedKey", receivedKey) + scopedLog.ErrorContext(ctx, "App name Parsing: Ignoring the key with invalid extension", "receivedKey", receivedKey) continue } @@ -1338,7 +1334,7 @@ func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentIn if appList[idx].AppName == appName { found = true if appList[idx].ObjectHash != *remoteObj.Etag || appList[idx].RepoState == enterpriseApi.RepoStateDeleted { - scopedLog.Info("App change detected. Marking for an update.", "appName", appName) + scopedLog.InfoContext(ctx, "App change detected. Marking for an update.", "appName", appName) appList[idx].ObjectHash = *remoteObj.Etag appList[idx].IsUpdate = true appList[idx].DeployStatus = enterpriseApi.DeployStatusPending @@ -1349,7 +1345,7 @@ func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentIn // Make the state active for an app that was deleted earlier, and got activated again if appList[idx].RepoState == enterpriseApi.RepoStateDeleted { - scopedLog.Info("App change. Enabling the App that was previously disabled/deleted", "appName", appName) + scopedLog.InfoContext(ctx, "App change. Enabling the App that was previously disabled/deleted", "appName", appName) appList[idx].RepoState = enterpriseApi.RepoStateActive } appChangesDetected = true @@ -1362,7 +1358,7 @@ func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentIn // Update our local list if it is a new app if !found { - scopedLog.Info("New App found", "appName", appName) + scopedLog.InfoContext(ctx, "New App found", "appName", appName) appDeployInfo.AppName = appName appDeployInfo.ObjectHash = *remoteObj.Etag appDeployInfo.RepoState = enterpriseApi.RepoStateActive @@ -1390,8 +1386,8 @@ func AddOrUpdateAppSrcDeploymentInfoList(ctx context.Context, appSrcDeploymentIn // Note:- Used in only for Phase-2 func markAppsStatusToComplete(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appConf *enterpriseApi.AppFrameworkSpec, appSrcDeploymentStatus map[string]enterpriseApi.AppSrcDeployInfo) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("markAppsStatusToComplete") + + scopedLog := logging.FromContext(ctx).With("func", "markAppsStatusToComplete") // ToDo: Passing appSrcDeploymentStatus is redundant, but this function will go away in phase-3, so ok for now. for appSrc := range appSrcDeploymentStatus { @@ -1399,7 +1395,7 @@ func markAppsStatusToComplete(ctx context.Context, client splcommon.ControllerCl changeAppSrcDeployInfoStatus(ctx, appSrc, appSrcDeploymentStatus, enterpriseApi.RepoStateDeleted, enterpriseApi.DeployStatusPending, enterpriseApi.DeployStatusComplete) } - scopedLog.Info("Marked the App deployment status to complete") + scopedLog.InfoContext(ctx, "Marked the App deployment status to complete") // ToDo: Caller of this API also needs to set "IsDeploymentInProgress = false" once after completing this function call for all the app sources return err @@ -1435,8 +1431,8 @@ func setupAppsStagingVolume(ctx context.Context, client splcommon.ControllerClie // isAppAlreadyDownloaded checks if the app is already present on the operator pod func isAppAlreadyDownloaded(ctx context.Context, downloadWorker *PipelineWorker) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isAppAlreadyDownloaded").WithValues("app name", downloadWorker.appDeployInfo.AppName) + + scopedLog := logging.FromContext(ctx).With("func", "isAppAlreadyDownloaded", "app name", downloadWorker.appDeployInfo.AppName) scope := getAppSrcScope(ctx, downloadWorker.afwConfig, downloadWorker.appSrcName) kind := downloadWorker.cr.GetObjectKind().GroupVersionKind().Kind @@ -1448,7 +1444,7 @@ func isAppAlreadyDownloaded(ctx context.Context, downloadWorker *PipelineWorker) fileInfo, err := os.Stat(localAppFileName) if errors.Is(err, os.ErrNotExist) { - scopedLog.Info("App not present on operator pod") + scopedLog.InfoContext(ctx, "App not present on operator pod") return false } @@ -1456,7 +1452,7 @@ func isAppAlreadyDownloaded(ctx context.Context, downloadWorker *PipelineWorker) remoteSize := int64(downloadWorker.appDeployInfo.Size) if localSize != remoteSize { err = fmt.Errorf("local size does not match with size on remote storage. localSize=%d, remoteSize=%d", localSize, remoteSize) - scopedLog.Error(err, "incorrect app size") + scopedLog.ErrorContext(ctx, "incorrect app size", "error", err) return false } return true @@ -1464,24 +1460,24 @@ func isAppAlreadyDownloaded(ctx context.Context, downloadWorker *PipelineWorker) // SetLastAppInfoCheckTime sets the last check time to current time func SetLastAppInfoCheckTime(ctx context.Context, appInfoStatus *enterpriseApi.AppDeploymentContext) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("SetLastAppInfoCheckTime") + + scopedLog := logging.FromContext(ctx).With("func", "SetLastAppInfoCheckTime") currentEpoch := time.Now().Unix() - scopedLog.Info("Setting the LastAppInfoCheckTime to current time", "current epoch time", currentEpoch) + scopedLog.InfoContext(ctx, "Setting the LastAppInfoCheckTime to current time", "current epoch time", currentEpoch) appInfoStatus.LastAppInfoCheckTime = currentEpoch } // HasAppRepoCheckTimerExpired checks if the polling interval has expired func HasAppRepoCheckTimerExpired(ctx context.Context, appInfoContext *enterpriseApi.AppDeploymentContext) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("HasAppRepoCheckTimerExpired") + + scopedLog := logging.FromContext(ctx).With("func", "HasAppRepoCheckTimerExpired") currentEpoch := time.Now().Unix() isTimerExpired := appInfoContext.LastAppInfoCheckTime+appInfoContext.AppsRepoStatusPollInterval <= currentEpoch if isTimerExpired { - scopedLog.Info("App repo polling interval timer has expired", "LastAppInfoCheckTime", strconv.FormatInt(appInfoContext.LastAppInfoCheckTime, 10), "current epoch time", strconv.FormatInt(currentEpoch, 10)) + scopedLog.InfoContext(ctx, "App repo polling interval timer has expired", "LastAppInfoCheckTime", strconv.FormatInt(appInfoContext.LastAppInfoCheckTime, 10), "current epoch time", strconv.FormatInt(currentEpoch, 10)) } return isTimerExpired @@ -1492,8 +1488,8 @@ func HasAppRepoCheckTimerExpired(ctx context.Context, appInfoContext *enterprise // Hence we need to subtract the delta time elapsed from the actual polling interval, // so that the next reconcile would happen at the right time. func GetNextRequeueTime(ctx context.Context, appRepoPollInterval, lastCheckTime int64) time.Duration { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetNextRequeueTime") + + scopedLog := logging.FromContext(ctx).With("func", "GetNextRequeueTime") currentEpoch := time.Now().Unix() var nextRequeueTimeInSec int64 @@ -1502,7 +1498,7 @@ func GetNextRequeueTime(ctx context.Context, appRepoPollInterval, lastCheckTime nextRequeueTimeInSec = 5 } - scopedLog.Info("Getting next requeue time", "LastAppInfoCheckTime", lastCheckTime, "Current Epoch time", currentEpoch, "nextRequeueTimeInSec", nextRequeueTimeInSec) + scopedLog.InfoContext(ctx, "Getting next requeue time", "LastAppInfoCheckTime", lastCheckTime, "Current Epoch time", currentEpoch, "nextRequeueTimeInSec", nextRequeueTimeInSec) return time.Second * (time.Duration(nextRequeueTimeInSec)) } @@ -1516,16 +1512,16 @@ func isAppRepoPollingEnabled(appStatusContext *enterpriseApi.AppDeploymentContex // shouldCheckAppRepoStatus func shouldCheckAppRepoStatus(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appStatusContext *enterpriseApi.AppDeploymentContext, kind string, turnOffManualChecking *bool) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("shouldCheckAppRepoStatus") + + scopedLog := logging.FromContext(ctx).With("func", "shouldCheckAppRepoStatus") // If polling is disabled, check if manual update is on. if !isAppRepoPollingEnabled(appStatusContext) { configMapName := GetSplunkManualAppUpdateConfigMapName(cr.GetNamespace()) // Check if we need to manually check for app updates for this CR kind - scopedLog.Info("checking if namespace specific configmap contains manualUpdate settings") + scopedLog.InfoContext(ctx, "checking if namespace specific configmap contains manualUpdate settings") if getManualUpdateStatus(ctx, client, cr, configMapName) == "on" { - scopedLog.Info("namespace specific configmap contains manualUpdate is set to on", "configMapName", configMapName) + scopedLog.InfoContext(ctx, "namespace specific configmap contains manualUpdate is set to on", "configMapName", configMapName) // There can be more than 1 CRs of this kind. We should only // turn off the status once all the CRs have finished the reconciles if getManualUpdateRefCount(ctx, client, cr, configMapName) == 1 { @@ -1540,13 +1536,13 @@ func shouldCheckAppRepoStatus(ctx context.Context, client splcommon.ControllerCl } func IsManualUpdateSetInCRConfig(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appStatusContext *enterpriseApi.AppDeploymentContext, kind string, turnOffManualChecking *bool) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("shouldCheckAppRepoStatusPerCR") + + scopedLog := logging.FromContext(ctx).With("func", "shouldCheckAppRepoStatusPerCR") configMapName := fmt.Sprintf(perCrConfigMapNameStr, KindToInstanceString(cr.GroupVersionKind().Kind), cr.GetName()) - scopedLog.Info("checking if per CR specific configmap contains manualUpdate settings") + scopedLog.InfoContext(ctx, "checking if per CR specific configmap contains manualUpdate settings") if getManualUpdatePerCrStatus(ctx, client, cr, configMapName) == "on" { - scopedLog.Info("CR specific configmap contains manualUpdate is set to on", "configMapName", configMapName) + scopedLog.InfoContext(ctx, "CR specific configmap contains manualUpdate is set to on", "configMapName", configMapName) //*turnOffManualChecking = true return true } @@ -1581,8 +1577,8 @@ func getCleanObjectDigest(rawObjectDigest *string) (*string, error) { // Returns: // - error: An error if the operation fails, otherwise nil. func updateManualAppUpdateConfigMapLocked(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appStatusContext *enterpriseApi.AppDeploymentContext, kind string, turnOffManualChecking bool) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateManualAppUpdateConfigMap").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "updateManualAppUpdateConfigMap", "name", cr.GetName(), "namespace", cr.GetNamespace()) var status string configMapName := GetSplunkManualAppUpdateConfigMapName(cr.GetNamespace()) @@ -1594,7 +1590,7 @@ func updateManualAppUpdateConfigMapLocked(ctx context.Context, client splcommon. defer mux.Unlock() configMap, err := splctrl.GetConfigMap(ctx, client, namespacedName) if err != nil { - scopedLog.Error(err, "Unable to get configMap", "name", namespacedName.Name) + scopedLog.ErrorContext(ctx, "Unable to get configMap", "name", namespacedName.Name, "error", err) return err } @@ -1604,7 +1600,7 @@ func updateManualAppUpdateConfigMapLocked(ctx context.Context, client splcommon. // turn off the manual checking for this CR kind in the configMap if turnOffManualChecking { - scopedLog.Info("Turning off manual checking of apps update", "Kind", kind) + scopedLog.InfoContext(ctx, "Turning off manual checking of apps update", "Kind", kind) // reset the status back to "off" and // refCount to original count status = "off" @@ -1625,7 +1621,7 @@ func updateManualAppUpdateConfigMapLocked(ctx context.Context, client splcommon. err = splutil.UpdateResource(ctx, client, configMap) if err != nil { - scopedLog.Error(err, "Could not update the configMap", "name", namespacedName.Name) + scopedLog.ErrorContext(ctx, "Could not update the configMap", "name", namespacedName.Name, "error", err) return err } } @@ -1635,8 +1631,7 @@ func updateManualAppUpdateConfigMapLocked(ctx context.Context, client splcommon. func updateCrSpecificManualAppUpdateConfigMap(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appStatusContext *enterpriseApi.AppDeploymentContext, kind string, turnOffManualChecking bool) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateManualAppUpdateConfigMap").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "updateManualAppUpdateConfigMap", "name", cr.GetName(), "namespace", cr.GetNamespace()) configMapName := GetSplunkManualAppUpdateConfigMapName(cr.GetNamespace()) namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} // now check namespace specific configmap if it contains manualUpdate settings @@ -1644,17 +1639,17 @@ func updateCrSpecificManualAppUpdateConfigMap(ctx context.Context, client splcom crNamespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: crScopedConfigMapName} configMap, err := splctrl.GetConfigMap(ctx, client, crNamespacedName) if err != nil { - scopedLog.Error(err, "Unable to get configMap", "name", namespacedName.Name) + scopedLog.ErrorContext(ctx, "Unable to get configMap", "name", namespacedName.Name, "error", err) return err } if configMap.Data["manualUpdate"] == "on" { - scopedLog.Info("Turning off manual checking of apps update in per CR configmap", "Kind", kind) + scopedLog.InfoContext(ctx, "Turning off manual checking of apps update in per CR configmap", "Kind", kind) configMap.Data["manualUpdate"] = "off" } err = splutil.UpdateResource(ctx, client, configMap) if err != nil { - scopedLog.Error(err, "Could not update the per CR configMap", "name", crNamespacedName.Name) + scopedLog.ErrorContext(ctx, "Could not update the per CR configMap", "name", crNamespacedName.Name, "error", err) return err } return err @@ -1663,8 +1658,8 @@ func updateCrSpecificManualAppUpdateConfigMap(ctx context.Context, client splcom // initAndCheckAppInfoStatus initializes the RemoteDataClients and checks the status of apps on remote storage. func initAndCheckAppInfoStatus(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appFrameworkConf *enterpriseApi.AppFrameworkSpec, appStatusContext *enterpriseApi.AppDeploymentContext) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("initAndCheckAppInfoStatus").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "initAndCheckAppInfoStatus", "name", cr.GetName(), "namespace", cr.GetNamespace()) var err error // Register the RemoteData Clients specific to providers if not done already @@ -1673,7 +1668,7 @@ func initAndCheckAppInfoStatus(ctx context.Context, client splcommon.ControllerC // to match the spec in the previous run. err = initAppFrameWorkContext(ctx, client, cr, appFrameworkConf, appStatusContext) if err != nil { - scopedLog.Error(err, "Unable initialize app framework") + scopedLog.ErrorContext(ctx, "Unable initialize app framework", "error", err) return err } @@ -1686,41 +1681,41 @@ func initAndCheckAppInfoStatus(ctx context.Context, client splcommon.ControllerC !reflect.DeepEqual(appStatusContext.AppFrameworkConfig, *appFrameworkConf) { if appStatusContext.IsDeploymentInProgress { - scopedLog.Info("App installation is already in progress. Not checking for any latest app repo changes") + scopedLog.InfoContext(ctx, "App installation is already in progress. Not checking for any latest app repo changes") return nil } appStatusContext.IsDeploymentInProgress = true var sourceToAppsList map[string]splclient.RemoteDataListResponse - scopedLog.Info("Checking status of apps on remote storage...") + scopedLog.InfoContext(ctx, "Checking status of apps on remote storage...") sourceToAppsList, err = GetAppListFromRemoteBucket(ctx, client, cr, appFrameworkConf) // TODO: gaurav, we need to handle this case better in Phase-3. There can be a possibility // where if an appSource is missing in remote store, we mark it for deletion. But if it comes up // next time, we will recycle the pod to install the app. We need to find a way to reduce the pod recycles. if len(sourceToAppsList) != len(appFrameworkConf.AppSources) { - scopedLog.Error(err, "Unable to get apps list, will retry in next reconcile...") + scopedLog.ErrorContext(ctx, "Unable to get apps list, will retry in next reconcile...", "error", err) } else { for _, appSource := range appFrameworkConf.AppSources { // Clean-up for the object digest value for i := range sourceToAppsList[appSource.Name].Objects { cleanDigest, err := getCleanObjectDigest(sourceToAppsList[appSource.Name].Objects[i].Etag) if err != nil { - scopedLog.Error(err, "unable to fetch clean object digest value", "Object Hash", sourceToAppsList[appSource.Name].Objects[i].Etag) + scopedLog.ErrorContext(ctx, "unable to fetch clean object digest value", "Object Hash", sourceToAppsList[appSource.Name].Objects[i].Etag, "error", err) return err } sourceToAppsList[appSource.Name].Objects[i].Etag = cleanDigest } - scopedLog.Info("Apps List retrieved from remote storage", "App Source", appSource.Name, "Content", sourceToAppsList[appSource.Name].Objects) + scopedLog.InfoContext(ctx, "Apps List retrieved from remote storage", "App Source", appSource.Name, "Content", sourceToAppsList[appSource.Name].Objects) } // Only handle the app repo changes if we were able to successfully get the apps list _, err = handleAppRepoChanges(ctx, client, cr, appStatusContext, sourceToAppsList, appFrameworkConf) if err != nil { - scopedLog.Error(err, "Unable to use the App list retrieved from the remote storage") + scopedLog.ErrorContext(ctx, "Unable to use the App list retrieved from the remote storage", "error", err) return err } @@ -1734,12 +1729,12 @@ func initAndCheckAppInfoStatus(ctx context.Context, client splcommon.ControllerC if !isAppRepoPollingEnabled(appStatusContext) { err = updateManualAppUpdateConfigMapLocked(ctx, client, cr, appStatusContext, kind, turnOffManualChecking) if err != nil { - scopedLog.Error(err, "failed to update the manual app udpate configMap") + scopedLog.ErrorContext(ctx, "failed to update the manual app udpate configMap", "error", err) return err } err = updateCrSpecificManualAppUpdateConfigMap(ctx, client, cr, appStatusContext, kind, turnOffManualChecking) if err != nil { - scopedLog.Error(err, "failed to update the manual app udpate CR specific configMap") + scopedLog.ErrorContext(ctx, "failed to update the manual app udpate CR specific configMap", "error", err) return err } } @@ -1750,8 +1745,8 @@ func initAndCheckAppInfoStatus(ctx context.Context, client splcommon.ControllerC // SetConfigMapOwnerRef sets the owner references for the configMap func SetConfigMapOwnerRef(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, configMap *corev1.ConfigMap) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("SetConfigMapOwnerRef").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "SetConfigMapOwnerRef", "name", cr.GetName(), "namespace", cr.GetNamespace()) currentOwnerRef := configMap.GetOwnerReferences() // Check if owner ref exists @@ -1767,14 +1762,14 @@ func SetConfigMapOwnerRef(ctx context.Context, client splcommon.ControllerClient // Update the configMap now err := splutil.UpdateResource(ctx, client, configMap) if err != nil { - scopedLog.Error(err, "Unable to update configMap", "name", configMap.Name) + scopedLog.ErrorContext(ctx, "Unable to update configMap", "name", configMap.Name, "error", err) return err } return nil - } +// getNumOfOwnerRefsKind returns the number of owner references of a given kind func getNumOfOwnerRefsKind(configMap *corev1.ConfigMap, kind string) int { var numOfObjects int currentOwnerRefs := configMap.GetOwnerReferences() @@ -1789,8 +1784,8 @@ func getNumOfOwnerRefsKind(configMap *corev1.ConfigMap, kind string) int { // UpdateOrRemoveEntryFromConfigMapLocked removes/updates the entry for the CR type from the manual app update configMap func UpdateOrRemoveEntryFromConfigMapLocked(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, instanceType InstanceType) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("UpdateOrRemoveEntryFromConfigMapLocked").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "UpdateOrRemoveEntryFromConfigMapLocked", "name", cr.GetName(), "namespace", cr.GetNamespace()) configMapName := GetSplunkManualAppUpdateConfigMapName(cr.GetNamespace()) namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: configMapName} @@ -1800,7 +1795,7 @@ func UpdateOrRemoveEntryFromConfigMapLocked(ctx context.Context, c splcommon.Con defer mux.Unlock() configMap, err := splctrl.GetConfigMap(ctx, c, namespacedName) if err != nil { - scopedLog.Error(err, "Unable to get config map", "name", namespacedName.Name) + scopedLog.ErrorContext(ctx, "Unable to get config map", "name", namespacedName.Name, "error", err) return err } @@ -1828,7 +1823,7 @@ refCount: %d`, getManualUpdateStatus(ctx, c, cr, configMapName), numOfObjects) // Update configMap now err = splutil.UpdateResource(ctx, c, configMap) if err != nil { - scopedLog.Error(err, "Unable to update configMap", "name", namespacedName.Name) + scopedLog.ErrorContext(ctx, "Unable to update configMap", "name", namespacedName.Name, "error", err) return err } @@ -1878,8 +1873,8 @@ func extractFieldFromConfigMapData(fieldRegex, data string) string { // checkIfFileExistsOnPod confirms if the given file path exits on a given Pod func checkIfFileExistsOnPod(ctx context.Context, cr splcommon.MetaObject, filePath string, podExecClient splutil.PodExecClientImpl) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("checkIfFileExistsOnPod").WithValues("podName", podExecClient.GetTargetPodName(), "namespace", cr.GetNamespace()).WithValues("filePath", filePath) + + scopedLog := logging.FromContext(ctx).With("func", "checkIfFileExistsOnPod", "podName", podExecClient.GetTargetPodName(), "namespace", cr.GetNamespace(), "filePath", filePath) // Make sure the destination directory is existing fPath := path.Clean(filePath) command := fmt.Sprintf("test -f %s; echo -n $?", fPath) @@ -1887,7 +1882,7 @@ func checkIfFileExistsOnPod(ctx context.Context, cr splcommon.MetaObject, filePa stdOut, stdErr, err := podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if stdErr != "" || err != nil { - scopedLog.Error(err, "error in checking the file availability on the Pod", "stdErr", stdErr, "stdOut", stdOut, "filePath", fPath) + scopedLog.ErrorContext(ctx, "error in checking the file availability on the Pod", "stdErr", stdErr, "stdOut", stdOut, "filePath", fPath, "error", err) return false } @@ -1925,8 +1920,8 @@ func createDirOnSplunkPods(ctx context.Context, cr splcommon.MetaObject, replica // CopyFileToPod copies a file from Operator Pod to any given Pod of a custom resource func CopyFileToPod(ctx context.Context, c splcommon.ControllerClient, namespace string, srcPath string, destPath string, podExecClient splutil.PodExecClientImpl) (string, string, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("CopyFileToPod").WithValues("podName", podExecClient.GetTargetPodName(), "namespace", namespace).WithValues("srcPath", srcPath, "destPath", destPath) + + scopedLog := logging.FromContext(ctx).With("func", "CopyFileToPod", "podName", podExecClient.GetTargetPodName(), "namespace", namespace, "srcPath", srcPath, "destPath", destPath) var err error reader, writer := io.Pipe() @@ -1977,7 +1972,7 @@ func CopyFileToPod(ctx context.Context, c splcommon.ControllerClient, namespace defer writer.Close() err := cpMakeTar(localPath{file: srcPath}, remotePath{file: destPath}, writer) if err != nil { - scopedLog.Error(err, "Failed to send file on writer pipe", "srcPath", srcPath, "destPath", destPath) + scopedLog.ErrorContext(ctx, "Failed to send file on writer pipe", "srcPath", srcPath, "destPath", destPath, "error", err) return } }() @@ -2046,8 +2041,8 @@ func validateMonitoringConsoleRef(ctx context.Context, c splcommon.ControllerCli // setInstallStateForClusterScopedApps sets the install state for cluster scoped apps func setInstallStateForClusterScopedApps(ctx context.Context, appDeployContext *enterpriseApi.AppDeploymentContext) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("setInstallStateForClusterScopedApps") + + scopedLog := logging.FromContext(ctx).With("func", "setInstallStateForClusterScopedApps") for appSrcName, appSrcDeployInfo := range appDeployContext.AppsSrcDeployStatus { // Mark only cluster scoped apps @@ -2060,9 +2055,9 @@ func setInstallStateForClusterScopedApps(ctx context.Context, appDeployContext * if deployInfoList[i].PhaseInfo.Phase == enterpriseApi.PhasePodCopy && deployInfoList[i].PhaseInfo.Status == enterpriseApi.AppPkgPodCopyComplete { deployInfoList[i].PhaseInfo.Phase = enterpriseApi.PhaseInstall deployInfoList[i].PhaseInfo.Status = enterpriseApi.AppPkgInstallComplete - scopedLog.Info("Cluster scoped app installed", "app name", deployInfoList[i].AppName, "digest", deployInfoList[i].ObjectHash) + scopedLog.InfoContext(ctx, "Cluster scoped app installed", "app name", deployInfoList[i].AppName, "digest", deployInfoList[i].ObjectHash) } else if deployInfoList[i].PhaseInfo.Phase != enterpriseApi.PhaseInstall || deployInfoList[i].PhaseInfo.Status != enterpriseApi.AppPkgInstallComplete { - scopedLog.Error(nil, "app missing from bundle push", "app name", deployInfoList[i].AppName, "digest", deployInfoList[i].ObjectHash, "phase", deployInfoList[i].PhaseInfo.Phase, "status", deployInfoList[i].PhaseInfo.Status) + scopedLog.ErrorContext(ctx, "app missing from bundle push", "app name", deployInfoList[i].AppName, "digest", deployInfoList[i].ObjectHash, "phase", deployInfoList[i].PhaseInfo.Phase, "status", deployInfoList[i].PhaseInfo.Status) } } } @@ -2110,14 +2105,14 @@ func releaseStorage(releaseSize uint64) error { // updateReconcileRequeueTime updates the reconcile requeue result func updateReconcileRequeueTime(ctx context.Context, result *reconcile.Result, rqTime time.Duration, requeue bool) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateReconcileRequeueTime") + + scopedLog := logging.FromContext(ctx).With("func", "updateReconcileRequeueTime") if result == nil { - scopedLog.Error(nil, "invalid result") + scopedLog.ErrorContext(ctx, "invalid result") return } if rqTime <= 0 { - scopedLog.Error(nil, "invalid requeue time", "time value", rqTime) + scopedLog.ErrorContext(ctx, "invalid requeue time", "time value", rqTime) return } @@ -2131,8 +2126,8 @@ func updateReconcileRequeueTime(ctx context.Context, result *reconcile.Result, r // handleAppFrameworkActivity handles any pending app framework activity func handleAppFrameworkActivity(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, appDeployContext *enterpriseApi.AppDeploymentContext, appFrameworkConfig *enterpriseApi.AppFrameworkSpec) *reconcile.Result { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("afwSchedulerEntry").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "afwSchedulerEntry", "name", cr.GetName(), "namespace", cr.GetNamespace()) finalResult := &reconcile.Result{ Requeue: false, RequeueAfter: maxRecDuration, @@ -2147,7 +2142,7 @@ func handleAppFrameworkActivity(ctx context.Context, client splcommon.Controller if appDeployContext.AppsSrcDeployStatus != nil { requeue, err := afwSchedulerEntry(ctx, client, cr, appDeployContext, appFrameworkConfig) if err != nil { - scopedLog.Error(err, "app framework returned error") + scopedLog.ErrorContext(ctx, "app framework returned error", "error", err) } if requeue { updateReconcileRequeueTime(ctx, finalResult, time.Second*5, true) @@ -2177,8 +2172,8 @@ func checkAndMigrateAppDeployStatus(ctx context.Context, client splcommon.Contro // migrateAfwStatus migrates the appframework status context to the latest version func migrateAfwStatus(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, afwStatusContext *enterpriseApi.AppDeploymentContext) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("migrateAfwStatus").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "migrateAfwStatus", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Upgrade one version at a time // Start with the lowest version, then move towards the latest @@ -2186,7 +2181,7 @@ func migrateAfwStatus(ctx context.Context, client splcommon.ControllerClient, cr switch { // Always start with the lowest version case afwStatusContext.Version < enterpriseApi.AfwPhase3: - scopedLog.Info("Migrating the App framework", "old version", afwStatusContext.Version, "new version", enterpriseApi.AfwPhase3) + scopedLog.InfoContext(ctx, "Migrating the App framework", "old version", afwStatusContext.Version, "new version", enterpriseApi.AfwPhase3) err := migrateAfwFromPhase2ToPhase3(ctx, client, cr, afwStatusContext) if err != nil { return false @@ -2199,7 +2194,7 @@ func migrateAfwStatus(ctx context.Context, client splcommon.ControllerClient, cr // Update the new status err := client.Status().Update(context.TODO(), cr) if err != nil { - scopedLog.Error(err, "status update failed") + scopedLog.ErrorContext(ctx, "status update failed", "error", err) } return err == nil @@ -2207,8 +2202,8 @@ func migrateAfwStatus(ctx context.Context, client splcommon.ControllerClient, cr // migrateAfwFromPhase2ToPhase3 migrates app framework status from Phase-2 to Phase-3 func migrateAfwFromPhase2ToPhase3(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, afwStatusContext *enterpriseApi.AppDeploymentContext) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("migrateAfwFromPhase2ToPhase3").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + + scopedLog := logging.FromContext(ctx).With("func", "migrateAfwFromPhase2ToPhase3", "name", cr.GetName(), "namespace", cr.GetNamespace()) sts := afwGetReleventStatefulsetByKind(ctx, cr, client) @@ -2218,7 +2213,7 @@ func migrateAfwFromPhase2ToPhase3(ctx context.Context, client splcommon.Controll // Remove the special characters from Object hash cleanDigest, err := getCleanObjectDigest(&deployInfoList[i].ObjectHash) if err != nil { - scopedLog.Error(err, "clean-up failed", "digest", deployInfoList[i].ObjectHash) + scopedLog.ErrorContext(ctx, "clean-up failed", "digest", deployInfoList[i].ObjectHash, "error", err) return err } deployInfoList[i].ObjectHash = *cleanDigest @@ -2248,7 +2243,7 @@ func migrateAfwFromPhase2ToPhase3(ctx context.Context, client splcommon.Controll } afwStatusContext.Version = enterpriseApi.AfwPhase3 - scopedLog.Info("migration completed") + scopedLog.InfoContext(ctx, "migration completed") return nil } @@ -2259,25 +2254,25 @@ func isAppFrameworkMigrationNeeded(afwStatusContext *enterpriseApi.AppDeployment // updateCRStatus fetches the latest CR, and on top of that, updates latest status including error messages as well func updateCRStatus(ctx context.Context, client splcommon.ControllerClient, origCR splcommon.MetaObject, crError *error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateCRStatus").WithValues("original cr version", origCR.GetResourceVersion()) + + scopedLog := logging.FromContext(ctx).With("func", "updateCRStatus", "original cr version", origCR.GetResourceVersion()) var tryCnt int for tryCnt = 0; tryCnt < maxRetryCountForCRStatusUpdate; tryCnt++ { latestCR, err := fetchCurrentCRWithStatusUpdate(ctx, client, origCR, crError) if err != nil { if origCR.GetDeletionTimestamp() == nil { - scopedLog.Error(err, "Unable to Read the latest CR from the K8s") + scopedLog.ErrorContext(ctx, "Unable to Read the latest CR from the K8s", "error", err) } continue } - scopedLog.Info("Trying to update", "count", tryCnt) + scopedLog.InfoContext(ctx, "Trying to update", "count", tryCnt) curCRVersion := latestCR.GetResourceVersion() err = client.Status().Update(ctx, latestCR) if err == nil { updatedCRVersion := latestCR.GetResourceVersion() - scopedLog.Info("Status update successful", "current CR version", curCRVersion, "updated CR version", updatedCRVersion) + scopedLog.InfoContext(ctx, "Status update successful", "current CR version", curCRVersion, "updated CR version", updatedCRVersion) // While the current reconcile is in progress, there may be new event(s) from the // list of watchers satisfying the predicates. That triggeres a new reconcile right after @@ -2290,7 +2285,7 @@ func updateCRStatus(ctx context.Context, client splcommon.ControllerClient, orig for chkCnt := 0; chkCnt < maxRetryCountForCRStatusUpdate; chkCnt++ { crAfterUpdate, err := fetchCurrentCRWithStatusUpdate(ctx, client, latestCR, crError) if err == nil && updatedCRVersion == crAfterUpdate.GetResourceVersion() { - scopedLog.Info("Cache is reflecting the latest CR", "updated CR version", updatedCRVersion) + scopedLog.InfoContext(ctx, "Cache is reflecting the latest CR", "updated CR version", updatedCRVersion) // Latest CR is reflecting in the cache break } @@ -2301,14 +2296,14 @@ func updateCRStatus(ctx context.Context, client splcommon.ControllerClient, orig // Status update successful break } else { - scopedLog.Error(err, "Error trying to update the CR status") + scopedLog.ErrorContext(ctx, "Error trying to update the CR status", "error", err) } time.Sleep(time.Duration(tryCnt) * 10 * time.Millisecond) } if origCR.GetDeletionTimestamp() == nil && tryCnt >= maxRetryCountForCRStatusUpdate { - scopedLog.Error(nil, "Status update failed", "Attempt count", tryCnt) + scopedLog.ErrorContext(ctx, "Status update failed", "Attempt count", tryCnt) } } @@ -2476,12 +2471,12 @@ func fetchCurrentCRWithStatusUpdate(ctx context.Context, client splcommon.Contro // ReadFile reads the contents of the given file name passed as string func ReadFile(ctx context.Context, fileLocation string) (string, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ReadFile").WithValues("FileLocation", fileLocation) + + scopedLog := logging.FromContext(ctx).With("func", "ReadFile", "FileLocation", fileLocation) byteString, err := os.ReadFile(fileLocation) if err != nil { - scopedLog.Error(err, "Failed to read file") + scopedLog.ErrorContext(ctx, "Failed to read file", "error", err) return "", err } @@ -2493,8 +2488,8 @@ func setProbeLevelOnSplunkPod(ctx context.Context, podExecClient splutil.PodExec var err error var stdOut string var command string - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("setProbeLevelOnSplunkPod").WithValues("podName", podExecClient.GetTargetPodName(), "probeLevel", probeLevel) + + scopedLog := logging.FromContext(ctx).With("func", "setProbeLevelOnSplunkPod", "podName", podExecClient.GetTargetPodName(), "probeLevel", probeLevel) switch probeLevel { case livenessProbeLevelDefault: command = fmt.Sprintf("[[ -f %s ]] && > %s", GetLivenessDriverFilePath(), GetLivenessDriverFilePath()) @@ -2511,11 +2506,11 @@ func setProbeLevelOnSplunkPod(ctx context.Context, podExecClient splutil.PodExec stdOut, _, err = podExecClient.RunPodExecCommand(ctx, streamOptions, []string{"/bin/sh"}) if err != nil { err = fmt.Errorf("unable to run command %s. stdout: %s, err: %s", command, stdOut, err) - scopedLog.Error(err, "Failed to set probe level", "Command", command) + scopedLog.ErrorContext(ctx, "Failed to set probe level", "Command", command, "error", err) return err } - scopedLog.Info("Successfully set probe level on pod", "Command", command) + scopedLog.InfoContext(ctx, "Successfully set probe level on pod", "Command", command) return err } From 168d8b00c9962760aa6407d89f598ee1fb609afa Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Tue, 31 Mar 2026 15:00:44 +0200 Subject: [PATCH 11/19] Update lefrover calls --- .../controller/clustermanager_controller.go | 2 +- .../controller/clustermaster_controller.go | 2 +- .../controller/indexercluster_controller.go | 2 +- .../controller/ingestorcluster_controller.go | 2 +- .../controller/licensemanager_controller.go | 2 +- .../controller/licensemaster_controller.go | 2 +- .../monitoringconsole_controller.go | 2 +- .../searchheadcluster_controller.go | 2 +- internal/controller/standalone_controller.go | 2 +- internal/controller/telemetry_controller.go | 2 +- pkg/splunk/client/awss3client.go | 37 +++++----- pkg/splunk/client/azureblobclient.go | 43 ++++++----- pkg/splunk/client/gcpbucketclient.go | 27 ++++--- pkg/splunk/client/minioclient.go | 35 +++++---- pkg/splunk/client/remotedataclient.go | 7 +- pkg/splunk/client/util.go | 14 ++-- pkg/splunk/enterprise/clustermanager.go | 48 ++++++------- pkg/splunk/enterprise/clustermaster.go | 39 +++++----- pkg/splunk/enterprise/configuration.go | 72 +++++++++---------- pkg/splunk/enterprise/events.go | 7 +- pkg/splunk/enterprise/licensemanager.go | 23 +++--- pkg/splunk/enterprise/licensemaster.go | 12 ++-- pkg/splunk/enterprise/monitoringconsole.go | 14 ++-- pkg/splunk/enterprise/searchheadcluster.go | 49 ++++++------- pkg/splunk/enterprise/telemetry.go | 71 +++++++++--------- pkg/splunk/enterprise/upgrade.go | 23 +++--- pkg/splunk/enterprise/util_test.go | 6 +- pkg/splunk/enterprise/validation/server.go | 14 ++-- pkg/splunk/splkcontroller/configmap.go | 13 ++-- pkg/splunk/splkcontroller/controller.go | 13 ++-- pkg/splunk/splkcontroller/deployment.go | 15 ++-- pkg/splunk/splkcontroller/finalizers.go | 23 +++--- pkg/splunk/splkcontroller/secret.go | 16 ++--- pkg/splunk/splkcontroller/service.go | 9 ++- pkg/splunk/splkcontroller/serviceaccount.go | 12 ++-- pkg/splunk/splkcontroller/statefulset.go | 53 +++++++------- pkg/splunk/splkcontroller/util.go | 66 ++++++++--------- pkg/splunk/util/util.go | 27 ++++--- 38 files changed, 367 insertions(+), 441 deletions(-) diff --git a/internal/controller/clustermanager_controller.go b/internal/controller/clustermanager_controller.go index ec3626f05..438ffdf00 100644 --- a/internal/controller/clustermanager_controller.go +++ b/internal/controller/clustermanager_controller.go @@ -80,7 +80,7 @@ func (r *ClusterManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterManager") - logger := slog.Default().With("controller", "ClusterManager", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "ClusterManager", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the ClusterManager diff --git a/internal/controller/clustermaster_controller.go b/internal/controller/clustermaster_controller.go index 4b14eede1..371459681 100644 --- a/internal/controller/clustermaster_controller.go +++ b/internal/controller/clustermaster_controller.go @@ -80,7 +80,7 @@ func (r *ClusterMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "ClusterMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "ClusterMaster") - logger := slog.Default().With("controller", "ClusterMaster", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "ClusterMaster", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the ClusterMaster diff --git a/internal/controller/indexercluster_controller.go b/internal/controller/indexercluster_controller.go index 975e31114..f73d783e1 100644 --- a/internal/controller/indexercluster_controller.go +++ b/internal/controller/indexercluster_controller.go @@ -80,7 +80,7 @@ func (r *IndexerClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "IndexerCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "IndexerCluster") - logger := slog.Default().With("controller", "IndexerCluster", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "IndexerCluster", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the IndexerCluster diff --git a/internal/controller/ingestorcluster_controller.go b/internal/controller/ingestorcluster_controller.go index 7a553c943..3a32f7e1a 100644 --- a/internal/controller/ingestorcluster_controller.go +++ b/internal/controller/ingestorcluster_controller.go @@ -69,7 +69,7 @@ func (r *IngestorClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "IngestorCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "IngestorCluster") - logger := slog.Default().With("controller", "IngestorCluster", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "IngestorCluster", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the IngestorCluster diff --git a/internal/controller/licensemanager_controller.go b/internal/controller/licensemanager_controller.go index 41f759864..6bff14d7f 100644 --- a/internal/controller/licensemanager_controller.go +++ b/internal/controller/licensemanager_controller.go @@ -78,7 +78,7 @@ func (r *LicenseManagerReconciler) Reconcile(ctx context.Context, req ctrl.Reque metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseManager")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseManager") - logger := slog.Default().With("controller", "LicenseManager", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "LicenseManager", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the LicenseManager diff --git a/internal/controller/licensemaster_controller.go b/internal/controller/licensemaster_controller.go index 545c344e1..348deb11e 100644 --- a/internal/controller/licensemaster_controller.go +++ b/internal/controller/licensemaster_controller.go @@ -79,7 +79,7 @@ func (r *LicenseMasterReconciler) Reconcile(ctx context.Context, req ctrl.Reques metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "LicenseMaster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "LicenseMaster") - logger := slog.Default().With("controller", "LicenseMaster", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "LicenseMaster", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the LicenseMaster diff --git a/internal/controller/monitoringconsole_controller.go b/internal/controller/monitoringconsole_controller.go index 5507d9802..8841fe813 100644 --- a/internal/controller/monitoringconsole_controller.go +++ b/internal/controller/monitoringconsole_controller.go @@ -79,7 +79,7 @@ func (r *MonitoringConsoleReconciler) Reconcile(ctx context.Context, req ctrl.Re metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "MonitoringConsole")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "MonitoringConsole") - logger := slog.Default().With("controller", "MonitoringConsole", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "MonitoringConsole", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the MonitoringConsole diff --git a/internal/controller/searchheadcluster_controller.go b/internal/controller/searchheadcluster_controller.go index 439389e77..1cb694086 100644 --- a/internal/controller/searchheadcluster_controller.go +++ b/internal/controller/searchheadcluster_controller.go @@ -78,7 +78,7 @@ func (r *SearchHeadClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "SearchHeadCluster")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "SearchHeadCluster") - logger := slog.Default().With("controller", "SearchHeadCluster", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "SearchHeadCluster", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the SearchHeadCluster diff --git a/internal/controller/standalone_controller.go b/internal/controller/standalone_controller.go index 2c69730ae..7c92d48ed 100644 --- a/internal/controller/standalone_controller.go +++ b/internal/controller/standalone_controller.go @@ -83,7 +83,7 @@ func (r *StandaloneReconciler) Reconcile(ctx context.Context, req ctrl.Request) metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "Standalone")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "Standalone") - logger := slog.Default().With("controller", "Standalone", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "Standalone", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) // Fetch the Standalone diff --git a/internal/controller/telemetry_controller.go b/internal/controller/telemetry_controller.go index 22e9fd43e..09acd1bde 100644 --- a/internal/controller/telemetry_controller.go +++ b/internal/controller/telemetry_controller.go @@ -54,7 +54,7 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( metrics.ReconcileCounters.With(metrics.GetPrometheusLabels(req, "Telemetry")).Inc() defer recordInstrumentionData(time.Now(), req, "controller", "Telemetry") - logger := slog.Default().With("controller", "Telemetry", "name", req.Name, "namespace", req.Namespace) + logger := slog.Default().With("controller", "Telemetry", "name", req.Name, "namespace", req.Namespace, "reconcileID", controller.ReconcileIDFromContext(ctx)) ctx = logging.WithLogger(ctx, logger) logger.InfoContext(ctx, "Reconciling telemetry") diff --git a/pkg/splunk/client/awss3client.go b/pkg/splunk/client/awss3client.go index b35d989c8..cdf955b0d 100644 --- a/pkg/splunk/client/awss3client.go +++ b/pkg/splunk/client/awss3client.go @@ -32,7 +32,7 @@ import ( "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/splunk/splunk-operator/pkg/logging" ) // blank assignment to verify that AWSS3Client implements RemoteDataClient @@ -83,8 +83,7 @@ func InitAWSClientWrapper(ctx context.Context, region, accessKeyID, secretAccess // InitAWSClientConfig initializes and returns a client config object func InitAWSClientConfig(ctx context.Context, regionWithEndpoint, accessKeyID, secretAccessKey string) SplunkAWSS3Client { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("InitAWSClientConfig") + scopedLog := logging.FromContext(ctx).With("func", "InitAWSClientConfig") // Enforcing minimum version TLS1.2 tr := &http.Transport{ @@ -105,8 +104,8 @@ func InitAWSClientConfig(ctx context.Context, regionWithEndpoint, accessKeyID, s // Extract region and endpoint regEndSl := strings.Split(regionWithEndpoint, awsRegionEndPointDelimiter) if len(regEndSl) != 2 || strings.Count(regionWithEndpoint, awsRegionEndPointDelimiter) != 1 { - scopedLog.Error(err, "Unable to extract region and endpoint correctly for AWS client", - "regWithEndpoint", regionWithEndpoint) + scopedLog.ErrorContext(ctx, "Unable to extract region and endpoint correctly for AWS client", + "regWithEndpoint", regionWithEndpoint, "error", err) return nil } region = regEndSl[0] @@ -123,7 +122,7 @@ func InitAWSClientConfig(ctx context.Context, regionWithEndpoint, accessKeyID, s "")), // token ) } else { - scopedLog.Info("No valid access/secret keys. Attempt to connect without them") + scopedLog.InfoContext(ctx, "No valid access/secret keys. Attempt to connect without them") cfg, err = config.LoadDefaultConfig(ctx, config.WithRegion(region), config.WithRetryMaxAttempts(3), @@ -131,7 +130,7 @@ func InitAWSClientConfig(ctx context.Context, regionWithEndpoint, accessKeyID, s ) } if err != nil { - scopedLog.Error(err, "Failed to initialize an AWS S3 config.") + scopedLog.ErrorContext(ctx, "Failed to initialize an AWS S3 config.", "error", err) return nil } s3Client := s3.NewFromConfig(cfg, func(o *s3.Options) { @@ -144,7 +143,7 @@ func InitAWSClientConfig(ctx context.Context, regionWithEndpoint, accessKeyID, s tlsVersion = getTLSVersion(tr) } - scopedLog.Info("AWS Client Config initialization successful.", "region", region, "TLS Version", tlsVersion) + scopedLog.InfoContext(ctx, "AWS Client Config initialization successful.", "region", region, "TLS Version", tlsVersion) return s3Client } @@ -211,10 +210,9 @@ func getTLSVersion(tr *http.Transport) string { // GetAppsList get the list of apps from remote storage func (awsclient *AWSS3Client) GetAppsList(ctx context.Context) (RemoteDataListResponse, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetAppsList") + scopedLog := logging.FromContext(ctx).With("func", "GetAppsList") - scopedLog.Info("Getting Apps list", "AWS S3 Bucket", awsclient.BucketName) + scopedLog.InfoContext(ctx, "Getting Apps list", "AWS S3 Bucket", awsclient.BucketName) remoteDataClientResponse := RemoteDataListResponse{} options := &s3.ListObjectsV2Input{ @@ -228,24 +226,24 @@ func (awsclient *AWSS3Client) GetAppsList(ctx context.Context) (RemoteDataListRe client := awsclient.Client resp, err := client.ListObjectsV2(ctx, options) if err != nil { - scopedLog.Error(err, "Unable to list items in bucket", "AWS S3 Bucket", awsclient.BucketName, "endpoint", awsclient.Endpoint) + scopedLog.ErrorContext(ctx, "Unable to list items in bucket", "AWS S3 Bucket", awsclient.BucketName, "endpoint", awsclient.Endpoint, "error", err) return remoteDataClientResponse, err } if resp.Contents == nil { - scopedLog.Info("empty objects list in bucket. No apps to install", "bucketName", awsclient.BucketName) + scopedLog.InfoContext(ctx, "empty objects list in bucket. No apps to install", "bucketName", awsclient.BucketName) return remoteDataClientResponse, nil } tmp, err := json.Marshal(resp.Contents) if err != nil { - scopedLog.Error(err, "Failed to marshal s3 response", "AWS S3 Bucket", awsclient.BucketName) + scopedLog.ErrorContext(ctx, "Failed to marshal s3 response", "AWS S3 Bucket", awsclient.BucketName, "error", err) return remoteDataClientResponse, err } err = json.Unmarshal(tmp, &(remoteDataClientResponse.Objects)) if err != nil { - scopedLog.Error(err, "Failed to unmarshal s3 response", "AWS S3 Bucket", awsclient.BucketName) + scopedLog.ErrorContext(ctx, "Failed to unmarshal s3 response", "AWS S3 Bucket", awsclient.BucketName, "error", err) return remoteDataClientResponse, err } @@ -254,14 +252,13 @@ func (awsclient *AWSS3Client) GetAppsList(ctx context.Context) (RemoteDataListRe // DownloadApp downloads the app from remote storage to local file system func (awsclient *AWSS3Client) DownloadApp(ctx context.Context, downloadRequest RemoteDataDownloadRequest) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DownloadApp").WithValues("remoteFile", downloadRequest.RemoteFile, "localFile", + scopedLog := logging.FromContext(ctx).With("func", "DownloadApp", "remoteFile", downloadRequest.RemoteFile, "localFile", downloadRequest.LocalFile, "etag", downloadRequest.Etag) var numBytes int64 file, err := os.Create(downloadRequest.LocalFile) if err != nil { - scopedLog.Error(err, "Unable to open local file") + scopedLog.ErrorContext(ctx, "Unable to open local file", "error", err) return false, err } defer file.Close() @@ -274,12 +271,12 @@ func (awsclient *AWSS3Client) DownloadApp(ctx context.Context, downloadRequest R IfMatch: aws.String(downloadRequest.Etag), }) if err != nil { - scopedLog.Error(err, "Unable to download item", "RemoteFile", downloadRequest.RemoteFile) + scopedLog.ErrorContext(ctx, "Unable to download item", "RemoteFile", downloadRequest.RemoteFile, "error", err) os.Remove(downloadRequest.RemoteFile) return false, err } - scopedLog.Info("File downloaded", "numBytes: ", numBytes) + scopedLog.InfoContext(ctx, "File downloaded", "numBytes", numBytes) return true, err } diff --git a/pkg/splunk/client/azureblobclient.go b/pkg/splunk/client/azureblobclient.go index c74036269..61b618332 100644 --- a/pkg/splunk/client/azureblobclient.go +++ b/pkg/splunk/client/azureblobclient.go @@ -26,7 +26,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/splunk/splunk-operator/pkg/logging" ) var _ RemoteDataClient = &AzureBlobClient{} @@ -122,10 +122,9 @@ func NewAzureBlobClient( endpoint string, // Custom endpoint (optional) initFunc GetInitFunc, // Initialization function ) (RemoteDataClient, error) { // Matches GetRemoteDataClient signature - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("NewAzureBlobClient") + scopedLog := logging.FromContext(ctx).With("func", "NewAzureBlobClient") - scopedLog.Info("Initializing AzureBlobClient") + scopedLog.InfoContext(ctx, "Initializing AzureBlobClient") // Execute the initialization function if provided. if initFunc != nil { @@ -152,12 +151,12 @@ func NewAzureBlobClient( if secretAccessKey != "" { // Use Shared Key authentication. - scopedLog.Info("Using Shared Key authentication") + scopedLog.InfoContext(ctx, "Using Shared Key authentication") // Create a Shared Key Credential. sharedKeyCredential, err := azblob.NewSharedKeyCredential(storageAccountName, secretAccessKey) if err != nil { - scopedLog.Error(err, "Failed to create SharedKeyCredential") + scopedLog.ErrorContext(ctx, "Failed to create SharedKeyCredential", "error", err) return nil, fmt.Errorf("failed to create SharedKeyCredential: %w", err) } @@ -168,7 +167,7 @@ func NewAzureBlobClient( nil, ) if err != nil { - scopedLog.Error(err, "Failed to create ContainerClient with SharedKeyCredential") + scopedLog.ErrorContext(ctx, "Failed to create ContainerClient with SharedKeyCredential", "error", err) return nil, fmt.Errorf("failed to create ContainerClient with SharedKeyCredential: %w", err) } @@ -178,7 +177,7 @@ func NewAzureBlobClient( credentialType = CredentialTypeSharedKey } else { // Use Azure AD authentication. - scopedLog.Info("Using Azure AD authentication") + scopedLog.InfoContext(ctx, "Using Azure AD authentication") // Create a Token Credential using DefaultAzureCredential. // The Azure SDK uses environment variables to configure authentication when using DefaultAzureCredential. @@ -200,7 +199,7 @@ func NewAzureBlobClient( tokenCredential, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { - scopedLog.Error(err, "Failed to create DefaultAzureCredential") + scopedLog.ErrorContext(ctx, "Failed to create DefaultAzureCredential", "error", err) return nil, fmt.Errorf("failed to create DefaultAzureCredential: %w", err) } @@ -211,7 +210,7 @@ func NewAzureBlobClient( nil, ) if err != nil { - scopedLog.Error(err, "Failed to create ContainerClient with TokenCredential") + scopedLog.ErrorContext(ctx, "Failed to create ContainerClient with TokenCredential", "error", err) return nil, fmt.Errorf("failed to create ContainerClient with TokenCredential: %w", err) } @@ -221,7 +220,7 @@ func NewAzureBlobClient( credentialType = CredentialTypeAzureAD } - scopedLog.Info("AzureBlobClient initialized successfully", + scopedLog.InfoContext(ctx, "AzureBlobClient initialized successfully", "CredentialType", credentialType, "BucketName", bucketName, "StorageAccountName", storageAccountName, @@ -240,10 +239,9 @@ func NewAzureBlobClient( // GetAppsList retrieves a list of blobs (apps) from the Azure Blob container. func (client *AzureBlobClient) GetAppsList(ctx context.Context) (RemoteDataListResponse, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("AzureBlob:GetAppsList").WithValues("Bucket", client.BucketName) + scopedLog := logging.FromContext(ctx).With("func", "AzureBlob:GetAppsList", "Bucket", client.BucketName) - scopedLog.Info("Fetching list of apps") + scopedLog.InfoContext(ctx, "Fetching list of apps") // Define options for listing blobs. options := &container.ListBlobsFlatOptions{ @@ -262,7 +260,7 @@ func (client *AzureBlobClient) GetAppsList(ctx context.Context) (RemoteDataListR for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { - scopedLog.Error(err, "Error listing blobs") + scopedLog.ErrorContext(ctx, "Error listing blobs", "error", err) return RemoteDataListResponse{}, fmt.Errorf("error listing blobs: %w", err) } @@ -282,21 +280,20 @@ func (client *AzureBlobClient) GetAppsList(ctx context.Context) (RemoteDataListR } } - scopedLog.Info("Successfully fetched list of apps", "TotalBlobs", len(blobs)) + scopedLog.InfoContext(ctx, "Successfully fetched list of apps", "TotalBlobs", len(blobs)) return RemoteDataListResponse{Objects: blobs}, nil } // DownloadApp downloads a specific blob from Azure Blob Storage to a local file. func (client *AzureBlobClient) DownloadApp(ctx context.Context, downloadRequest RemoteDataDownloadRequest) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("AzureBlob:DownloadApp").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "AzureBlob:DownloadApp", "Bucket", client.BucketName, "RemoteFile", downloadRequest.RemoteFile, "LocalFile", downloadRequest.LocalFile, ) - scopedLog.Info("Initiating blob download") + scopedLog.InfoContext(ctx, "Initiating blob download") // Create a blob client for the specific blob. blobClient := client.ContainerClient.NewBlobClient(downloadRequest.RemoteFile) @@ -304,7 +301,7 @@ func (client *AzureBlobClient) DownloadApp(ctx context.Context, downloadRequest // Download the blob content. get, err := blobClient.DownloadStream(ctx, nil) if err != nil { - scopedLog.Error(err, "Failed to download blob") + scopedLog.ErrorContext(ctx, "Failed to download blob", "error", err) return false, fmt.Errorf("failed to download blob: %w", err) } defer get.Body.Close() @@ -312,7 +309,7 @@ func (client *AzureBlobClient) DownloadApp(ctx context.Context, downloadRequest // Create or truncate the local file. localFile, err := os.Create(downloadRequest.LocalFile) if err != nil { - scopedLog.Error(err, "Failed to create local file") + scopedLog.ErrorContext(ctx, "Failed to create local file", "error", err) return false, fmt.Errorf("failed to create local file: %w", err) } defer localFile.Close() @@ -320,11 +317,11 @@ func (client *AzureBlobClient) DownloadApp(ctx context.Context, downloadRequest // Write the content to the local file. _, err = io.Copy(localFile, get.Body) if err != nil { - scopedLog.Error(err, "Failed to write blob content to local file") + scopedLog.ErrorContext(ctx, "Failed to write blob content to local file", "error", err) return false, fmt.Errorf("failed to write blob content to local file: %w", err) } - scopedLog.Info("Blob downloaded successfully") + scopedLog.InfoContext(ctx, "Blob downloaded successfully") return true, nil } diff --git a/pkg/splunk/client/gcpbucketclient.go b/pkg/splunk/client/gcpbucketclient.go index 1bda36d08..cfa26df6c 100644 --- a/pkg/splunk/client/gcpbucketclient.go +++ b/pkg/splunk/client/gcpbucketclient.go @@ -23,9 +23,9 @@ import ( "cloud.google.com/go/storage" //"golang.org/x/oauth2/google" + "github.com/splunk/splunk-operator/pkg/logging" "google.golang.org/api/iterator" "google.golang.org/api/option" - "sigs.k8s.io/controller-runtime/pkg/log" ) // blank assignment to verify that GCSClient implements RemoteDataClient @@ -109,8 +109,7 @@ type GCSClient struct { // InitGCSClient initializes and returns a GCS client implementing GCSClientInterface func InitGCSClient(ctx context.Context, gcpCredentials string) (GCSClientInterface, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("InitGCSClient") + scopedLog := logging.FromContext(ctx).With("func", "InitGCSClient") var client *storage.Client var err error @@ -134,11 +133,11 @@ func InitGCSClient(ctx context.Context, gcpCredentials string) (GCSClientInterfa } if err != nil { - scopedLog.Error(err, "Failed to initialize a GCS client.") + scopedLog.ErrorContext(ctx, "Failed to initialize a GCS client.", "error", err) return nil, err } - scopedLog.Info("GCS Client initialization successful.") + scopedLog.InfoContext(ctx, "GCS Client initialization successful.") return &GCSClientWrapper{Client: client}, nil } @@ -175,10 +174,9 @@ func RegisterGCSClient() { // GetAppsList gets the list of apps from remote storage func (gcsClient *GCSClient) GetAppsList(ctx context.Context) (RemoteDataListResponse, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetAppsList") + scopedLog := logging.FromContext(ctx).With("func", "GetAppsList") - scopedLog.Info("Getting Apps list", "GCS Bucket", gcsClient.BucketName) + scopedLog.InfoContext(ctx, "Getting Apps list", "GCS Bucket", gcsClient.BucketName) remoteDataClientResponse := RemoteDataListResponse{} query := &storage.Query{ @@ -202,7 +200,7 @@ func (gcsClient *GCSClient) GetAppsList(ctx context.Context) (RemoteDataListResp break } if err != nil { - scopedLog.Error(err, "Error fetching object from GCS", "GCS Bucket", gcsClient.BucketName) + scopedLog.ErrorContext(ctx, "Error fetching object from GCS", "GCS Bucket", gcsClient.BucketName, "error", err) return remoteDataClientResponse, err } @@ -234,13 +232,12 @@ func (gcsClient *GCSClient) GetAppsList(ctx context.Context) (RemoteDataListResp // DownloadApp downloads the app from remote storage to the local file system func (gcsClient *GCSClient) DownloadApp(ctx context.Context, downloadRequest RemoteDataDownloadRequest) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DownloadApp").WithValues("remoteFile", downloadRequest.RemoteFile, "localFile", + scopedLog := logging.FromContext(ctx).With("func", "DownloadApp", "remoteFile", downloadRequest.RemoteFile, "localFile", downloadRequest.LocalFile, "etag", downloadRequest.Etag) file, err := os.Create(downloadRequest.LocalFile) if err != nil { - scopedLog.Error(err, "Unable to open local file") + scopedLog.ErrorContext(ctx, "Unable to open local file", "error", err) return false, err } defer file.Close() @@ -248,18 +245,18 @@ func (gcsClient *GCSClient) DownloadApp(ctx context.Context, downloadRequest Rem objHandle := gcsClient.BucketHandle.Object(downloadRequest.RemoteFile) reader, err := objHandle.NewReader(ctx) if err != nil { - scopedLog.Error(err, "Unable to download item", "RemoteFile", downloadRequest.RemoteFile) + scopedLog.ErrorContext(ctx, "Unable to download item", "RemoteFile", downloadRequest.RemoteFile, "error", err) os.Remove(downloadRequest.LocalFile) return false, err } defer reader.Close() if _, err := io.Copy(file, reader); err != nil { - scopedLog.Error(err, "Unable to copy data to local file") + scopedLog.ErrorContext(ctx, "Unable to copy data to local file", "error", err) return false, err } - scopedLog.Info("File downloaded") + scopedLog.InfoContext(ctx, "File downloaded") return true, nil } diff --git a/pkg/splunk/client/minioclient.go b/pkg/splunk/client/minioclient.go index 7d1269594..72b9e3306 100644 --- a/pkg/splunk/client/minioclient.go +++ b/pkg/splunk/client/minioclient.go @@ -27,7 +27,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/splunk/splunk-operator/pkg/logging" ) // blank assignment to verify that MinioClient implements RemoteDataClient @@ -88,27 +88,26 @@ func InitMinioClientWrapper(ctx context.Context, appS3Endpoint string, accessKey // InitMinioClientSession initializes and returns a client session object func InitMinioClientSession(ctx context.Context, appS3Endpoint string, accessKeyID string, secretAccessKey string) SplunkMinioClient { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("InitMinioClientSession") + scopedLog := logging.FromContext(ctx).With("func", "InitMinioClientSession") // Check if SSL is needed useSSL := true if strings.HasPrefix(appS3Endpoint, "http://") { // We should always use a secure SSL endpoint, so we won't set useSSL = false - scopedLog.Info("Using insecure endpoint, useSSL=false for Minio Client Session", "appS3Endpoint", appS3Endpoint) + scopedLog.InfoContext(ctx, "Using insecure endpoint, useSSL=false for Minio Client Session", "appS3Endpoint", appS3Endpoint) appS3Endpoint = strings.TrimPrefix(appS3Endpoint, "http://") useSSL = false } else if strings.HasPrefix(appS3Endpoint, "https://") { appS3Endpoint = strings.TrimPrefix(appS3Endpoint, "https://") } else { // Unsupported endpoint - scopedLog.Info("Unsupported endpoint for Minio S3 client", "appS3Endpoint", appS3Endpoint) + scopedLog.InfoContext(ctx, "Unsupported endpoint for Minio S3 client", "appS3Endpoint", appS3Endpoint) return nil } // New returns an Minio compatible client object. API compatibility (v2 or v4) is automatically // determined based on the Endpoint value. - scopedLog.Info("Connecting to Minio S3 for apps", "appS3Endpoint", appS3Endpoint) + scopedLog.InfoContext(ctx, "Connecting to Minio S3 for apps", "appS3Endpoint", appS3Endpoint) var s3Client *minio.Client var err error @@ -130,12 +129,12 @@ func InitMinioClientSession(ctx context.Context, appS3Endpoint string, accessKey if accessKeyID != "" && secretAccessKey != "" { options.Creds = credentials.NewStaticV4(accessKeyID, secretAccessKey, "") } else { - scopedLog.Info("No Access/Secret Keys, attempt connection without them using IAM", "appS3Endpoint", appS3Endpoint) + scopedLog.InfoContext(ctx, "No Access/Secret Keys, attempt connection without them using IAM", "appS3Endpoint", appS3Endpoint) options.Creds = credentials.NewIAM("") } s3Client, err = minio.New(appS3Endpoint, options) if err != nil { - scopedLog.Info("Error creating new Minio Client Session", "err", err) + scopedLog.InfoContext(ctx, "Error creating new Minio Client Session", "err", err) return nil } @@ -144,10 +143,9 @@ func InitMinioClientSession(ctx context.Context, appS3Endpoint string, accessKey // GetAppsList get the list of apps from remote storage func (client *MinioClient) GetAppsList(ctx context.Context) (RemoteDataListResponse, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetAppsList") + scopedLog := logging.FromContext(ctx).With("func", "GetAppsList") - scopedLog.Info("Getting Apps list", " S3 Bucket", client.BucketName, "Prefix", client.Prefix) + scopedLog.InfoContext(ctx, "Getting Apps list", "S3 Bucket", client.BucketName, "Prefix", client.Prefix) remoteDataClientResponse := RemoteDataListResponse{} s3Client := client.Client @@ -164,7 +162,7 @@ func (client *MinioClient) GetAppsList(ctx context.Context) (RemoteDataListRespo err := fmt.Errorf("got an object error: %v for bucket: %s", object.Err, client.BucketName) return remoteDataClientResponse, err } - scopedLog.Info("Got an object", "object", object) + scopedLog.InfoContext(ctx, "Got an object", "object", object) // Create a new object to add to append to the response newETag := object.ETag @@ -181,13 +179,12 @@ func (client *MinioClient) GetAppsList(ctx context.Context) (RemoteDataListRespo // DownloadApp downloads an app package from remote storage func (client *MinioClient) DownloadApp(ctx context.Context, downloadRequest RemoteDataDownloadRequest) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DownloadApp").WithValues("remoteFile", downloadRequest.RemoteFile, - downloadRequest.LocalFile, downloadRequest.Etag) + scopedLog := logging.FromContext(ctx).With("func", "DownloadApp", "remoteFile", downloadRequest.RemoteFile, + "localFile", downloadRequest.LocalFile, "etag", downloadRequest.Etag) file, err := os.Create(downloadRequest.LocalFile) if err != nil { - scopedLog.Error(err, "Unable to create local file") + scopedLog.ErrorContext(ctx, "Unable to create local file", "error", err) return false, err } defer file.Close() @@ -197,17 +194,17 @@ func (client *MinioClient) DownloadApp(ctx context.Context, downloadRequest Remo options := minio.GetObjectOptions{} // set the option to match the specified etag on remote storage if err = options.SetMatchETag(downloadRequest.Etag); err != nil { - scopedLog.Error(err, "Unable to set match etag") + scopedLog.ErrorContext(ctx, "Unable to set match etag", "error", err) return false, err } err = s3Client.FGetObject(ctx, client.BucketName, downloadRequest.RemoteFile, downloadRequest.LocalFile, options) if err != nil { - scopedLog.Error(err, "Unable to download remote file") + scopedLog.ErrorContext(ctx, "Unable to download remote file", "error", err) return false, err } - scopedLog.Info("File downloaded") + scopedLog.InfoContext(ctx, "File downloaded") return true, nil } diff --git a/pkg/splunk/client/remotedataclient.go b/pkg/splunk/client/remotedataclient.go index 3120622ab..c973a10c2 100644 --- a/pkg/splunk/client/remotedataclient.go +++ b/pkg/splunk/client/remotedataclient.go @@ -19,7 +19,7 @@ import ( "context" "time" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/splunk/splunk-operator/pkg/logging" ) // RemoteDataClientsMap is a map of remote storage provider name to @@ -113,8 +113,7 @@ type SplunkRemoteDataClient struct { // RegisterRemoteDataClient registers the respective Client func RegisterRemoteDataClient(ctx context.Context, provider string) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("RegisterRemoteDataClient") + scopedLog := logging.FromContext(ctx).With("func", "RegisterRemoteDataClient") switch provider { case "aws": RegisterAWSS3Client() @@ -125,6 +124,6 @@ func RegisterRemoteDataClient(ctx context.Context, provider string) { case "gcp": RegisterGCSClient() default: - scopedLog.Error(nil, "Invalid provider specified", "provider", provider) + scopedLog.ErrorContext(ctx, "Invalid provider specified", "provider", provider) } } diff --git a/pkg/splunk/client/util.go b/pkg/splunk/client/util.go index 9ceb5edda..0927b66c2 100644 --- a/pkg/splunk/client/util.go +++ b/pkg/splunk/client/util.go @@ -21,8 +21,8 @@ import ( "fmt" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" - "sigs.k8s.io/controller-runtime/pkg/log" ) // NewMockAWSS3Client returns an AWS S3 mock client for testing @@ -99,20 +99,19 @@ func NewMockAzureBlobClient(ctx context.Context, bucketName string, storageAccou // ConvertRemoteDataListResponse converts S3 Response to a mock client response func ConvertRemoteDataListResponse(ctx context.Context, RemoteDataListResponse RemoteDataListResponse) (spltest.MockRemoteDataClient, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ConvertRemoteDataListResponse") + scopedLog := logging.FromContext(ctx).With("func", "ConvertRemoteDataListResponse") var mockResponse spltest.MockRemoteDataClient tmp, err := json.Marshal(RemoteDataListResponse) if err != nil { - scopedLog.Error(err, "Unable to marshal s3 response") + scopedLog.ErrorContext(ctx, "Unable to marshal s3 response", "error", err) return mockResponse, err } err = json.Unmarshal(tmp, &mockResponse) if err != nil { - scopedLog.Error(err, "Unable to unmarshal s3 response") + scopedLog.ErrorContext(ctx, "Unable to unmarshal s3 response", "error", err) return mockResponse, err } @@ -137,8 +136,7 @@ func GetAppSrcVolume(ctx context.Context, appSource enterpriseApi.AppSourceSpec, var err error var vol enterpriseApi.VolumeSpec - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetAppSrcVolume") + scopedLog := logging.FromContext(ctx).With("func", "GetAppSrcVolume") // get the volume spec from the volume name if appSource.VolName != "" { @@ -149,7 +147,7 @@ func GetAppSrcVolume(ctx context.Context, appSource enterpriseApi.AppSourceSpec, index, err = CheckIfVolumeExists(appFrameworkRef.VolList, volName) if err != nil { - scopedLog.Error(err, "Invalid volume name provided. Please specify a valid volume name.", "App source", appSource.Name, "Volume name", volName) + scopedLog.ErrorContext(ctx, "Invalid volume name provided. Please specify a valid volume name.", "App source", appSource.Name, "Volume name", volName, "error", err) return vol, err } diff --git a/pkg/splunk/enterprise/clustermanager.go b/pkg/splunk/enterprise/clustermanager.go index e572facec..7b236c6e7 100644 --- a/pkg/splunk/enterprise/clustermanager.go +++ b/pkg/splunk/enterprise/clustermanager.go @@ -21,10 +21,12 @@ import ( "reflect" "time" + "log/slog" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" rclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/go-logr/logr" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -34,7 +36,6 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -47,8 +48,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyClusterManager") + logger := logging.FromContext(ctx).With("func", "ApplyClusterManager") eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -220,7 +220,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } // Create podExecClient (use injected one if provided, otherwise create real one) @@ -266,7 +266,7 @@ func ApplyClusterManager(ctx context.Context, client splcommon.ControllerClient, // clusterManagerPodManager is used to manage the cluster manager pod type clusterManagerPodManager struct { - log logr.Logger + log *slog.Logger cr *enterpriseApi.ClusterManager secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient @@ -319,8 +319,7 @@ func getClusterManagerStatefulSet(ctx context.Context, client splcommon.Controll // CheckIfsmartstoreConfigMapUpdatedToPod checks if the smartstore configMap is updated on Pod or not func CheckIfsmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.ClusterManager, podExecClient splutil.PodExecClientImpl) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("CheckIfsmartstoreConfigMapUpdatedToPod").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "CheckIfsmartstoreConfigMapUpdatedToPod", "name", cr.GetName(), "namespace", cr.GetNamespace()) eventPublisher := GetEventPublisher(ctx, cr) @@ -337,7 +336,7 @@ func CheckIfsmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.Con if smartStoreConfigMap != nil { tokenFromConfigMap := smartStoreConfigMap.Data[configToken] if tokenFromConfigMap == stdOut { - scopedLog.Info("Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) + logger.InfoContext(ctx, "Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) return nil } eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) @@ -356,8 +355,7 @@ var PerformCmBundlePush = func(ctx context.Context, c splcommon.ControllerClient return nil } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PerformCmBundlePush").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "PerformCmBundlePush", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Reconciler can be called for multiple reasons. If we are waiting on configMap update to happen, // do not increment the Retry Count unless the last check was 5 seconds ago. // This helps, to wait for the required time @@ -367,7 +365,7 @@ var PerformCmBundlePush = func(ctx context.Context, c splcommon.ControllerClient return fmt.Errorf("will re-attempt to push the bundle after the 5 seconds period passed from last check. LastCheckInterval=%d, current epoch=%d", cr.Status.BundlePushTracker.LastCheckInterval, currentEpoch) } - scopedLog.Info("Attempting to push the bundle") + logger.InfoContext(ctx, "Attempting to push the bundle") cr.Status.BundlePushTracker.LastCheckInterval = currentEpoch // The amount of time it takes for the configMap update to Pod depends on @@ -394,7 +392,7 @@ var PerformCmBundlePush = func(ctx context.Context, c splcommon.ControllerClient err = PushManagerAppsBundle(ctx, c, cr) if err == nil { - scopedLog.Info("Bundle push success") + logger.InfoContext(ctx, "Bundle push success") cr.Status.BundlePushTracker.NeedToPushManagerApps = false } @@ -403,8 +401,7 @@ var PerformCmBundlePush = func(ctx context.Context, c splcommon.ControllerClient // PushManagerAppsBundle issues the REST command to for cluster manager bundle push func PushManagerAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PushManagerApps").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "PushManagerApps", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -422,7 +419,7 @@ func PushManagerAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr return fmt.Errorf("could not find admin password while trying to push the manager apps bundle") } - scopedLog.Info("Issuing REST call to push manager aps bundle") + logger.InfoContext(ctx, "Issuing REST call to push manager aps bundle") managerIdxcName := cr.GetName() fqdnName := splcommon.GetServiceFQDN(cr.GetNamespace(), GetSplunkServiceName(SplunkClusterManager, managerIdxcName, false)) @@ -435,8 +432,7 @@ func PushManagerAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr // helper function to get the list of ClusterManager types in the current namespace func getClusterManagerList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []rclient.ListOption) (int, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getClusterManagerList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getClusterManagerList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.ClusterManagerList{} @@ -444,7 +440,7 @@ func getClusterManagerList(ctx context.Context, c splcommon.ControllerClient, cr numOfObjects := len(objectList.Items) if err != nil { - scopedLog.Error(err, "ClusterManager types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "ClusterManager types not found in namespace", "error", err, "namsespace", cr.GetNamespace()) return numOfObjects, err } @@ -455,16 +451,15 @@ func getClusterManagerList(ctx context.Context, c splcommon.ControllerClient, cr // If it fails to connect to the cluster manager (e.g., pod not ready yet), it returns basic env vars as fallback // This function is used also in mock tests var GetCMMultisiteEnvVarsCall = func(ctx context.Context, cr *enterpriseApi.ClusterManager, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("Verify if Multisite Indexer Cluster").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "GetCMMultisiteEnvVars", "name", cr.GetName(), "namespace", cr.GetNamespace()) extraEnv := getClusterManagerExtraEnv(cr, &cr.Spec.CommonSplunkSpec) - mgr := clusterManagerPodManager{log: scopedLog, cr: cr, secrets: namespaceScopedSecret, newSplunkClient: splclient.NewSplunkClient} + mgr := clusterManagerPodManager{log: logger, cr: cr, secrets: namespaceScopedSecret, newSplunkClient: splclient.NewSplunkClient} cm := mgr.getClusterManagerClient(cr) clusterInfo, err := cm.GetClusterInfo(false) if err != nil { - scopedLog.Error(err, "Failed to get cluster info from cluster manager pod, using basic environment variables") + logger.ErrorContext(ctx, "Failed to get cluster info from cluster manager pod, using basic environment variables", "error", err) return extraEnv, err } @@ -480,8 +475,7 @@ var GetCMMultisiteEnvVarsCall = func(ctx context.Context, cr *enterpriseApi.Clus // changeClusterManagerAnnotations updates the splunk/image-tag field of the ClusterManager annotations to trigger the reconcile loop // on update, and returns error if something is wrong func changeClusterManagerAnnotations(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApi.LicenseManager) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName(EventReasonAnnotationUpdateFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "changeClusterManagerAnnotations", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -533,13 +527,13 @@ func changeClusterManagerAnnotations(ctx context.Context, c splcommon.Controller image, err := getCurrentImage(ctx, c, cr, SplunkLicenseManager) if err != nil { eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not get the LicenseManager Image. Reason %v", err)) - scopedLog.Error(err, "Get LicenseManager Image failed with", "error", err) + logger.ErrorContext(ctx, "Get LicenseManager Image failed with", "error", err) return err } err = changeAnnotations(ctx, c, image, clusterManagerInstance) if err != nil { eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not update annotations. Reason %v", err)) - scopedLog.Error(err, "ClusterManager types update after changing annotations failed with", "error", err) + logger.ErrorContext(ctx, "ClusterManager types update after changing annotations failed with", "error", err) return err } diff --git a/pkg/splunk/enterprise/clustermaster.go b/pkg/splunk/enterprise/clustermaster.go index 42bacf374..9c17b7e83 100644 --- a/pkg/splunk/enterprise/clustermaster.go +++ b/pkg/splunk/enterprise/clustermaster.go @@ -21,10 +21,12 @@ import ( "reflect" "time" + "log/slog" + enterpriseApi "github.com/splunk/splunk-operator/api/v4" - "github.com/go-logr/logr" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -33,7 +35,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -45,8 +46,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyClusterMaster") + logger := logging.FromContext(ctx).With("func", "ApplyClusterMaster") eventPublisher := GetEventPublisher(ctx, cr) cr.Kind = "ClusterMaster" @@ -209,7 +209,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } // Create podExecClient @@ -247,7 +247,7 @@ func ApplyClusterMaster(ctx context.Context, client splcommon.ControllerClient, // clusterMasterPodMaster is used to manage the cluster manager pod type clusterMasterPodManager struct { - log logr.Logger + log *slog.Logger cr *enterpriseApiV3.ClusterMaster secrets *corev1.Secret newSplunkClient func(managementURI, username, password string) *splclient.SplunkClient @@ -300,8 +300,7 @@ func getClusterMasterStatefulSet(ctx context.Context, client splcommon.Controlle // CheckIfMastersmartstoreConfigMapUpdatedToPod checks if the smartstore configMap is updated on Pod or not func CheckIfMastersmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApiV3.ClusterMaster, podExecClient splutil.PodExecClientImpl) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("CheckIfMastersmartstoreConfigMapUpdatedToPod").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "CheckIfMastersmartstoreConfigMapUpdatedToPod", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -319,7 +318,7 @@ func CheckIfMastersmartstoreConfigMapUpdatedToPod(ctx context.Context, c splcomm if smartStoreConfigMap != nil { tokenFromConfigMap := smartStoreConfigMap.Data[configToken] if tokenFromConfigMap == stdOut { - scopedLog.Info("Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) + logger.InfoContext(ctx, "Token Matched.", "on Pod=", stdOut, "from configMap=", tokenFromConfigMap) return nil } eventPublisher.Warning(ctx, EventReasonSmartStoreConfigPending, fmt.Sprintf("waiting for the configMap update to the Pod. Token on Pod=%s, Token from configMap=%s", stdOut, tokenFromConfigMap)) @@ -338,8 +337,7 @@ var PerformCmasterBundlePush = func(ctx context.Context, c splcommon.ControllerC return nil } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PerformCmasterBundlePush").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "PerformCmasterBundlePush", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Reconciler can be called for multiple reasons. If we are waiting on configMap update to happen, // do not increment the Retry Count unless the last check was 5 seconds ago. // This helps, to wait for the required time @@ -349,7 +347,7 @@ var PerformCmasterBundlePush = func(ctx context.Context, c splcommon.ControllerC return fmt.Errorf("will re-attempt to push the bundle after the 5 seconds period passed from last check. LastCheckInterval=%d, current epoch=%d", cr.Status.BundlePushTracker.LastCheckInterval, currentEpoch) } - scopedLog.Info("Attempting to push the bundle") + logger.InfoContext(ctx, "Attempting to push the bundle") cr.Status.BundlePushTracker.LastCheckInterval = currentEpoch // The amount of time it takes for the configMap update to Pod depends on @@ -373,7 +371,7 @@ var PerformCmasterBundlePush = func(ctx context.Context, c splcommon.ControllerC err = PushMasterAppsBundle(ctx, c, cr) if err == nil { - scopedLog.Info("Bundle push success") + logger.InfoContext(ctx, "Bundle push success") cr.Status.BundlePushTracker.NeedToPushMasterApps = false } @@ -382,8 +380,7 @@ var PerformCmasterBundlePush = func(ctx context.Context, c splcommon.ControllerC // PushMasterAppsBundle issues the REST command to for cluster manager bundle push func PushMasterAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr *enterpriseApiV3.ClusterMaster) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PushMasterApps").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "PushMasterApps", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -402,7 +399,7 @@ func PushMasterAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr return fmt.Errorf("could not find admin password while trying to push the manager apps bundle") } - scopedLog.Info("Issuing REST call to push manager aps bundle") + logger.InfoContext(ctx, "Issuing REST call to push manager aps bundle") managerIdxcName := cr.GetName() fqdnName := splcommon.GetServiceFQDN(cr.GetNamespace(), GetSplunkServiceName(SplunkClusterMaster, managerIdxcName, false)) @@ -415,8 +412,7 @@ func PushMasterAppsBundle(ctx context.Context, c splcommon.ControllerClient, cr // helper function to get the list of ClusterMaster types in the current namespace func getClusterMasterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []client.ListOption) (int, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getClusterMasterList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getClusterMasterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApiV3.ClusterMasterList{} @@ -424,7 +420,7 @@ func getClusterMasterList(ctx context.Context, c splcommon.ControllerClient, cr numOfObjects := len(objectList.Items) if err != nil { - scopedLog.Error(err, "ClusterMaster types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "ClusterMaster types not found in namespace", "error", err, "namsespace", cr.GetNamespace()) return numOfObjects, err } @@ -435,9 +431,8 @@ func getClusterMasterList(ctx context.Context, c splcommon.ControllerClient, cr // Defined as a variable to allow mocking in unit tests var VerifyCMasterisMultisite = func(ctx context.Context, cr *enterpriseApiV3.ClusterMaster, namespaceScopedSecret *corev1.Secret) ([]corev1.EnvVar, error) { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("Verify if Multisite Indexer Cluster").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) - mgr := clusterMasterPodManager{log: scopedLog, cr: cr, secrets: namespaceScopedSecret, newSplunkClient: splclient.NewSplunkClient} + logger := logging.FromContext(ctx).With("func", "VerifyCMasterisMultisite", "name", cr.GetName(), "namespace", cr.GetNamespace()) + mgr := clusterMasterPodManager{log: logger, cr: cr, secrets: namespaceScopedSecret, newSplunkClient: splclient.NewSplunkClient} cm := mgr.getClusterMasterClient(cr) clusterInfo, err := cm.GetClusterInfo(false) if err != nil { diff --git a/pkg/splunk/enterprise/configuration.go b/pkg/splunk/enterprise/configuration.go index f3d52bf14..3e56c5153 100644 --- a/pkg/splunk/enterprise/configuration.go +++ b/pkg/splunk/enterprise/configuration.go @@ -33,7 +33,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/log" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" @@ -1376,8 +1375,7 @@ func getManualUpdateRefCount(ctx context.Context, client splcommon.ControllerCli // createOrUpdateAppUpdateConfigMap creates or updates the manual app update configMap func createOrUpdateAppUpdateConfigMap(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject) (*corev1.ConfigMap, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("createOrUpdateAppUpdateConfigMap").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "createOrUpdateAppUpdateConfigMap", "name", cr.GetName(), "namespace", cr.GetNamespace()) var crKindMap map[string]string var configMapData, status string @@ -1405,7 +1403,7 @@ func createOrUpdateAppUpdateConfigMap(ctx context.Context, client splcommon.Cont } } - scopedLog.Info("existing configMap data", "data", configMap.Data) + logger.InfoContext(ctx, "existing configMap data", "data", configMap.Data) crKindMap = configMap.Data // get the number of instance types of this kind @@ -1431,7 +1429,7 @@ refCount: %d`, status, numOfObjects+1) // Create/update the configMap to store the values of manual trigger per CR kind. configMap, err = ApplyManualAppUpdateConfigMap(ctx, client, cr, crKindMap) if err != nil { - scopedLog.Error(err, "create/update configMap for app update failed") + logger.ErrorContext(ctx, "create/update configMap for app update failed", "error", err) return configMap, err } @@ -1640,28 +1638,27 @@ func ValidateAppFrameworkSpec(ctx context.Context, appFramework *enterpriseApi.A return nil } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ValidateAppFrameworkSpec") + logger := logging.FromContext(ctx).With("func", "ValidateAppFrameworkSpec") - scopedLog.Info("configCheck", "scope", localScope) + logger.InfoContext(ctx, "configCheck", "scope", localScope) // Set the value in status field to be same as that in spec. appContext.AppsRepoStatusPollInterval = appFramework.AppsRepoPollInterval appContext.AppsStatusMaxConcurrentAppDownloads = appFramework.MaxConcurrentAppDownloads if appContext.AppsRepoStatusPollInterval <= 0 { - scopedLog.Error(err, "appsRepoPollIntervalSeconds is not configured. Disabling polling of apps repo changes, defaulting to manual updates") + logger.ErrorContext(ctx, "appsRepoPollIntervalSeconds is not configured. Disabling polling of apps repo changes, defaulting to manual updates", "error", err) appContext.AppsRepoStatusPollInterval = 0 } else if appFramework.AppsRepoPollInterval < splcommon.MinAppsRepoPollInterval { - scopedLog.Error(err, "configured appsRepoPollIntervalSeconds is too small", "configured value", appFramework.AppsRepoPollInterval, "Setting it to the default min. value(seconds)", splcommon.MinAppsRepoPollInterval) + logger.ErrorContext(ctx, "configured appsRepoPollIntervalSeconds is too small", "error", err, "configured value", appFramework.AppsRepoPollInterval, "Setting it to the default min. value(seconds)", splcommon.MinAppsRepoPollInterval) appContext.AppsRepoStatusPollInterval = splcommon.MinAppsRepoPollInterval } else if appFramework.AppsRepoPollInterval > splcommon.MaxAppsRepoPollInterval { - scopedLog.Error(err, "configured appsRepoPollIntervalSeconds is too large", "configured value", appFramework.AppsRepoPollInterval, "Setting it to the default max. value(seconds)", splcommon.MaxAppsRepoPollInterval, "seconds", nil) + logger.ErrorContext(ctx, "configured appsRepoPollIntervalSeconds is too large", "error", err, "configured value", appFramework.AppsRepoPollInterval, "Setting it to the default max. value(seconds)", splcommon.MaxAppsRepoPollInterval) appContext.AppsRepoStatusPollInterval = splcommon.MaxAppsRepoPollInterval } if appContext.AppsStatusMaxConcurrentAppDownloads <= 0 { - scopedLog.Info("Invalid value of maxConcurrentAppDownloads", "configured value", appContext.AppsStatusMaxConcurrentAppDownloads, "Setting it to default value", splcommon.DefaultMaxConcurrentAppDownloads) + logger.InfoContext(ctx, "Invalid value of maxConcurrentAppDownloads", "configured value", appContext.AppsStatusMaxConcurrentAppDownloads, "Setting it to default value", splcommon.DefaultMaxConcurrentAppDownloads) appContext.AppsStatusMaxConcurrentAppDownloads = splcommon.DefaultMaxConcurrentAppDownloads } @@ -1670,7 +1667,7 @@ func ValidateAppFrameworkSpec(ctx context.Context, appFramework *enterpriseApi.A // check whether the temporary volume to download apps is mounted or not on the operator pod if _, err := os.Stat(appDownloadVolume); errors.Is(err, os.ErrNotExist) { - scopedLog.Error(err, "Volume needs to be mounted on operator pod to download apps. Please mount it as a separate volume on operator pod.", "volume path", appDownloadVolume) + logger.ErrorContext(ctx, "Volume needs to be mounted on operator pod to download apps. Please mount it as a separate volume on operator pod.", "error", err, "volume path", appDownloadVolume) return err } @@ -1681,7 +1678,7 @@ func ValidateAppFrameworkSpec(ctx context.Context, appFramework *enterpriseApi.A err = validateSplunkAppSources(appFramework, localScope, crKind) if err == nil { - scopedLog.Info("App framework configuration is valid") + logger.InfoContext(ctx, "App framework configuration is valid") } return err @@ -1692,8 +1689,7 @@ func validateRemoteVolumeSpec(ctx context.Context, volList []enterpriseApi.Volum duplicateChecker := make(map[string]bool) - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("validateRemoteVolumeSpec") + logger := logging.FromContext(ctx).With("func", "validateRemoteVolumeSpec") // Make sure that all the Volumes are provided with the mandatory config values. for i, volume := range volList { @@ -1713,7 +1709,7 @@ func validateRemoteVolumeSpec(ctx context.Context, volList []enterpriseApi.Volum } // Make the secretRef optional if theyre using IAM roles if volume.SecretRef == "" { - scopedLog.Info("No valid SecretRef for volume.", "volumeName", volume.Name) + logger.InfoContext(ctx, "No valid SecretRef for volume.", "volumeName", volume.Name) } // provider is used in App framework to pick the S3 client(supported providers are aws and minio), @@ -1821,8 +1817,7 @@ func ValidateSplunkSmartstoreSpec(ctx context.Context, smartstore *enterpriseApi func GetSmartstoreVolumesConfig(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, smartstore *enterpriseApi.SmartStoreSpec, mapData map[string]string) (string, error) { var volumesConf string - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetSmartstoreVolumesConfig") + logger := logging.FromContext(ctx).With("func", "GetSmartstoreVolumesConfig") volumes := smartstore.VolList for i := 0; i < len(volumes); i++ { @@ -1842,7 +1837,7 @@ remote.s3.endpoint = %s remote.s3.auth_region = %s `, volumesConf, volumes[i].Name, volumes[i].Path, s3AccessKey, s3SecretKey, volumes[i].Endpoint, volumes[i].Region) } else { - scopedLog.Info("No valid secretRef configured. Configure volume without access/secret keys", "volumeName", volumes[i].Name) + logger.InfoContext(ctx, "No valid secretRef configured. Configure volume without access/secret keys", "volumeName", volumes[i].Name) volumesConf = fmt.Sprintf(`%s [volume:%s] storageType = remote @@ -2007,11 +2002,10 @@ func validateProbe(probe *enterpriseApi.Probe) error { // validateLivenessProbe validates the liveness probe config func validateLivenessProbe(ctx context.Context, cr splcommon.MetaObject, livenessProbe *enterpriseApi.Probe) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("validateLivenessProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "validateLivenessProbe", "name", cr.GetName(), "namespace", cr.GetNamespace()) if livenessProbe == nil { - scopedLog.Info("empty liveness probe.") + logger.InfoContext(ctx, "empty liveness probe.") return err } @@ -2021,19 +2015,19 @@ func validateLivenessProbe(ctx context.Context, cr splcommon.MetaObject, livenes } if livenessProbe.InitialDelaySeconds != 0 && livenessProbe.InitialDelaySeconds < livenessProbeDefaultDelaySec { - scopedLog.Info("Liveness Probe: Configured InitialDelaySeconds is too small, recommended default minimum will be used", "configured", livenessProbe.InitialDelaySeconds, "recommended minimum", livenessProbeDefaultDelaySec) + logger.InfoContext(ctx, "Liveness Probe: Configured InitialDelaySeconds is too small, recommended default minimum will be used", "configured", livenessProbe.InitialDelaySeconds, "recommended minimum", livenessProbeDefaultDelaySec) } if livenessProbe.TimeoutSeconds != 0 && livenessProbe.TimeoutSeconds < livenessProbeTimeoutSec { - scopedLog.Info("Liveness Probe: Configured TimeoutSeconds is too small, recommended default minimum will be used", "configured", livenessProbe.TimeoutSeconds, "recommended minimum", livenessProbeTimeoutSec) + logger.InfoContext(ctx, "Liveness Probe: Configured TimeoutSeconds is too small, recommended default minimum will be used", "configured", livenessProbe.TimeoutSeconds, "recommended minimum", livenessProbeTimeoutSec) } if livenessProbe.PeriodSeconds != 0 && livenessProbe.PeriodSeconds < livenessProbePeriodSec { - scopedLog.Info("Liveness Probe: Configured PeriodSeconds is too small, recommended default minimum will be used", "configured", livenessProbe.PeriodSeconds, "recommended minimum", livenessProbePeriodSec) + logger.InfoContext(ctx, "Liveness Probe: Configured PeriodSeconds is too small, recommended default minimum will be used", "configured", livenessProbe.PeriodSeconds, "recommended minimum", livenessProbePeriodSec) } if livenessProbe.FailureThreshold != 0 && livenessProbe.FailureThreshold < livenessProbeFailureThreshold { - scopedLog.Info("Liveness Probe: Configured FailureThreshold is too small, recommended default minimum will be used", "configured", livenessProbe.FailureThreshold, "recommended minimum", livenessProbeFailureThreshold) + logger.InfoContext(ctx, "Liveness Probe: Configured FailureThreshold is too small, recommended default minimum will be used", "configured", livenessProbe.FailureThreshold, "recommended minimum", livenessProbeFailureThreshold) } return err @@ -2042,11 +2036,10 @@ func validateLivenessProbe(ctx context.Context, cr splcommon.MetaObject, livenes // validateReadinessProbe validates the Readiness probe config func validateReadinessProbe(ctx context.Context, cr splcommon.MetaObject, readinessProbe *enterpriseApi.Probe) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("validateReadinessProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "validateReadinessProbe", "name", cr.GetName(), "namespace", cr.GetNamespace()) if readinessProbe == nil { - scopedLog.Info("empty readiness probe.") + logger.InfoContext(ctx, "empty readiness probe.") return err } @@ -2056,19 +2049,19 @@ func validateReadinessProbe(ctx context.Context, cr splcommon.MetaObject, readin } if readinessProbe.InitialDelaySeconds != 0 && readinessProbe.InitialDelaySeconds < readinessProbeDefaultDelaySec { - scopedLog.Info("Readiness Probe: Configured InitialDelaySeconds is too small, recommended default minimum will be used", "configured", readinessProbe.InitialDelaySeconds, "recommended minimum", readinessProbeDefaultDelaySec) + logger.InfoContext(ctx, "Readiness Probe: Configured InitialDelaySeconds is too small, recommended default minimum will be used", "configured", readinessProbe.InitialDelaySeconds, "recommended minimum", readinessProbeDefaultDelaySec) } if readinessProbe.TimeoutSeconds != 0 && readinessProbe.TimeoutSeconds < readinessProbeTimeoutSec { - scopedLog.Info("Readiness Probe: Configured TimeoutSeconds is too small, recommended default minimum will be used", "configured", readinessProbe.TimeoutSeconds, "recommended minimum", readinessProbeTimeoutSec) + logger.InfoContext(ctx, "Readiness Probe: Configured TimeoutSeconds is too small, recommended default minimum will be used", "configured", readinessProbe.TimeoutSeconds, "recommended minimum", readinessProbeTimeoutSec) } if readinessProbe.PeriodSeconds != 0 && readinessProbe.PeriodSeconds < readinessProbePeriodSec { - scopedLog.Info("Readiness Probe: Configured PeriodSeconds is too small, recommended default minimum will be used", "configured", readinessProbe.PeriodSeconds, "recommended minimum", readinessProbePeriodSec) + logger.InfoContext(ctx, "Readiness Probe: Configured PeriodSeconds is too small, recommended default minimum will be used", "configured", readinessProbe.PeriodSeconds, "recommended minimum", readinessProbePeriodSec) } if readinessProbe.FailureThreshold != 0 && readinessProbe.FailureThreshold < readinessProbeFailureThreshold { - scopedLog.Info("Readiness Probe: Configured FailureThreshold is too small, recommended default minimum will be used", "configured", readinessProbe.FailureThreshold, "recommended minimum", readinessProbeFailureThreshold) + logger.InfoContext(ctx, "Readiness Probe: Configured FailureThreshold is too small, recommended default minimum will be used", "configured", readinessProbe.FailureThreshold, "recommended minimum", readinessProbeFailureThreshold) } return err } @@ -2076,11 +2069,10 @@ func validateReadinessProbe(ctx context.Context, cr splcommon.MetaObject, readin // validateStartupProbe validates the startup probe config func validateStartupProbe(ctx context.Context, cr splcommon.MetaObject, startupProbe *enterpriseApi.Probe) error { var err error - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("validateStartupProbe").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "validateStartupProbe", "name", cr.GetName(), "namespace", cr.GetNamespace()) if startupProbe == nil { - scopedLog.Info("empty startup probe.") + logger.InfoContext(ctx, "empty startup probe.") return err } @@ -2090,15 +2082,15 @@ func validateStartupProbe(ctx context.Context, cr splcommon.MetaObject, startupP } if startupProbe.InitialDelaySeconds != 0 && startupProbe.InitialDelaySeconds < startupProbeDefaultDelaySec { - scopedLog.Info("Startup Probe: InitialDelaySeconds is too small, recommended default minimum will be used", "configured", startupProbe.InitialDelaySeconds, "recommended minimum", startupProbeDefaultDelaySec) + logger.InfoContext(ctx, "Startup Probe: InitialDelaySeconds is too small, recommended default minimum will be used", "configured", startupProbe.InitialDelaySeconds, "recommended minimum", startupProbeDefaultDelaySec) } if startupProbe.TimeoutSeconds != 0 && startupProbe.TimeoutSeconds < startupProbeTimeoutSec { - scopedLog.Info("Startup Probe: TimeoutSeconds is too small, recommended default minimum will be used", "configured", startupProbe.TimeoutSeconds, "recommended minimum", startupProbeTimeoutSec) + logger.InfoContext(ctx, "Startup Probe: TimeoutSeconds is too small, recommended default minimum will be used", "configured", startupProbe.TimeoutSeconds, "recommended minimum", startupProbeTimeoutSec) } if startupProbe.PeriodSeconds != 0 && startupProbe.PeriodSeconds < startupProbePeriodSec { - scopedLog.Info("Startup Probe: PeriodSeconds is too small, recommended default minimum will be used", "configured", startupProbe.PeriodSeconds, "recommended minimum", startupProbePeriodSec) + logger.InfoContext(ctx, "Startup Probe: PeriodSeconds is too small, recommended default minimum will be used", "configured", startupProbe.PeriodSeconds, "recommended minimum", startupProbePeriodSec) } return err } diff --git a/pkg/splunk/enterprise/events.go b/pkg/splunk/enterprise/events.go index d1f7e721a..7066d71f8 100644 --- a/pkg/splunk/enterprise/events.go +++ b/pkg/splunk/enterprise/events.go @@ -18,10 +18,10 @@ package enterprise import ( "context" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/log" ) // K8EventPublisher structure used to publish k8s event @@ -56,9 +56,8 @@ func (k *K8EventPublisher) publishEvent(ctx context.Context, eventType, reason, return } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("PublishEvent") - scopedLog.Info("publishing event", "eventType", eventType, "reason", reason, "message", message) + logger := logging.FromContext(ctx).With("func", "PublishEvent") + logger.InfoContext(ctx, "publishing event", "eventType", eventType, "reason", reason, "message", message) // Use the EventRecorder to emit the event k.recorder.Event(k.instance, eventType, reason, message) diff --git a/pkg/splunk/enterprise/licensemanager.go b/pkg/splunk/enterprise/licensemanager.go index a00b277a5..6798f8416 100644 --- a/pkg/splunk/enterprise/licensemanager.go +++ b/pkg/splunk/enterprise/licensemanager.go @@ -22,6 +22,7 @@ import ( "time" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" @@ -29,7 +30,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" @@ -47,8 +47,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyLicenseManager") + logger := logging.FromContext(ctx).With("func", "ApplyLicenseManager") eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -169,7 +168,7 @@ func ApplyLicenseManager(ctx context.Context, client splcommon.ControllerClient, namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } // Add a splunk operator telemetry app @@ -231,8 +230,7 @@ func validateLicenseManagerSpec(ctx context.Context, c splcommon.ControllerClien // checkLicenseRelatedPodFailures checks license status via Splunk API // and publishes warning event when expired license is detected func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.LicenseManager, statefulSet *appsv1.StatefulSet) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("checkLicenseRelatedPodFailures") + logger := logging.FromContext(ctx).With("func", "checkLicenseRelatedPodFailures") eventPublisher := GetEventPublisher(ctx, cr) replicas := int32(1) @@ -247,13 +245,13 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro var pod corev1.Pod err := client.Get(ctx, namespacedName, &pod) if err != nil { - scopedLog.Info("Pod not found, skipping license check", "podName", podName) + logger.InfoContext(ctx, "Pod not found, skipping license check", "podName", podName) continue } // Only check license if pod is running if pod.Status.Phase != corev1.PodRunning { - scopedLog.Info("Pod not in running state, skipping license check", "podName", podName, "phase", pod.Status.Phase) + logger.InfoContext(ctx, "Pod not in running state, skipping license check", "podName", podName, "phase", pod.Status.Phase) continue } @@ -276,7 +274,7 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro // Get license information from Splunk API licenses, err := splunkClient.GetLicenseInfo() if err != nil { - scopedLog.Error(err, "Failed to get license information from Splunk API", "podName", podName) + logger.ErrorContext(ctx, "Failed to get license information from Splunk API", "error", err, "podName", podName) continue } @@ -285,7 +283,7 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro if licenseInfo.Status == "EXPIRED" { eventPublisher.Warning(ctx, EventReasonLicenseExpired, fmt.Sprintf("License '%s' has expired", licenseName)) - scopedLog.Error(nil, "Detected expired license", "licenseName", licenseName, "title", licenseInfo.Title) + logger.ErrorContext(ctx, "Detected expired license", "licenseName", licenseName, "title", licenseInfo.Title) } } } @@ -295,14 +293,13 @@ func checkLicenseRelatedPodFailures(ctx context.Context, client splcommon.Contro // helper function to get the list of LicenseManager types in the current namespace func getLicenseManagerList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []client.ListOption) (enterpriseApi.LicenseManagerList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getLicenseManagerList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getLicenseManagerList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.LicenseManagerList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "LicenseManager types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "LicenseManager types not found in namespace", "error", err, "namsespace", cr.GetNamespace()) return objectList, err } diff --git a/pkg/splunk/enterprise/licensemaster.go b/pkg/splunk/enterprise/licensemaster.go index 23720f299..8d72a1811 100644 --- a/pkg/splunk/enterprise/licensemaster.go +++ b/pkg/splunk/enterprise/licensemaster.go @@ -27,10 +27,10 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" @@ -44,8 +44,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyLicenseMaster") + logger := logging.FromContext(ctx).With("func", "ApplyLicenseMaster") eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -160,7 +159,7 @@ func ApplyLicenseMaster(ctx context.Context, client splcommon.ControllerClient, namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } // Add a splunk operator telemetry app @@ -214,8 +213,7 @@ func validateLicenseMasterSpec(ctx context.Context, c splcommon.ControllerClient // helper function to get the list of LicenseMaster types in the current namespace func getLicenseMasterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []client.ListOption) (int, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getLicenseMasterList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getLicenseMasterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApiV3.LicenseMasterList{} @@ -223,7 +221,7 @@ func getLicenseMasterList(ctx context.Context, c splcommon.ControllerClient, cr numOfObjects := len(objectList.Items) if err != nil { - scopedLog.Error(err, "LicenseMaster types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "LicenseMaster types not found in namespace", "error", err, "namsespace", cr.GetNamespace()) return numOfObjects, err } diff --git a/pkg/splunk/enterprise/monitoringconsole.go b/pkg/splunk/enterprise/monitoringconsole.go index af6f02039..3456f5896 100644 --- a/pkg/splunk/enterprise/monitoringconsole.go +++ b/pkg/splunk/enterprise/monitoringconsole.go @@ -25,6 +25,7 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" @@ -34,7 +35,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" rclient "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -203,14 +203,13 @@ func getMonitoringConsoleStatefulSet(ctx context.Context, client splcommon.Contr // helper function to get the list of MonitoringConsole types in the current namespace func getMonitoringConsoleList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []rclient.ListOption) (enterpriseApi.MonitoringConsoleList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getMonitoringConsoleList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getMonitoringConsoleList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.MonitoringConsoleList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "MonitoringConsole types not found in namespace", "namsespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "MonitoringConsole types not found in namespace", "error", err, "namsespace", cr.GetNamespace()) return objectList, err } @@ -370,8 +369,7 @@ func DeleteURLsConfigMap(revised *corev1.ConfigMap, crName string, newURLs []cor // changeMonitoringConsoleAnnotations updates the splunk/image-tag field of the MonitoringConsole annotations to trigger the reconcile loop // on update, and returns error if something is wrong. func changeMonitoringConsoleAnnotations(ctx context.Context, client splcommon.ControllerClient, cr *enterpriseApi.ClusterManager) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName(EventReasonAnnotationUpdateFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "changeMonitoringConsoleAnnotations", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) @@ -423,13 +421,13 @@ func changeMonitoringConsoleAnnotations(ctx context.Context, client splcommon.Co image, err := getCurrentImage(ctx, client, cr, SplunkClusterManager) if err != nil { eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not get the ClusterManager Image. Reason %v", err)) - scopedLog.Error(err, "Get ClusterManager Image failed with", "error", err) + logger.ErrorContext(ctx, "Get ClusterManager Image failed with", "error", err) return err } err = changeAnnotations(ctx, client, image, monitoringConsoleInstance) if err != nil { eventPublisher.Warning(ctx, EventReasonAnnotationUpdateFailed, fmt.Sprintf("Could not update annotations. Reason %v", err)) - scopedLog.Error(err, "MonitoringConsole types update after changing annotations failed with", "error", err) + logger.ErrorContext(ctx, "MonitoringConsole types update after changing annotations failed with", "error", err) return err } diff --git a/pkg/splunk/enterprise/searchheadcluster.go b/pkg/splunk/enterprise/searchheadcluster.go index 8dca63445..4f20923fd 100644 --- a/pkg/splunk/enterprise/searchheadcluster.go +++ b/pkg/splunk/enterprise/searchheadcluster.go @@ -25,6 +25,7 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splctrl "github.com/splunk/splunk-operator/pkg/splunk/splkcontroller" @@ -34,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/remotecommand" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -45,8 +45,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie Requeue: true, RequeueAfter: time.Second * 5, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplySearchHeadCluster") + logger := logging.FromContext(ctx).With("func", "ApplySearchHeadCluster") eventPublisher := GetEventPublisher(ctx, cr) ctx = context.WithValue(ctx, splcommon.EventPublisherKey, eventPublisher) @@ -222,7 +221,7 @@ func ApplySearchHeadCluster(ctx context.Context, client splcommon.ControllerClie namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: GetSplunkStatefulsetName(SplunkMonitoringConsole, cr.GetNamespace())} err = splctrl.DeleteReferencesToAutomatedMCIfExists(ctx, client, cr, namespacedName) if err != nil { - scopedLog.Error(err, "Error in deleting automated monitoring console resource") + logger.ErrorContext(ctx, "Error in deleting automated monitoring console resource", "error", err) } // Reset secrets related status structs @@ -267,13 +266,12 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli return err } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyShcSecret").WithValues("Desired replicas", replicas, "ShcSecretChanged", mgr.cr.Status.ShcSecretChanged, "AdminSecretChanged", mgr.cr.Status.AdminSecretChanged, "CrStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "NamespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) + logger := logging.FromContext(ctx).With("func", "ApplyShcSecret", "desiredReplicas", replicas, "shcSecretChanged", mgr.cr.Status.ShcSecretChanged, "adminSecretChanged", mgr.cr.Status.AdminSecretChanged, "crStatusNamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "namespaceSecretResourceVersion", namespaceSecret.GetObjectMeta().GetResourceVersion()) // If namespace scoped secret revision is the same ignore if len(mgr.cr.Status.NamespaceSecretResourceVersion) == 0 { // First time, set resource version in CR - scopedLog.Info("Setting CrStatusNamespaceSecretResourceVersion for the first time") + logger.InfoContext(ctx, "Setting CrStatusNamespaceSecretResourceVersion for the first time") mgr.cr.Status.NamespaceSecretResourceVersion = namespaceSecret.ObjectMeta.ResourceVersion return nil } else if mgr.cr.Status.NamespaceSecretResourceVersion == namespaceSecret.ObjectMeta.ResourceVersion { @@ -281,7 +279,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli return nil } - scopedLog.Info("Namespaced scoped secret revision has changed") + logger.InfoContext(ctx, "Namespaced scoped secret revision has changed") // Retrieve shc_secret password from secret data nsShcSecret := string(namespaceSecret.Data["shc_secret"]) @@ -295,8 +293,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli // Get search head pod's name shPodName := GetSplunkStatefulsetPodName(SplunkSearchHead, mgr.cr.GetName(), i) - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyShcSecretPodLoop").WithValues("Desired replicas", replicas, "ShcSecretChanged", mgr.cr.Status.ShcSecretChanged, "AdminSecretChanged", mgr.cr.Status.AdminSecretChanged, "NamespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "pod", shPodName) + podLogger := logging.FromContext(ctx).With("func", "ApplyShcSecretPodLoop", "desiredReplicas", replicas, "shcSecretChanged", mgr.cr.Status.ShcSecretChanged, "adminSecretChanged", mgr.cr.Status.AdminSecretChanged, "namespaceSecretResourceVersion", mgr.cr.Status.NamespaceSecretResourceVersion, "pod", shPodName) // Retrieve shc_secret password from Pod shcSecret, err := splutil.GetSpecificSecretTokenFromPod(ctx, mgr.c, shPodName, mgr.cr.GetNamespace(), "shc_secret") @@ -317,7 +314,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli // If shc secret is different from namespace scoped secret change it if shcSecret != nsShcSecret { - scopedLog.Info("shcSecret different from namespace scoped secret, changing shc secret") + podLogger.InfoContext(ctx, "shcSecret different from namespace scoped secret, changing shc secret") // If shc secret already changed, ignore if i < int32(len(mgr.cr.Status.ShcSecretChanged)) { if mgr.cr.Status.ShcSecretChanged[i] { @@ -338,7 +335,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli } return err } - scopedLog.Info("shcSecret changed") + podLogger.InfoContext(ctx, "shcSecret changed") howManyPodsHaveSecretChanged += 1 @@ -353,7 +350,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli } return err } - scopedLog.Info("Restarted Splunk") + podLogger.InfoContext(ctx, "Restarted Splunk") // Set the shc_secret changed flag to true if i < int32(len(mgr.cr.Status.ShcSecretChanged)) { @@ -365,7 +362,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli // If admin secret is different from namespace scoped secret change it if adminPwd != nsAdminSecret { - scopedLog.Info("admin password different from namespace scoped secret, changing admin password") + podLogger.InfoContext(ctx, "admin password different from namespace scoped secret, changing admin password") // If admin password already changed, ignore if i < int32(len(mgr.cr.Status.AdminSecretChanged)) { if mgr.cr.Status.AdminSecretChanged[i] { @@ -380,7 +377,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli if err != nil { return err } - scopedLog.Info("admin password changed on the splunk instance of pod") + podLogger.InfoContext(ctx, "admin password changed on the splunk instance of pod") // Get client for Pod and restart splunk instance on pod shClient := mgr.getClient(ctx, i) @@ -388,13 +385,13 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli if err != nil { return err } - scopedLog.Info("Restarted Splunk") + podLogger.InfoContext(ctx, "Restarted Splunk") // Set the adminSecretChanged changed flag to true if i < int32(len(mgr.cr.Status.AdminSecretChanged)) { mgr.cr.Status.AdminSecretChanged[i] = true } else { - scopedLog.Info("Appending to AdminSecretChanged") + podLogger.InfoContext(ctx, "Appending to AdminSecretChanged") mgr.cr.Status.AdminSecretChanged = append(mgr.cr.Status.AdminSecretChanged, true) } @@ -404,7 +401,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli return err } mgr.cr.Status.AdminPasswordChangedSecrets[podSecret.GetName()] = true - scopedLog.Info("Secret mounted on pod(to be changed) added to map") + podLogger.InfoContext(ctx, "Secret mounted on pod(to be changed) added to map") } } @@ -428,7 +425,7 @@ func ApplyShcSecret(ctx context.Context, mgr *searchHeadClusterPodManager, repli if err != nil { return err } - scopedLog.Info("admin password changed on the secret mounted on pod") + logger.InfoContext(ctx, "admin password changed on the secret mounted on pod") } } @@ -460,8 +457,7 @@ func getSearchHeadStatefulSet(ctx context.Context, client splcommon.ControllerCl // Use default otherwise // Make sure to set the resources ONLY for the deployer func setDeployerConfig(ctx context.Context, cr *enterpriseApi.SearchHeadCluster, podTemplate *corev1.PodTemplateSpec) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("setDeployerConfig").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "setDeployerConfig", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Break out if this is not a deployer if !strings.Contains("deployer", podTemplate.Labels["app.kubernetes.io/name"]) { @@ -471,19 +467,19 @@ func setDeployerConfig(ctx context.Context, cr *enterpriseApi.SearchHeadCluster, for i := range podTemplate.Spec.Containers { if len(depRes.Requests) != 0 { podTemplate.Spec.Containers[i].Resources.Requests = cr.Spec.DeployerResourceSpec.Requests - scopedLog.Info("Setting deployer resources requests", "requests", cr.Spec.DeployerResourceSpec.Requests) + logger.InfoContext(ctx, "Setting deployer resources requests", "requests", cr.Spec.DeployerResourceSpec.Requests) } if len(depRes.Limits) != 0 { podTemplate.Spec.Containers[i].Resources.Limits = cr.Spec.DeployerResourceSpec.Limits - scopedLog.Info("Setting deployer resources limits", "limits", cr.Spec.DeployerResourceSpec.Limits) + logger.InfoContext(ctx, "Setting deployer resources limits", "limits", cr.Spec.DeployerResourceSpec.Limits) } } // Add node affinity if configured if cr.Spec.DeployerNodeAffinity != nil { podTemplate.Spec.Affinity.NodeAffinity = cr.Spec.DeployerNodeAffinity - scopedLog.Info("Setting deployer node affinity", "nodeAffinity", cr.Spec.DeployerNodeAffinity) + logger.InfoContext(ctx, "Setting deployer node affinity", "nodeAffinity", cr.Spec.DeployerNodeAffinity) } return nil @@ -526,14 +522,13 @@ func validateSearchHeadClusterSpec(ctx context.Context, c splcommon.ControllerCl // helper function to get the list of SearchHeadCluster types in the current namespace func getSearchHeadClusterList(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, listOpts []client.ListOption) (enterpriseApi.SearchHeadClusterList, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getSearchHeadClusterList").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "getSearchHeadClusterList", "name", cr.GetName(), "namespace", cr.GetNamespace()) objectList := enterpriseApi.SearchHeadClusterList{} err := c.List(context.TODO(), &objectList, listOpts...) if err != nil { - scopedLog.Error(err, "SearchHeadCluster types not found in namespace", "namespace", cr.GetNamespace()) + logger.ErrorContext(ctx, "SearchHeadCluster types not found in namespace", "error", err, "namespace", cr.GetNamespace()) return objectList, err } diff --git a/pkg/splunk/enterprise/telemetry.go b/pkg/splunk/enterprise/telemetry.go index 3d356fc8e..93917fba8 100644 --- a/pkg/splunk/enterprise/telemetry.go +++ b/pkg/splunk/enterprise/telemetry.go @@ -5,15 +5,16 @@ import ( "encoding/json" "errors" "fmt" + "time" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "time" corev1 "k8s.io/api/core/v1" ) @@ -56,11 +57,10 @@ func ApplyTelemetry(ctx context.Context, client splcommon.ControllerClient, cm * RequeueAfter: time.Second * requeAfterInSeconds, } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyTelemetry") + logger := logging.FromContext(ctx).With("func", "ApplyTelemetry") for k, _ := range cm.Data { - scopedLog.Info("Retrieved telemetry keys", "key", k) + logger.InfoContext(ctx, "Retrieved telemetry keys", "key", k) } var data map[string]interface{} @@ -99,21 +99,20 @@ func ApplyTelemetry(ctx context.Context, client splcommon.ControllerClient, cm * } func updateLastTransmissionTime(ctx context.Context, client splcommon.ControllerClient, cm *corev1.ConfigMap, status *TelemetryStatus) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("updateLastTransmissionTime") + logger := logging.FromContext(ctx).With("func", "updateLastTransmissionTime") status.LastTransmission = time.Now().UTC().Format(time.RFC3339) updated, err := json.MarshalIndent(status, "", " ") if err != nil { - scopedLog.Error(err, "Failed to marshal telemetry status") + logger.ErrorContext(ctx, "Failed to marshal telemetry status", "error", err) return } cm.Data[telStatusKey] = string(updated) if err = client.Update(ctx, cm); err != nil { - scopedLog.Error(err, "Failed to update telemetry status in configmap") + logger.ErrorContext(ctx, "Failed to update telemetry status in configmap", "error", err) return } - scopedLog.Info("Updated last transmission time in configmap", "newStatus", cm.Data[telStatusKey]) + logger.InfoContext(ctx, "Updated last transmission time in configmap", "newStatus", cm.Data[telStatusKey]) } func collectResourceTelData(resources corev1.ResourceRequirements) map[string]string { @@ -178,13 +177,12 @@ type crListHandler struct { } func collectDeploymentTelData(ctx context.Context, client splcommon.ControllerClient, deploymentData map[string]interface{}) map[string][]splcommon.MetaObject { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("collectDeploymentTelData") + logger := logging.FromContext(ctx).With("func", "collectDeploymentTelData") var crWithTelAppList map[string][]splcommon.MetaObject crWithTelAppList = make(map[string][]splcommon.MetaObject) - scopedLog.Info("Start collecting deployment telemetry data") + logger.InfoContext(ctx, "Start collecting deployment telemetry data") // Define all CR handlers in a slice handlers := []crListHandler{ {kind: "Standalone", handlerFunc: handleStandalones, checkTelApp: true}, @@ -201,7 +199,7 @@ func collectDeploymentTelData(ctx context.Context, client splcommon.ControllerCl for _, handler := range handlers { data, crs, err := handler.handlerFunc(ctx, client) if err != nil { - scopedLog.Error(err, "Error processing CR type", "kind", handler.kind) + logger.ErrorContext(ctx, "Error processing CR type", "error", err, "kind", handler.kind) continue } if handler.checkTelApp && crs != nil && len(crs) > 0 { @@ -212,7 +210,7 @@ func collectDeploymentTelData(ctx context.Context, client splcommon.ControllerCl } } - scopedLog.Info("Successfully collected deployment telemetry data", "deploymentData", deploymentData) + logger.InfoContext(ctx, "Successfully collected deployment telemetry data", "deploymentData", deploymentData) return crWithTelAppList } @@ -393,30 +391,28 @@ func handleMonitoringConsoles(ctx context.Context, client splcommon.ControllerCl } func CollectCMTelData(ctx context.Context, cm *corev1.ConfigMap, data map[string]interface{}) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("collectCMTelData") - scopedLog.Info("Start") + logger := logging.FromContext(ctx).With("func", "collectCMTelData") + logger.InfoContext(ctx, "Start") for key, val := range cm.Data { if key == telStatusKey { continue } var compData interface{} - scopedLog.Info("Processing telemetry input from other components", "key", key) + logger.InfoContext(ctx, "Processing telemetry input from other components", "key", key) err := json.Unmarshal([]byte(val), &compData) if err != nil { - scopedLog.Info("Not able to unmarshal. Will include the input as string", "key", key, "value", val) + logger.InfoContext(ctx, "Not able to unmarshal. Will include the input as string", "key", key, "value", val) data[key] = val } else { data[key] = compData - scopedLog.Info("Got telemetry input", "key", key, "value", val) + logger.InfoContext(ctx, "Got telemetry input", "key", key, "value", val) } } } func getCurrentStatus(ctx context.Context, cm *corev1.ConfigMap) *TelemetryStatus { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("getCurrentStatus") + logger := logging.FromContext(ctx).With("func", "getCurrentStatus") defaultStatus := &TelemetryStatus{ LastTransmission: "", @@ -427,25 +423,24 @@ func getCurrentStatus(ctx context.Context, cm *corev1.ConfigMap) *TelemetryStatu var status TelemetryStatus err := json.Unmarshal([]byte(val), &status) if err != nil { - scopedLog.Error(err, "Failed to unmarshal telemetry status", "value", val) + logger.ErrorContext(ctx, "Failed to unmarshal telemetry status", "error", err, "value", val) return defaultStatus } else { - scopedLog.Info("Got current telemetry status from configmap", "status", status) + logger.InfoContext(ctx, "Got current telemetry status from configmap", "status", status) return &status } } - scopedLog.Info("No status set in configmap") + logger.InfoContext(ctx, "No status set in configmap") return defaultStatus } func SendTelemetry(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, data map[string]interface{}, test bool) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("sendTelemetry").WithValues( + logger := logging.FromContext(ctx).With("func", "sendTelemetry", "name", cr.GetObjectMeta().GetName(), "namespace", cr.GetObjectMeta().GetNamespace(), "kind", cr.GetObjectKind().GroupVersionKind().Kind) - scopedLog.Info("Start") + logger.InfoContext(ctx, "Start") var instanceID InstanceType switch cr.GetObjectKind().GroupVersionKind().Kind { @@ -462,25 +457,25 @@ func SendTelemetry(ctx context.Context, client splcommon.ControllerClient, cr sp case "ClusterManager": instanceID = SplunkClusterManager default: - scopedLog.Error(fmt.Errorf("unknown CR kind"), "Failed to determine instance type for telemetry") + logger.ErrorContext(ctx, "Failed to determine instance type for telemetry", "error", fmt.Errorf("unknown CR kind")) return false } serviceName := GetSplunkServiceName(instanceID, cr.GetName(), false) serviceFQDN := splcommon.GetServiceFQDN(cr.GetNamespace(), serviceName) - scopedLog.Info("Got service FQDN", "serviceFQDN", serviceFQDN) + logger.InfoContext(ctx, "Got service FQDN", "serviceFQDN", serviceFQDN) defaultSecretObjName := splcommon.GetNamespaceScopedSecretName(cr.GetNamespace()) defaultSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), defaultSecretObjName) if err != nil { - scopedLog.Error(err, "Could not access default secret object") + logger.ErrorContext(ctx, "Could not access default secret object", "error", err) return false } //Get the admin password from the secret object adminPwd, foundSecret := defaultSecret.Data["password"] if !foundSecret { - scopedLog.Info("Failed to find admin password") + logger.InfoContext(ctx, "Failed to find admin password") return false } splunkClient := splclient.NewSplunkClient(fmt.Sprintf("https://%s:8089", serviceFQDN), "admin", string(adminPwd)) @@ -488,7 +483,7 @@ func SendTelemetry(ctx context.Context, client splcommon.ControllerClient, cr sp var licenseInfo map[string]splclient.LicenseInfo licenseInfo, err = splunkClient.GetLicenseInfo() if err != nil { - scopedLog.Error(err, "Failed to retrieve the license info") + logger.ErrorContext(ctx, "Failed to retrieve the license info", "error", err) return false } else { data[telLicenseInfoKey] = licenseInfo @@ -504,17 +499,17 @@ func SendTelemetry(ctx context.Context, client splcommon.ControllerClient, cr sp path := fmt.Sprintf("/servicesNS/nobody/%s/telemetry-metric", telAppNameStr) bodyBytes, err := json.Marshal(telemetry) if err != nil { - scopedLog.Error(err, "Failed to marshal to bytes") + logger.ErrorContext(ctx, "Failed to marshal to bytes", "error", err) return false } - scopedLog.Info("Sending request", "path", path) + logger.InfoContext(ctx, "Sending request", "path", path) response, err := splunkClient.SendTelemetry(path, bodyBytes) if err != nil { - scopedLog.Error(err, "Failed to send telemetry") + logger.ErrorContext(ctx, "Failed to send telemetry", "error", err) return false } - scopedLog.Info("Successfully sent telemetry", "response", response) + logger.InfoContext(ctx, "Successfully sent telemetry", "response", response) return true } diff --git a/pkg/splunk/enterprise/upgrade.go b/pkg/splunk/enterprise/upgrade.go index 737ba5804..dbef5ab71 100644 --- a/pkg/splunk/enterprise/upgrade.go +++ b/pkg/splunk/enterprise/upgrade.go @@ -5,13 +5,13 @@ import ( "fmt" enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" runtime "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) // helps in mock function @@ -33,14 +33,13 @@ var GetClusterInfoCall = func(ctx context.Context, mgr *indexerClusterPodManager // function returns bool and error , true - go ahead with upgrade // false - exit the reconciliation loop with error func UpgradePathValidation(ctx context.Context, c splcommon.ControllerClient, cr splcommon.MetaObject, spec enterpriseApi.CommonSplunkSpec, mgr *indexerClusterPodManager) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName(EventReasonUpgradeCheckFailed).WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + logger := logging.FromContext(ctx).With("func", "UpgradePathValidation", "name", cr.GetName(), "namespace", cr.GetNamespace()) // Get event publisher from context eventPublisher := GetEventPublisher(ctx, cr) kind := cr.GroupVersionKind().Kind - scopedLog.Info("kind is set to ", "kind", kind) + logger.InfoContext(ctx, "kind is set to", "kind", kind) // start from standalone first goto Standalone @@ -78,7 +77,7 @@ LicenseManager: lmImage, err := getCurrentImage(ctx, c, licenseManager, SplunkLicenseManager) if err != nil { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the License Manager Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get licenseManager current image") + logger.ErrorContext(ctx, "Unable to get licenseManager current image", "error", err) return false, err } // if license manager status is ready and CR spec and current license manager image are not same @@ -125,7 +124,7 @@ ClusterManager: err := c.Get(ctx, namespacedName, clusterManager) if err != nil { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Cluster Manager. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager") + logger.ErrorContext(ctx, "Unable to get clusterManager", "error", err) goto SearchHeadCluster } @@ -133,7 +132,7 @@ ClusterManager: cmImage, err := getCurrentImage(ctx, c, clusterManager, SplunkClusterManager) if err != nil { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the Cluster Manager Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager current image") + logger.ErrorContext(ctx, "Unable to get clusterManager current image", "error", err) return false, err } @@ -255,7 +254,7 @@ SearchHeadCluster: shcImage, err := getCurrentImage(ctx, c, &searchHeadClusterInstance, SplunkSearchHead) if err != nil { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not get the Search Head Cluster Image. Reason %v", err)) - scopedLog.Error(err, "Unable to get SearchHeadCluster current image") + logger.ErrorContext(ctx, "Unable to get SearchHeadCluster current image", "error", err) return false, err } @@ -278,7 +277,7 @@ MonitoringConsole: err := c.List(ctx, clusterManagerList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Cluster Manager list. Reason %v", err)) - scopedLog.Error(err, "Unable to get clusterManager list") + logger.ErrorContext(ctx, "Unable to get clusterManager list", "error", err) return false, err } @@ -296,7 +295,7 @@ MonitoringConsole: err = c.List(ctx, searchHeadClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Search Head Cluster list. Reason %v", err)) - scopedLog.Error(err, "Unable to get Search Head Cluster list") + logger.ErrorContext(ctx, "Unable to get Search Head Cluster list", "error", err) return false, err } @@ -314,7 +313,7 @@ MonitoringConsole: err = c.List(ctx, indexerClusterList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Indexer list. Reason %v", err)) - scopedLog.Error(err, "Unable to get indexer cluster list") + logger.ErrorContext(ctx, "Unable to get indexer cluster list", "error", err) return false, err } @@ -332,7 +331,7 @@ MonitoringConsole: err = c.List(ctx, standaloneList, listOpts...) if err != nil && err.Error() != "NotFound" { eventPublisher.Warning(ctx, EventReasonUpgradeCheckFailed, fmt.Sprintf("Could not find the Standalone list. Reason %v", err)) - scopedLog.Error(err, "Unable to get standalone list") + logger.ErrorContext(ctx, "Unable to get standalone list", "error", err) return false, err } diff --git a/pkg/splunk/enterprise/util_test.go b/pkg/splunk/enterprise/util_test.go index b2ac6ba4d..3a8d2c3fa 100644 --- a/pkg/splunk/enterprise/util_test.go +++ b/pkg/splunk/enterprise/util_test.go @@ -36,9 +36,10 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/splunk/splunk-operator/pkg/logging" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" @@ -2591,8 +2592,7 @@ func TestUpdateReconcileRequeueTime(t *testing.T) { // to test the value var result *reconcile.Result ctx := context.TODO() - // set logger in context - ctx = log.IntoContext(ctx, log.Log) + ctx = logging.WithLogger(ctx, logging.FromContext(ctx)) rqTime := time.Duration(time.Second * 12) // failure when result is nil diff --git a/pkg/splunk/enterprise/validation/server.go b/pkg/splunk/enterprise/validation/server.go index 9f8429d8f..8015fc028 100644 --- a/pkg/splunk/enterprise/validation/server.go +++ b/pkg/splunk/enterprise/validation/server.go @@ -25,11 +25,11 @@ import ( "net/http" "time" + "github.com/splunk/splunk-operator/pkg/logging" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log" ) var serverLog = ctrl.Log.WithName("webhook-server") @@ -138,8 +138,8 @@ func (s *WebhookServer) Start(ctx context.Context) error { // handleValidate handles validation requests func (s *WebhookServer) handleValidate(w http.ResponseWriter, r *http.Request) { - reqLog := log.FromContext(r.Context()).WithName("webhook-server") - reqLog.V(1).Info("Received validation request", "method", r.Method, "path", r.URL.Path) + logger := logging.FromContext(r.Context()).With("func", "handleValidate") + logger.DebugContext(r.Context(), "Received validation request", "method", r.Method, "path", r.URL.Path) if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) @@ -148,7 +148,7 @@ func (s *WebhookServer) handleValidate(w http.ResponseWriter, r *http.Request) { body, err := io.ReadAll(r.Body) if err != nil { - reqLog.Error(err, "Failed to read request body") + logger.ErrorContext(r.Context(), "Failed to read request body", "error", err) http.Error(w, "Failed to read request body", http.StatusBadRequest) return } @@ -156,13 +156,13 @@ func (s *WebhookServer) handleValidate(w http.ResponseWriter, r *http.Request) { var admissionReview admissionv1.AdmissionReview if err := json.Unmarshal(body, &admissionReview); err != nil { - reqLog.Error(err, "Failed to decode admission review") + logger.ErrorContext(r.Context(), "Failed to decode admission review", "error", err) http.Error(w, "Failed to decode admission review", http.StatusBadRequest) return } if admissionReview.Request != nil { - reqLog.Info("Processing admission request", + logger.InfoContext(r.Context(), "Processing admission request", "kind", admissionReview.Request.Kind.Kind, "name", admissionReview.Request.Name, "namespace", admissionReview.Request.Namespace, @@ -177,7 +177,7 @@ func (s *WebhookServer) handleValidate(w http.ResponseWriter, r *http.Request) { } if validationErr != nil { - reqLog.Info("Validation failed", + logger.InfoContext(r.Context(), "Validation failed", "kind", admissionReview.Request.Kind.Kind, "name", admissionReview.Request.Name, "error", validationErr.Error()) diff --git a/pkg/splunk/splkcontroller/configmap.go b/pkg/splunk/splkcontroller/configmap.go index 2d27a4adb..b6c79ab7f 100644 --- a/pkg/splunk/splkcontroller/configmap.go +++ b/pkg/splunk/splkcontroller/configmap.go @@ -26,15 +26,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" - "sigs.k8s.io/controller-runtime/pkg/log" ) // ApplyConfigMap creates or updates a Kubernetes ConfigMap func ApplyConfigMap(ctx context.Context, client splcommon.ControllerClient, configMap *corev1.ConfigMap) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyConfigMap").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "ApplyConfigMap", "name", configMap.GetObjectMeta().GetName(), "namespace", configMap.GetObjectMeta().GetNamespace()) @@ -53,14 +52,14 @@ func ApplyConfigMap(ctx context.Context, client splcommon.ControllerClient, conf // from the data on the configMap passed as an argument to this function dataDifferent := false if !reflect.DeepEqual(configMap.Data, current.Data) { - scopedLog.Info("Updating existing ConfigMap", "ResourceVerison", current.GetResourceVersion()) + scopedLog.InfoContext(ctx, "Updating existing ConfigMap", "ResourceVerison", current.GetResourceVersion()) current.Data = configMap.Data updateNeeded = true dataDifferent = true configMap = ¤t } if !reflect.DeepEqual(configMap.GetOwnerReferences(), current.GetOwnerReferences()) { - scopedLog.Info("Updating existing ConfigMap", "ResourceVerison", current.GetResourceVersion()) + scopedLog.InfoContext(ctx, "Updating existing ConfigMap", "ResourceVerison", current.GetResourceVersion()) current.OwnerReferences = configMap.OwnerReferences updateNeeded = true configMap = ¤t @@ -77,7 +76,7 @@ func ApplyConfigMap(ctx context.Context, client splcommon.ControllerClient, conf } } } else { - scopedLog.Info("No changes for ConfigMap") + scopedLog.InfoContext(ctx, "No changes for ConfigMap") } } else if k8serrors.IsNotFound(err) { @@ -87,7 +86,7 @@ func ApplyConfigMap(ctx context.Context, client splcommon.ControllerClient, conf retryCount := 0 gerr := client.Get(ctx, namespacedName, ¤t) for ; gerr != nil; gerr = client.Get(ctx, namespacedName, ¤t) { - scopedLog.Error(gerr, "Newly created resource still not in cache sleeping for 10 micro second", "configmap", configMap.Name, "error", gerr.Error()) + scopedLog.ErrorContext(ctx, "Newly created resource still not in cache sleeping for 10 micro second", "configmap", configMap.Name, "error", gerr) time.Sleep(10 * time.Microsecond) retryCount++ if retryCount > 20 { diff --git a/pkg/splunk/splkcontroller/controller.go b/pkg/splunk/splkcontroller/controller.go index 34389d7ab..cb4ae529b 100644 --- a/pkg/splunk/splkcontroller/controller.go +++ b/pkg/splunk/splkcontroller/controller.go @@ -18,12 +18,12 @@ package splkcontroller import ( "context" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -97,9 +97,8 @@ type splunkReconciler struct { func (r splunkReconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { instance := r.splctrl.GetInstance() gvk := instance.GroupVersionKind() - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("Reconcile").WithValues("Group", gvk.Group, "Version", gvk.Version, "Kind", gvk.Kind, "Namespace", request.Namespace, "Name", request.Name) - scopedLog.Info("Reconciling custom resource") + scopedLog := logging.FromContext(ctx).With("func", "Reconcile", "Group", gvk.Group, "Version", gvk.Version, "Kind", gvk.Kind, "Namespace", request.Namespace, "Name", request.Name) + scopedLog.InfoContext(ctx, "Reconciling custom resource") // Fetch the custom resource instance err := r.client.Get(context.TODO(), request.NamespacedName, instance) @@ -122,14 +121,14 @@ func (r splunkReconciler) Reconcile(ctx context.Context, request reconcile.Reque // log what happens next if err != nil { - scopedLog.Error(err, "Reconciliation requeued", "RequeueAfter", result.RequeueAfter) + scopedLog.ErrorContext(ctx, "Reconciliation requeued", "RequeueAfter", result.RequeueAfter, "error", err) return result, nil } if result.Requeue { - scopedLog.Info("Reconciliation requeued", "RequeueAfter", result.RequeueAfter) + scopedLog.InfoContext(ctx, "Reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil } - scopedLog.Info("Reconciliation complete") + scopedLog.InfoContext(ctx, "Reconciliation complete") return reconcile.Result{}, nil } diff --git a/pkg/splunk/splkcontroller/deployment.go b/pkg/splunk/splkcontroller/deployment.go index aab96833a..85c5c0f87 100644 --- a/pkg/splunk/splkcontroller/deployment.go +++ b/pkg/splunk/splkcontroller/deployment.go @@ -19,20 +19,19 @@ import ( "context" "fmt" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" appsv1 "k8s.io/api/apps/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" enterpriseApi "github.com/splunk/splunk-operator/api/v4" ) // ApplyDeployment creates or updates a Kubernetes Deployment func ApplyDeployment(ctx context.Context, c splcommon.ControllerClient, revised *appsv1.Deployment) (enterpriseApi.Phase, error) { - log := log.FromContext(ctx) - scopedLog := log.WithName("ApplyDeployment").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "ApplyDeployment", "name", revised.GetObjectMeta().GetName(), "namespace", revised.GetObjectMeta().GetNamespace()) @@ -56,11 +55,11 @@ func ApplyDeployment(ctx context.Context, c splcommon.ControllerClient, revised // check for scaling if revised.Spec.Replicas != nil { if *revised.Spec.Replicas < desiredReplicas { - scopedLog.Info(fmt.Sprintf("Scaling replicas up to %d", desiredReplicas)) + scopedLog.InfoContext(ctx, fmt.Sprintf("Scaling replicas up to %d", desiredReplicas)) *revised.Spec.Replicas = desiredReplicas return enterpriseApi.PhaseScalingUp, splutil.UpdateResource(ctx, c, revised) } else if *revised.Spec.Replicas > desiredReplicas { - scopedLog.Info(fmt.Sprintf("Scaling replicas down to %d", desiredReplicas)) + scopedLog.InfoContext(ctx, fmt.Sprintf("Scaling replicas down to %d", desiredReplicas)) *revised.Spec.Replicas = desiredReplicas return enterpriseApi.PhaseScalingDown, splutil.UpdateResource(ctx, c, revised) } @@ -73,13 +72,13 @@ func ApplyDeployment(ctx context.Context, c splcommon.ControllerClient, revised // check if updates are in progress if revised.Status.UpdatedReplicas < revised.Status.Replicas { - scopedLog.Info("Waiting for updates to complete") + scopedLog.InfoContext(ctx, "Waiting for updates to complete") return enterpriseApi.PhaseUpdating, nil } // check if replicas are not yet ready if revised.Status.ReadyReplicas < desiredReplicas { - scopedLog.Info("Waiting for pods to become ready") + scopedLog.InfoContext(ctx, "Waiting for pods to become ready") if revised.Status.ReadyReplicas > 0 { return enterpriseApi.PhaseScalingUp, nil } @@ -87,6 +86,6 @@ func ApplyDeployment(ctx context.Context, c splcommon.ControllerClient, revised } // all is good! - scopedLog.Info("All pods are ready") + scopedLog.InfoContext(ctx, "All pods are ready") return enterpriseApi.PhaseReady, nil } diff --git a/pkg/splunk/splkcontroller/finalizers.go b/pkg/splunk/splkcontroller/finalizers.go index fd7470a16..76bc5efb5 100644 --- a/pkg/splunk/splkcontroller/finalizers.go +++ b/pkg/splunk/splkcontroller/finalizers.go @@ -19,10 +19,9 @@ import ( "context" "fmt" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/log" - + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func init() { @@ -38,25 +37,24 @@ var SplunkFinalizerRegistry map[string]SplunkFinalizerMethod // CheckForDeletion checks to see if deletion was requested for the custom resource. // If so, it will process and remove any remaining finalizers. func CheckForDeletion(ctx context.Context, cr splcommon.MetaObject, c splcommon.ControllerClient) (bool, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("CheckSplunkDeletion").WithValues("kind", cr.GetObjectKind().GroupVersionKind().Kind, + scopedLog := logging.FromContext(ctx).With("func", "CheckSplunkDeletion", "kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) currentTime := metav1.Now() // sanity check: return early if missing GetDeletionTimestamp if cr.GetObjectMeta().GetDeletionTimestamp() == nil { - scopedLog.Info("DeletionTimestamp is nil") + scopedLog.InfoContext(ctx, "DeletionTimestamp is nil") return false, nil } // just log warning if deletion time is in the future if !cr.GetObjectMeta().GetDeletionTimestamp().Before(¤tTime) { - scopedLog.Info("DeletionTimestamp is in the future", + scopedLog.InfoContext(ctx, "DeletionTimestamp is in the future", "Now", currentTime, "DeletionTimestamp", cr.GetObjectMeta().GetDeletionTimestamp()) } - scopedLog.Info("Deletion requested") + scopedLog.InfoContext(ctx, "Deletion requested") // process each finalizer for _, finalizer := range cr.GetObjectMeta().GetFinalizers() { @@ -67,7 +65,7 @@ func CheckForDeletion(ctx context.Context, cr splcommon.MetaObject, c splcommon. } // process finalizer callback - scopedLog.Info("Processing callback", "Finalizer", finalizer) + scopedLog.InfoContext(ctx, "Processing callback", "Finalizer", finalizer) err := callback(ctx, cr, c) if err != nil { return false, err @@ -80,16 +78,15 @@ func CheckForDeletion(ctx context.Context, cr splcommon.MetaObject, c splcommon. } } - scopedLog.Info("Deletion complete") + scopedLog.InfoContext(ctx, "Deletion complete") return true, nil } // removeSplunkFinalizer removes a finalizer from a custom resource. func removeSplunkFinalizer(ctx context.Context, cr splcommon.MetaObject, c splcommon.ControllerClient, finalizer string) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("RemoveFinalizer").WithValues("kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) - scopedLog.Info("Removing finalizer", "name", finalizer) + scopedLog := logging.FromContext(ctx).With("func", "RemoveFinalizer", "kind", cr.GetObjectKind().GroupVersionKind().Kind, "name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog.InfoContext(ctx, "Removing finalizer", "name", finalizer) // create new list of finalizers that doesn't include the one being removed var newFinalizers []string diff --git a/pkg/splunk/splkcontroller/secret.go b/pkg/splunk/splkcontroller/secret.go index 1d8c0ce19..58db9e720 100644 --- a/pkg/splunk/splkcontroller/secret.go +++ b/pkg/splunk/splkcontroller/secret.go @@ -21,13 +21,12 @@ import ( "reflect" "time" + "github.com/splunk/splunk-operator/pkg/logging" + splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" + splutil "github.com/splunk/splunk-operator/pkg/splunk/util" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" - - splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - splutil "github.com/splunk/splunk-operator/pkg/splunk/util" ) // ApplySecret creates or updates a Kubernetes Secret, and returns active secrets if successful @@ -37,8 +36,7 @@ func ApplySecret(ctx context.Context, client splcommon.ControllerClient, secret return nil, errors.New(splcommon.InvalidSecretObjectError) } - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplySecret").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "ApplySecret", "name", secret.GetObjectMeta().GetName(), "namespace", secret.GetObjectMeta().GetNamespace()) @@ -47,7 +45,7 @@ func ApplySecret(ctx context.Context, client splcommon.ControllerClient, secret namespacedName := types.NamespacedName{Namespace: secret.GetNamespace(), Name: secret.GetName()} err := client.Get(ctx, namespacedName, &result) if err == nil { - scopedLog.Info("Found existing Secret, update if needed") + scopedLog.InfoContext(ctx, "Found existing Secret, update if needed") if !reflect.DeepEqual(&result, secret) { result = *secret err = splutil.UpdateResource(ctx, client, &result) @@ -57,7 +55,7 @@ func ApplySecret(ctx context.Context, client splcommon.ControllerClient, secret secret = &result } } else if k8serrors.IsNotFound(err) { - scopedLog.Info("Didn't find secret, creating one") + scopedLog.InfoContext(ctx, "Didn't find secret, creating one") err = splutil.CreateResource(ctx, client, secret) if err != nil { return nil, err @@ -65,7 +63,7 @@ func ApplySecret(ctx context.Context, client splcommon.ControllerClient, secret gerr := client.Get(ctx, namespacedName, secret) retryCount := 0 for ; gerr != nil; gerr = client.Get(ctx, namespacedName, secret) { - scopedLog.Error(gerr, "Newly created resource still not in cache sleeping for 10 micro second", "secret", namespacedName.Name, "error", gerr.Error()) + scopedLog.ErrorContext(ctx, "Newly created resource still not in cache sleeping for 10 micro second", "secret", namespacedName.Name, "error", gerr) time.Sleep(10 * time.Microsecond) // Avoid infinite loop diff --git a/pkg/splunk/splkcontroller/service.go b/pkg/splunk/splkcontroller/service.go index b422f1c10..112d653a0 100644 --- a/pkg/splunk/splkcontroller/service.go +++ b/pkg/splunk/splkcontroller/service.go @@ -18,18 +18,17 @@ package splkcontroller import ( "context" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" ) // ApplyService creates or updates a Kubernetes Service func ApplyService(ctx context.Context, client splcommon.ControllerClient, revised *corev1.Service) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyService").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "ApplyService", "name", revised.GetObjectMeta().GetName(), "namespace", revised.GetObjectMeta().GetNamespace()) @@ -49,7 +48,7 @@ func ApplyService(ctx context.Context, client splcommon.ControllerClient, revise // only update if there are material differences, as determined by comparison function if hasUpdates { - scopedLog.Info("Updating existing Service") + scopedLog.InfoContext(ctx, "Updating existing Service") err = splutil.UpdateResource(ctx, client, revised) if err != nil { return err @@ -61,6 +60,6 @@ func ApplyService(ctx context.Context, client splcommon.ControllerClient, revise } // all is good! - scopedLog.Info("No update to existing Service") + scopedLog.InfoContext(ctx, "No update to existing Service") return nil } diff --git a/pkg/splunk/splkcontroller/serviceaccount.go b/pkg/splunk/splkcontroller/serviceaccount.go index 88b11eb32..d18cd5eb9 100644 --- a/pkg/splunk/splkcontroller/serviceaccount.go +++ b/pkg/splunk/splkcontroller/serviceaccount.go @@ -18,18 +18,17 @@ import ( "context" "reflect" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/log" ) // ApplyServiceAccount creates or updates a Kubernetes serviceAccount func ApplyServiceAccount(ctx context.Context, client splcommon.ControllerClient, serviceAccount *corev1.ServiceAccount) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("ApplyServiceAccount").WithValues("serviceAccount", serviceAccount.GetName(), + scopedLog := logging.FromContext(ctx).With("func", "ApplyServiceAccount", "serviceAccount", serviceAccount.GetName(), "namespace", serviceAccount.GetNamespace()) namespacedName := types.NamespacedName{Namespace: serviceAccount.GetNamespace(), Name: serviceAccount.GetName()} @@ -38,7 +37,7 @@ func ApplyServiceAccount(ctx context.Context, client splcommon.ControllerClient, err := client.Get(ctx, namespacedName, ¤t) if err == nil { if !reflect.DeepEqual(serviceAccount, ¤t) { - scopedLog.Info("Updating service account") + scopedLog.InfoContext(ctx, "Updating service account") current = *serviceAccount err = splutil.UpdateResource(ctx, client, ¤t) if err != nil { @@ -61,10 +60,9 @@ func GetServiceAccount(ctx context.Context, client splcommon.ControllerClient, n var serviceAccount corev1.ServiceAccount err := client.Get(ctx, namespacedName, &serviceAccount) if err != nil { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("GetServiceAccount").WithValues("serviceAccount", namespacedName.Name, + scopedLog := logging.FromContext(ctx).With("func", "GetServiceAccount", "serviceAccount", namespacedName.Name, "namespace", namespacedName.Namespace, "error", err) - scopedLog.Info("ServiceAccount not found") + scopedLog.InfoContext(ctx, "ServiceAccount not found") return nil, err } return &serviceAccount, nil diff --git a/pkg/splunk/splkcontroller/statefulset.go b/pkg/splunk/splkcontroller/statefulset.go index 3028efbdd..32d38c51d 100644 --- a/pkg/splunk/splkcontroller/statefulset.go +++ b/pkg/splunk/splkcontroller/statefulset.go @@ -22,6 +22,7 @@ import ( enterpriseApi "github.com/splunk/splunk-operator/api/v4" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" splutil "github.com/splunk/splunk-operator/pkg/splunk/util" appsv1 "k8s.io/api/apps/v1" @@ -30,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" ) // DefaultStatefulSetPodManager is a simple StatefulSetPodManager that does nothing @@ -113,8 +113,7 @@ func ApplyStatefulSet(ctx context.Context, c splcommon.ControllerClient, revised // UpdateStatefulSetPods manages scaling and config updates for StatefulSets func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, statefulSet *appsv1.StatefulSet, mgr splcommon.StatefulSetPodManager, desiredReplicas int32) (enterpriseApi.Phase, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("UpdateStatefulSetPods").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "UpdateStatefulSetPods", "name", statefulSet.GetObjectMeta().GetName(), "namespace", statefulSet.GetObjectMeta().GetNamespace()) @@ -125,7 +124,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st namespacedName := types.NamespacedName{Namespace: statefulSet.GetNamespace(), Name: statefulSet.GetName()} err := c.Get(ctx, namespacedName, statefulSet) if err != nil { - scopedLog.Error(err, "Unable to re-fetch StatefulSet for latest status") + scopedLog.ErrorContext(ctx, "Unable to re-fetch StatefulSet for latest status", "error", err) return enterpriseApi.PhaseError, err } @@ -133,13 +132,13 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st replicas := *statefulSet.Spec.Replicas readyReplicas := statefulSet.Status.ReadyReplicas if readyReplicas < replicas { - scopedLog.Info("Waiting for pods to become ready") + scopedLog.InfoContext(ctx, "Waiting for pods to become ready") if readyReplicas > 0 { return enterpriseApi.PhaseScalingUp, nil } return enterpriseApi.PhasePending, nil } else if readyReplicas > replicas { - scopedLog.Info("Waiting for scale down to complete") + scopedLog.InfoContext(ctx, "Waiting for scale down to complete") return enterpriseApi.PhaseScalingDown, nil } @@ -148,7 +147,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st // check for scaling up if readyReplicas < desiredReplicas { // scale up StatefulSet to match desiredReplicas - scopedLog.Info("Scaling replicas up", "replicas", desiredReplicas) + scopedLog.InfoContext(ctx, "Scaling replicas up", "replicas", desiredReplicas) *statefulSet.Spec.Replicas = desiredReplicas return enterpriseApi.PhaseScalingUp, splutil.UpdateResource(ctx, c, statefulSet) } @@ -160,7 +159,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st podName := fmt.Sprintf("%s-%d", statefulSet.GetName(), n) ready, err := mgr.PrepareScaleDown(ctx, n) if err != nil { - scopedLog.Error(err, "Unable to decommission Pod", "podName", podName) + scopedLog.ErrorContext(ctx, "Unable to decommission Pod", "podName", podName, "error", err) return enterpriseApi.PhaseError, err } if !ready { @@ -169,11 +168,11 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st } // scale down statefulset to terminate pod - scopedLog.Info("Scaling replicas down", "replicas", n) + scopedLog.InfoContext(ctx, "Scaling replicas down", "replicas", n) *statefulSet.Spec.Replicas = n err = splutil.UpdateResource(ctx, c, statefulSet) if err != nil { - scopedLog.Error(err, "Scale down update failed for StatefulSet") + scopedLog.ErrorContext(ctx, "Scale down update failed for StatefulSet", "error", err) return enterpriseApi.PhaseError, err } @@ -186,13 +185,13 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st var pvc corev1.PersistentVolumeClaim err := c.Get(ctx, namespacedName, &pvc) if err != nil { - scopedLog.Error(err, "Unable to find PVC for deletion", "pvcName", pvc.ObjectMeta.Name) + scopedLog.ErrorContext(ctx, "Unable to find PVC for deletion", "pvcName", pvc.ObjectMeta.Name, "error", err) return enterpriseApi.PhaseError, err } - scopedLog.Info("Deleting PVC", "pvcName", pvc.ObjectMeta.Name) + scopedLog.InfoContext(ctx, "Deleting PVC", "pvcName", pvc.ObjectMeta.Name) err = c.Delete(ctx, &pvc) if err != nil { - scopedLog.Error(err, "Unable to delete PVC", "pvcName", pvc.ObjectMeta.Name) + scopedLog.ErrorContext(ctx, "Unable to delete PVC", "pvcName", pvc.ObjectMeta.Name, "error", err) return enterpriseApi.PhaseError, err } } @@ -211,11 +210,11 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st var pod corev1.Pod err := c.Get(ctx, namespacedName, &pod) if err != nil { - scopedLog.Error(err, "Unable to find Pod", "podName", podName) + scopedLog.ErrorContext(ctx, "Unable to find Pod", "podName", podName, "error", err) return enterpriseApi.PhaseError, err } if pod.Status.Phase != corev1.PodRunning || len(pod.Status.ContainerStatuses) == 0 || !pod.Status.ContainerStatuses[0].Ready { - scopedLog.Error(err, "Waiting for Pod to become ready", "podName", podName) + scopedLog.ErrorContext(ctx, "Waiting for Pod to become ready", "podName", podName, "error", err) return enterpriseApi.PhaseUpdating, err } @@ -224,7 +223,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st // pod needs to be updated; first, prepare it to be recycled ready, err := mgr.PrepareRecycle(ctx, n) if err != nil { - scopedLog.Error(err, "Unable to prepare Pod for recycling", "podName", podName) + scopedLog.ErrorContext(ctx, "Unable to prepare Pod for recycling", "podName", podName, "error", err) return enterpriseApi.PhaseError, err } if !ready { @@ -233,13 +232,13 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st } // deleting pod will cause StatefulSet controller to create a new one with latest template - scopedLog.Info("Recycling Pod for updates", "podName", podName, + scopedLog.InfoContext(ctx, "Recycling Pod for updates", "podName", podName, "statefulSetRevision", statefulSet.Status.UpdateRevision, "podRevision", pod.GetLabels()["controller-revision-hash"]) preconditions := client.Preconditions{UID: &pod.ObjectMeta.UID, ResourceVersion: &pod.ObjectMeta.ResourceVersion} err = c.Delete(context.Background(), &pod, preconditions) if err != nil { - scopedLog.Error(err, "Unable to delete Pod", "podName", podName) + scopedLog.ErrorContext(ctx, "Unable to delete Pod", "podName", podName, "error", err) return enterpriseApi.PhaseError, err } @@ -250,7 +249,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st // check if pod was previously prepared for recycling; if so, complete complete, err := mgr.FinishRecycle(ctx, n) if err != nil { - scopedLog.Error(err, "Unable to complete recycling of pod", "podName", podName) + scopedLog.ErrorContext(ctx, "Unable to complete recycling of pod", "podName", podName, "error", err) return enterpriseApi.PhaseError, err } if !complete { @@ -266,17 +265,17 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st } // all is good! - scopedLog.Info("All pods are ready") + scopedLog.InfoContext(ctx, "All pods are ready") // Finalize rolling upgrade process // It uses first pod to get a client err = mgr.FinishUpgrade(ctx, 0) if err != nil { - scopedLog.Error(err, "Unable to finalize rolling upgrade process") + scopedLog.ErrorContext(ctx, "Unable to finalize rolling upgrade process", "error", err) return enterpriseApi.PhaseError, err } - scopedLog.Info("Statefulset - Phase Ready") + scopedLog.InfoContext(ctx, "Statefulset - Phase Ready") return enterpriseApi.PhaseReady, nil } @@ -307,10 +306,9 @@ func SetStatefulSetOwnerRef(ctx context.Context, client splcommon.ControllerClie // RemoveUnwantedOwnerRefSs removes all the unwanted owner references for statefulset except the CR it belongs to func RemoveUnwantedOwnerRefSs(ctx context.Context, client splcommon.ControllerClient, namespacedName types.NamespacedName, cr splcommon.MetaObject) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("RemoveUnwantedOwnerRefSs").WithValues("statefulSet", namespacedName) + scopedLog := logging.FromContext(ctx).With("func", "RemoveUnwantedOwnerRefSs", "statefulSet", namespacedName) - scopedLog.Info("Removing unwanted owner references on CR deletion") + scopedLog.InfoContext(ctx, "Removing unwanted owner references on CR deletion") // Get statefulSet statefulset, err := GetStatefulSetByName(ctx, client, namespacedName) @@ -385,13 +383,12 @@ func isCurrentCROwner(cr splcommon.MetaObject, currentOwners []metav1.OwnerRefer // IsStatefulSetScalingUpOrDown checks if we are currently scaling up or down func IsStatefulSetScalingUpOrDown(ctx context.Context, client splcommon.ControllerClient, cr splcommon.MetaObject, name string, desiredReplicas int32) (enterpriseApi.StatefulSetScalingType, error) { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("isScalingUp").WithValues("name", cr.GetName(), "namespace", cr.GetNamespace()) + scopedLog := logging.FromContext(ctx).With("func", "isScalingUp", "name", cr.GetName(), "namespace", cr.GetNamespace()) namespacedName := types.NamespacedName{Namespace: cr.GetNamespace(), Name: name} current, err := GetStatefulSetByName(ctx, client, namespacedName) if err != nil { - scopedLog.Error(err, "Unable to get current stateful set", "name", namespacedName) + scopedLog.ErrorContext(ctx, "Unable to get current stateful set", "name", namespacedName, "error", err) return enterpriseApi.StatefulSetNotScaling, err } diff --git a/pkg/splunk/splkcontroller/util.go b/pkg/splunk/splkcontroller/util.go index b4264efb6..70170540a 100644 --- a/pkg/splunk/splkcontroller/util.go +++ b/pkg/splunk/splkcontroller/util.go @@ -26,8 +26,8 @@ import ( //stdlog "log" //"github.com/go-logr/stdr" + "github.com/splunk/splunk-operator/pkg/logging" splcommon "github.com/splunk/splunk-operator/pkg/splunk/common" - "sigs.k8s.io/controller-runtime/pkg/log" ) // kubernetes logger used by splunk.reconcile package @@ -53,20 +53,19 @@ func MergePodUpdates(ctx context.Context, current *corev1.PodTemplateSpec, revis // current. This enables us to minimize updates. It returns true if there // are material differences between them, or false otherwise. func MergePodMetaUpdates(ctx context.Context, current *metav1.ObjectMeta, revised *metav1.ObjectMeta, name string) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("MergePodMetaUpdates").WithValues("name", name) + scopedLog := logging.FromContext(ctx).With("func", "MergePodMetaUpdates", "name", name) result := false // check Annotations if !reflect.DeepEqual(current.Annotations, revised.Annotations) { - scopedLog.Info("Container Annotations differ", "current", current.Annotations, "revised", revised.Annotations) + scopedLog.InfoContext(ctx, "Container Annotations differ", "current", current.Annotations, "revised", revised.Annotations) current.Annotations = revised.Annotations result = true } // check Labels if !reflect.DeepEqual(current.Labels, revised.Labels) { - scopedLog.Info("Container Labels differ", "current", current.Labels, "revised", revised.Labels) + scopedLog.InfoContext(ctx, "Container Labels differ", "current", current.Labels, "revised", revised.Labels) current.Labels = revised.Labels result = true } @@ -79,13 +78,12 @@ func MergePodMetaUpdates(ctx context.Context, current *metav1.ObjectMeta, revise // current. This enables us to minimize updates. It returns true if there // are material differences between them, or false otherwise. func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised *corev1.PodSpec, name string) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("MergePodUpdates").WithValues("name", name) + scopedLog := logging.FromContext(ctx).With("func", "MergePodUpdates", "name", name) result := false // check for changes in ServiceAccount if splcommon.CompareByMarshall(current.ServiceAccountName, revised.ServiceAccountName) { - scopedLog.Info("Pod service account differs", + scopedLog.InfoContext(ctx, "Pod service account differs", "current", current.ServiceAccountName, "revised", revised.ServiceAccountName) current.ServiceAccountName = revised.ServiceAccountName @@ -94,7 +92,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in Affinity if splcommon.CompareByMarshall(current.Affinity, revised.Affinity) { - scopedLog.Info("Pod Affinity differs", + scopedLog.InfoContext(ctx, "Pod Affinity differs", "current", current.Affinity, "revised", revised.Affinity) current.Affinity = revised.Affinity @@ -103,7 +101,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in Tolerations if splcommon.CompareTolerations(current.Tolerations, revised.Tolerations) { - scopedLog.Info("Pod Tolerations differs", + scopedLog.InfoContext(ctx, "Pod Tolerations differs", "current", current.Tolerations, "revised", revised.Tolerations) current.Tolerations = revised.Tolerations @@ -112,7 +110,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in TopologySpreadConstraint if splcommon.CompareTopologySpreadConstraints(current.TopologySpreadConstraints, revised.TopologySpreadConstraints) { - scopedLog.Info("Pod TopologySpreadConstraint differs", + scopedLog.InfoContext(ctx, "Pod TopologySpreadConstraint differs", "current", current.TopologySpreadConstraints, "revised", revised.TopologySpreadConstraints) current.TopologySpreadConstraints = revised.TopologySpreadConstraints @@ -121,7 +119,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in ImagePullSecrets if splcommon.CompareImagePullSecrets(current.ImagePullSecrets, revised.ImagePullSecrets) { - scopedLog.Info("Pod ImagePullSecrets differs", + scopedLog.InfoContext(ctx, "Pod ImagePullSecrets differs", "current", current.ImagePullSecrets, "revised", revised.ImagePullSecrets) current.ImagePullSecrets = revised.ImagePullSecrets @@ -130,7 +128,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in SchedulerName if current.SchedulerName != revised.SchedulerName { - scopedLog.Info("Pod SchedulerName differs", + scopedLog.InfoContext(ctx, "Pod SchedulerName differs", "current", current.SchedulerName, "revised", revised.SchedulerName) current.SchedulerName = revised.SchedulerName @@ -139,7 +137,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // Check for changes in Volumes if splcommon.CompareVolumes(current.Volumes, revised.Volumes) { - scopedLog.Info("Pod Volumes differ", + scopedLog.InfoContext(ctx, "Pod Volumes differ", "current", current.Volumes, "revised", revised.Volumes) current.Volumes = revised.Volumes @@ -148,7 +146,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // Check for changes in Init containers if len(current.InitContainers) != len(revised.InitContainers) { - scopedLog.Info("Pod init containers differ", + scopedLog.InfoContext(ctx, "Pod init containers differ", "current", len(current.InitContainers), "revised", len(revised.InitContainers)) current.InitContainers = revised.InitContainers @@ -157,7 +155,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * for idx := range current.InitContainers { // check Image if current.InitContainers[idx].Image != revised.InitContainers[idx].Image { - scopedLog.Info("Init Container Images differ", + scopedLog.InfoContext(ctx, "Init Container Images differ", "current", current.InitContainers[idx].Image, "revised", revised.InitContainers[idx].Image) current.InitContainers[idx].Image = revised.InitContainers[idx].Image @@ -168,7 +166,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check for changes in container images; assume that the ordering is same for pods with > 1 container if len(current.Containers) != len(revised.Containers) { - scopedLog.Info("Pod Container counts differ", + scopedLog.InfoContext(ctx, "Pod Container counts differ", "current", len(current.Containers), "revised", len(revised.Containers)) current.Containers = revised.Containers @@ -177,7 +175,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * for idx := range current.Containers { // check Image if current.Containers[idx].Image != revised.Containers[idx].Image { - scopedLog.Info("Pod Container Images differ", + scopedLog.InfoContext(ctx, "Pod Container Images differ", "current", current.Containers[idx].Image, "revised", revised.Containers[idx].Image) current.Containers[idx].Image = revised.Containers[idx].Image @@ -186,7 +184,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check Ports if splcommon.CompareContainerPorts(current.Containers[idx].Ports, revised.Containers[idx].Ports) { - scopedLog.Info("Pod Container Ports differ", + scopedLog.InfoContext(ctx, "Pod Container Ports differ", "current", current.Containers[idx].Ports, "revised", revised.Containers[idx].Ports) current.Containers[idx].Ports = revised.Containers[idx].Ports @@ -195,7 +193,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check VolumeMounts if splcommon.CompareVolumeMounts(current.Containers[idx].VolumeMounts, revised.Containers[idx].VolumeMounts) { - scopedLog.Info("Pod Container VolumeMounts differ", + scopedLog.InfoContext(ctx, "Pod Container VolumeMounts differ", "current", current.Containers[idx].VolumeMounts, "revised", revised.Containers[idx].VolumeMounts) current.Containers[idx].VolumeMounts = revised.Containers[idx].VolumeMounts @@ -204,7 +202,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check Resources if splcommon.CompareByMarshall(¤t.Containers[idx].Resources, &revised.Containers[idx].Resources) { - scopedLog.Info("Pod Container Resources differ", + scopedLog.InfoContext(ctx, "Pod Container Resources differ", "current", current.Containers[idx].Resources, "revised", revised.Containers[idx].Resources) current.Containers[idx].Resources = revised.Containers[idx].Resources @@ -213,7 +211,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check Env if splcommon.CompareEnvs(current.Containers[idx].Env, revised.Containers[idx].Env) { - scopedLog.Info("Pod Container Envs differ", + scopedLog.InfoContext(ctx, "Pod Container Envs differ", "current", current.Containers[idx].Env, "revised", revised.Containers[idx].Env) current.Containers[idx].Env = revised.Containers[idx].Env @@ -222,7 +220,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // check probes if hasProbeChanged(current.Containers[idx].LivenessProbe, revised.Containers[idx].LivenessProbe) { - scopedLog.Info("Pod Container Liveness Probe differ", + scopedLog.InfoContext(ctx, "Pod Container Liveness Probe differ", "current", current.Containers[idx].LivenessProbe, "revised", revised.Containers[idx].LivenessProbe) current.Containers[idx].LivenessProbe = revised.Containers[idx].LivenessProbe @@ -230,7 +228,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * } if hasProbeChanged(current.Containers[idx].ReadinessProbe, revised.Containers[idx].ReadinessProbe) { - scopedLog.Info("Pod Container ReadinessProbe Probe differ", + scopedLog.InfoContext(ctx, "Pod Container ReadinessProbe Probe differ", "current", current.Containers[idx].ReadinessProbe, "revised", revised.Containers[idx].ReadinessProbe) current.Containers[idx].ReadinessProbe = revised.Containers[idx].ReadinessProbe @@ -238,7 +236,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * } if hasProbeChanged(current.Containers[idx].StartupProbe, revised.Containers[idx].StartupProbe) { - scopedLog.Info("Pod Container StartupProbe Probe differ", + scopedLog.InfoContext(ctx, "Pod Container StartupProbe Probe differ", "current", current.Containers[idx].StartupProbe, "revised", revised.Containers[idx].StartupProbe) current.Containers[idx].StartupProbe = revised.Containers[idx].StartupProbe @@ -252,8 +250,7 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised * // SortStatefulSetSlices sorts required slices in a statefulSet func SortStatefulSetSlices(ctx context.Context, current *corev1.PodSpec, name string) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("SortStatefulSetSlices").WithValues("name", name) + scopedLog := logging.FromContext(ctx).With("func", "SortStatefulSetSlices", "name", name) // Sort tolerations splcommon.SortSlice(current.Tolerations, splcommon.SortFieldKey) @@ -278,20 +275,19 @@ func SortStatefulSetSlices(ctx context.Context, current *corev1.PodSpec, name st // Sort env variables splcommon.SortSlice(current.Containers[idx].Env, splcommon.SortFieldName) } - scopedLog.Info("Successfully sorted slices in statefulSet") + scopedLog.InfoContext(ctx, "Successfully sorted slices in statefulSet") return nil } // MergeServiceSpecUpdates merges the current and revised spec of the service object func MergeServiceSpecUpdates(ctx context.Context, current *corev1.ServiceSpec, revised *corev1.ServiceSpec, name string) bool { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("MergeServiceSpecUpdates").WithValues("name", name) + scopedLog := logging.FromContext(ctx).With("func", "MergeServiceSpecUpdates", "name", name) result := false // check service Type if current.Type != revised.Type { - scopedLog.Info("Service Type differs", + scopedLog.InfoContext(ctx, "Service Type differs", "current", current.Type, "revised", revised.Type) current.Type = revised.Type @@ -299,7 +295,7 @@ func MergeServiceSpecUpdates(ctx context.Context, current *corev1.ServiceSpec, r } if current.ExternalName != revised.ExternalName { - scopedLog.Info("External Name differs", + scopedLog.InfoContext(ctx, "External Name differs", "current", current.ExternalName, "revised", revised.ExternalName) current.ExternalName = revised.ExternalName @@ -307,7 +303,7 @@ func MergeServiceSpecUpdates(ctx context.Context, current *corev1.ServiceSpec, r } if current.ExternalTrafficPolicy != revised.ExternalTrafficPolicy { - scopedLog.Info("External Traffic Policy differs", + scopedLog.InfoContext(ctx, "External Traffic Policy differs", "current", current.ExternalTrafficPolicy, "revised", revised.ExternalTrafficPolicy) current.ExternalTrafficPolicy = revised.ExternalTrafficPolicy @@ -315,7 +311,7 @@ func MergeServiceSpecUpdates(ctx context.Context, current *corev1.ServiceSpec, r } if splcommon.CompareSortedStrings(current.ExternalIPs, revised.ExternalIPs) { - scopedLog.Info("External IPs differs", + scopedLog.InfoContext(ctx, "External IPs differs", "current", current.ExternalIPs, "revised", revised.ExternalIPs) current.ExternalIPs = revised.ExternalIPs @@ -324,7 +320,7 @@ func MergeServiceSpecUpdates(ctx context.Context, current *corev1.ServiceSpec, r // check for changes in Ports if splcommon.CompareServicePorts(current.Ports, revised.Ports) { - scopedLog.Info("Service Ports differs", + scopedLog.InfoContext(ctx, "Service Ports differs", "current", current.Ports, "revised", revised.Ports) current.Ports = revised.Ports diff --git a/pkg/splunk/util/util.go b/pkg/splunk/util/util.go index 57a6442b0..04867ab52 100644 --- a/pkg/splunk/util/util.go +++ b/pkg/splunk/util/util.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/splunk/splunk-operator/pkg/logging" ) // kubernetes logger used by splunk.reconcile package @@ -78,54 +78,51 @@ func (cr *TestResource) DeepCopyObject() runtime.Object { // CreateResource creates a new Kubernetes resource using the REST API. func CreateResource(ctx context.Context, client splcommon.ControllerClient, obj splcommon.MetaObject) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("CreateResource").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "CreateResource", "name", obj.GetObjectMeta().GetName(), "namespace", obj.GetObjectMeta().GetNamespace()) err := client.Create(ctx, obj) if err != nil && !errors.IsAlreadyExists(err) { - scopedLog.Error(err, "Failed to create resource", "kind", obj.GetObjectKind()) + scopedLog.ErrorContext(ctx, "Failed to create resource", "kind", obj.GetObjectKind(), "error", err) return err } - scopedLog.Info("Created resource", "kind", obj.GetObjectKind()) + scopedLog.InfoContext(ctx, "Created resource", "kind", obj.GetObjectKind()) return nil } // UpdateResource updates an existing Kubernetes resource using the REST API. func UpdateResource(ctx context.Context, client splcommon.ControllerClient, obj splcommon.MetaObject) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("UpdateResource").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "UpdateResource", "name", obj.GetObjectMeta().GetName(), "namespace", obj.GetObjectMeta().GetNamespace()) err := client.Update(ctx, obj) if err != nil && !errors.IsAlreadyExists(err) { - scopedLog.Error(err, "Failed to update resource", "kind", obj.GetObjectKind()) + scopedLog.ErrorContext(ctx, "Failed to update resource", "kind", obj.GetObjectKind(), "error", err) return err } - scopedLog.Info("Updated resource", "kind", obj.GetObjectKind()) + scopedLog.InfoContext(ctx, "Updated resource", "kind", obj.GetObjectKind()) return nil } // DeleteResource deletes an existing Kubernetes resource using the REST API. func DeleteResource(ctx context.Context, client splcommon.ControllerClient, obj splcommon.MetaObject) error { - reqLogger := log.FromContext(ctx) - scopedLog := reqLogger.WithName("DeleteResource").WithValues( + scopedLog := logging.FromContext(ctx).With("func", "DeleteResource", "name", obj.GetObjectMeta().GetName(), "namespace", obj.GetObjectMeta().GetNamespace()) err := client.Delete(ctx, obj) if err != nil && !errors.IsAlreadyExists(err) { - scopedLog.Error(err, "Failed to delete resource", "kind", obj.GetObjectKind()) + scopedLog.ErrorContext(ctx, "Failed to delete resource", "kind", obj.GetObjectKind(), "error", err) return err } - scopedLog.Info("Deleted resource", "kind", obj.GetObjectKind()) + scopedLog.InfoContext(ctx, "Deleted resource", "kind", obj.GetObjectKind()) return nil } @@ -257,13 +254,13 @@ func suppressHarmlessErrorMessages(values ...*string) { // RunPodExecCommand runs the specific pod exec command func (podExecClient *PodExecClient) RunPodExecCommand(ctx context.Context, streamOptions *remotecommand.StreamOptions, baseCmd []string) (string, string, error) { - reqLogger := log.FromContext(ctx) + scopedLog := logging.FromContext(ctx) errmsg := "" stdOut, stdErr, err := PodExecCommand(ctx, podExecClient.client, podExecClient.targetPodName, podExecClient.cr.GetNamespace(), baseCmd, streamOptions, false, false, "") if err != nil { errmsg = err.Error() } - reqLogger.Info("podexec call returned", "cmd", strings.Join(baseCmd, " "), "stdout", stdOut, "stderr", stdErr, "err", errmsg) + scopedLog.InfoContext(ctx, "podexec call returned", "cmd", strings.Join(baseCmd, " "), "stdout", stdOut, "stderr", stdErr, "err", errmsg) // Note: splunk 9.0 throws a few harmless warning error messages, suppress it! suppressHarmlessErrorMessages(&stdErr, &stdOut) From 125a85d53b63b317c355c6bc40e1cfcb85d58d19 Mon Sep 17 00:00:00 2001 From: "igor.grzankowski" Date: Wed, 1 Apr 2026 10:37:47 +0200 Subject: [PATCH 12/19] Remove build versions from logs --- cmd/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index c82a68e95..04b15bf78 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -171,10 +171,7 @@ func main() { addSourcePtr = &logAddSource } logCfg := logging.LoadConfigWithFlags(logLevel, logFormat, addSourcePtr) - _ = logging.SetupLogger(logCfg, - slog.String("component", "splunk-operator"), - slog.String("version", version), - slog.String("build", gitCommit)) + _ = logging.SetupLogger(logCfg) // Log startup information using slog slog.Info("Splunk Operator starting", From 1450ec4ee9e3f4de7c7adc619279205e19193483 Mon Sep 17 00:00:00 2001 From: Jakub Buczak Date: Fri, 17 Apr 2026 16:51:19 +0200 Subject: [PATCH 13/19] CSPL-4630: add feature gate infrastructure using k8s FeatureGate pattern Introduce a feature gate registry backed by k8s.io/component-base/featuregate so new capabilities can be gated behind --feature-gates==true|false with Alpha/Beta/GA lifecycle. Migrate the validation webhook toggle from the ENABLE_VALIDATION_WEBHOOK env var to a ValidationWebhook feature gate with backwards-compatible env var support. - Add pkg/config/featuregates.go registry with SetupFeatureGates and IsFeatureGateEnabled helpers, plus unit tests - Wire feature gate flags into cmd/main.go and log effective gate states at startup - Use os.LookupEnv for ENABLE_VALIDATION_WEBHOOK so the deprecation warning fires whenever the env var is set, not just when it equals "true" - Update kustomization files with SPLUNK_GENERAL_TERMS value - Add docs/FeatureGates.md with usage and instructions for adding new gates - Update docs/ValidationWebhook.md with CLI-over-env-var precedence note --- cmd/main.go | 69 ++++++----- .../kustomization-cluster.yaml | 9 +- .../kustomization-namespace.yaml | 9 +- .../default-with-webhook/kustomization.yaml | 11 +- docs/FeatureGates.md | 110 ++++++++++++++++++ docs/ValidationWebhook.md | 39 ++++--- go.mod | 4 +- pkg/config/featuregates.go | 67 +++++++++++ pkg/config/featuregates_test.go | 91 +++++++++++++++ pkg/splunk/enterprise/telemetry.go | 3 +- 10 files changed, 350 insertions(+), 62 deletions(-) create mode 100644 docs/FeatureGates.md create mode 100644 pkg/config/featuregates.go create mode 100644 pkg/config/featuregates_test.go diff --git a/cmd/main.go b/cmd/main.go index b8188608d..f1633446a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,6 +25,7 @@ import ( "path/filepath" "time" + "github.com/spf13/pflag" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" intController "github.com/splunk/splunk-operator/internal/controller" @@ -89,24 +90,44 @@ func main() { // TLS certificate configuration for metrics var metricsCertPath, metricsCertName, metricsCertKey string - flag.StringVar(&logEncoder, "log-encoder", "json", "log encoding ('json' or 'console')") - flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&enableLeaderElection, "leader-elect", false, + pflag.StringVar(&logEncoder, "log-encoder", "json", "log encoding ('json' or 'console')") + pflag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + pflag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") - flag.BoolVar(&pprofActive, "pprof", true, "Enable pprof endpoint") - flag.IntVar(&logLevel, "log-level", int(zapcore.InfoLevel), "set log level") - flag.IntVar(&leaseDurationSecond, "lease-duration", leaseDurationSecond, "manager lease duration in seconds") - flag.IntVar(&renewDeadlineSecond, "renew-duration", renewDeadlineSecond, "manager renew duration in seconds") - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metrics endpoint binds to. "+ + pflag.BoolVar(&pprofActive, "pprof", true, "Enable pprof endpoint") + pflag.IntVar(&logLevel, "log-level", int(zapcore.InfoLevel), "set log level") + pflag.IntVar(&leaseDurationSecond, "lease-duration", leaseDurationSecond, "manager lease duration in seconds") + pflag.IntVar(&renewDeadlineSecond, "renew-duration", renewDeadlineSecond, "manager renew duration in seconds") + pflag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, + pflag.BoolVar(&secureMetrics, "metrics-secure", false, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") // TLS certificate flags for metrics server - flag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") - flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") - flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + pflag.StringVar(&metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.") + pflag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + pflag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") + + config.DefaultMutableFeatureGate.AddFlag(pflag.CommandLine) + + opts := zap.Options{ + Development: true, + TimeEncoder: zapcore.RFC3339NanoTimeEncoder, + } + opts.BindFlags(flag.CommandLine) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + if allGates := config.DefaultMutableFeatureGate.GetAll(); len(allGates) > 0 { + effectiveStates := make(map[string]bool, len(allGates)) + for gate := range allGates { + effectiveStates[string(gate)] = config.DefaultMutableFeatureGate.Enabled(gate) + } + setupLog.Info("Feature gates initialized", "gates", effectiveStates) + } // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: @@ -147,16 +168,6 @@ func main() { renewDeadline = time.Duration(renewDeadlineSecond) * time.Second } - opts := zap.Options{ - Development: true, - TimeEncoder: zapcore.RFC3339NanoTimeEncoder, - } - opts.BindFlags(flag.CommandLine) - flag.Parse() - - // Logging setup - ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - // Configure metrics certificate watcher if metrics certs are provided var metricsCertWatcher *certwatcher.CertWatcher if len(metricsCertPath) > 0 { @@ -280,10 +291,11 @@ func main() { os.Exit(1) } - // Setup centralized validation webhook server (opt-in via ENABLE_VALIDATION_WEBHOOK env var, defaults to false) - enableWebhooks := os.Getenv("ENABLE_VALIDATION_WEBHOOK") - if enableWebhooks == "true" { - // Parse optional timeout configurations from environment + if _, ok := os.LookupEnv("ENABLE_VALIDATION_WEBHOOK"); ok { + setupLog.Info("DEPRECATED: ENABLE_VALIDATION_WEBHOOK env var is deprecated and will be removed in a future release; use --feature-gates=ValidationWebhook=true instead") + } + + if config.DefaultMutableFeatureGate.Enabled(config.ValidationWebhook) { readTimeout := 10 * time.Second if val := os.Getenv("WEBHOOK_READ_TIMEOUT"); val != "" { if d, err := time.ParseDuration(val); err == nil { @@ -306,16 +318,15 @@ func main() { Client: mgr.GetClient(), }) - // Add webhook server as a runnable to the manager if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { return webhookServer.Start(ctx) })); err != nil { setupLog.Error(err, "unable to add webhook server to manager") os.Exit(1) } - setupLog.Info("Validation webhook enabled via ENABLE_VALIDATION_WEBHOOK=true") + setupLog.Info("Validation webhook enabled") } else { - setupLog.Info("Validation webhook disabled (set ENABLE_VALIDATION_WEBHOOK=true to enable)") + setupLog.Info("Validation webhook disabled (set --feature-gates=ValidationWebhook=true to enable)") } //+kubebuilder:scaffold:builder diff --git a/config/default-with-webhook/kustomization-cluster.yaml b/config/default-with-webhook/kustomization-cluster.yaml index c596f0c68..f4ddabdb2 100644 --- a/config/default-with-webhook/kustomization-cluster.yaml +++ b/config/default-with-webhook/kustomization-cluster.yaml @@ -1,7 +1,7 @@ # Adds namespace to all resources. # Cluster-scoped deployment WITH webhook enabled (opt-in) # Requires cert-manager to be installed in the cluster -namespace: splunk-operator +namespace: splunk-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -115,7 +115,7 @@ patches: patch: |- - op: add path: /spec/template/spec/containers/0/env - value: + value: - name: WATCH_NAMESPACE value: WATCH_NAMESPACE_VALUE - name: RELATED_IMAGE_SPLUNK_ENTERPRISE @@ -124,12 +124,13 @@ patches: value: splunk-operator - name: SPLUNK_GENERAL_TERMS value: SPLUNK_GENERAL_TERMS_VALUE - - name: ENABLE_VALIDATION_WEBHOOK - value: "true" - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name + - op: add + path: /spec/template/spec/containers/0/args/- + value: --feature-gates=ValidationWebhook=true # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics - path: manager_metrics_patch.yaml diff --git a/config/default-with-webhook/kustomization-namespace.yaml b/config/default-with-webhook/kustomization-namespace.yaml index 193791601..2bc1845c6 100644 --- a/config/default-with-webhook/kustomization-namespace.yaml +++ b/config/default-with-webhook/kustomization-namespace.yaml @@ -1,7 +1,7 @@ # Adds namespace to all resources. # Namespace-scoped deployment WITH webhook enabled (opt-in) # Requires cert-manager to be installed in the cluster -namespace: splunk-operator +namespace: splunk-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -115,7 +115,7 @@ patches: patch: |- - op: add path: /spec/template/spec/containers/0/env - value: + value: - name: WATCH_NAMESPACE valueFrom: fieldRef: @@ -126,12 +126,13 @@ patches: value: splunk-operator - name: SPLUNK_GENERAL_TERMS value: SPLUNK_GENERAL_TERMS_VALUE - - name: ENABLE_VALIDATION_WEBHOOK - value: "true" - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name + - op: add + path: /spec/template/spec/containers/0/args/- + value: --feature-gates=ValidationWebhook=true # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics - path: manager_metrics_patch.yaml diff --git a/config/default-with-webhook/kustomization.yaml b/config/default-with-webhook/kustomization.yaml index 5ba87fec1..f4ddabdb2 100644 --- a/config/default-with-webhook/kustomization.yaml +++ b/config/default-with-webhook/kustomization.yaml @@ -1,7 +1,7 @@ # Adds namespace to all resources. # Cluster-scoped deployment WITH webhook enabled (opt-in) # Requires cert-manager to be installed in the cluster -namespace: splunk-operator +namespace: splunk-operator # Value of this field is prepended to the # names of all resources, e.g. a deployment named @@ -115,7 +115,7 @@ patches: patch: |- - op: add path: /spec/template/spec/containers/0/env - value: + value: - name: WATCH_NAMESPACE value: WATCH_NAMESPACE_VALUE - name: RELATED_IMAGE_SPLUNK_ENTERPRISE @@ -123,13 +123,14 @@ patches: - name: OPERATOR_NAME value: splunk-operator - name: SPLUNK_GENERAL_TERMS - value: WATCH_NAMESPACE_VALUE - - name: ENABLE_VALIDATION_WEBHOOK - value: "true" + value: SPLUNK_GENERAL_TERMS_VALUE - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name + - op: add + path: /spec/template/spec/containers/0/args/- + value: --feature-gates=ValidationWebhook=true # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics - path: manager_metrics_patch.yaml diff --git a/docs/FeatureGates.md b/docs/FeatureGates.md new file mode 100644 index 000000000..6661ab055 --- /dev/null +++ b/docs/FeatureGates.md @@ -0,0 +1,110 @@ +# Feature Gates + +The Splunk Operator uses the Kubernetes [FeatureGate](https://pkg.go.dev/k8s.io/component-base/featuregate) pattern to control rollout of new functionality. Feature gates allow new code to be merged to the main branch without activating in production, giving teams a safe, per-environment opt-in mechanism. + +## Usage + +Enable or disable feature gates at operator startup: + +```bash +/manager --feature-gates=ValidationWebhook=true +``` + +## Maturity Lifecycle + +| Stage | Default | Can Override | Next Step | +|-----------|---------|-------------|-----------------------------------------| +| **Alpha** | off | Yes | Promote to Beta after validation | +| **Beta** | on | Yes | Promote to GA after sustained stability | +| **GA** | on | No | Remove gate in a future release | + +## Current Feature Gates + +| Gate | Default | Stage | Since | Description | +|-----------------------|---------|-------|---------|----------------------------------------------------------| +| `ValidationWebhook` | `false` | Alpha | v3.2.0 | Centralized validation webhook server for CR admission | + +## Adding a New Feature Gate + +Follow these steps: + +### 1. Register the gate in `pkg/config/featuregates.go` + +Add a constant and an entry in `defaultFeatureGates`: + +```go +const ( + MyNewFeature featuregate.Feature = "MyNewFeature" +) + +var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + // existing gates … + MyNewFeature: {Default: false, PreRelease: featuregate.Alpha}, +} +``` + +### 2. Guard the code path + +Check the gate wherever the feature-specific logic runs: + +```go +if config.DefaultMutableFeatureGate.Enabled(config.MyNewFeature) { + // feature-specific logic +} +``` + +This can guard anything — a reconciler code path, a helper function, a webhook handler, an HTTP endpoint, etc. + +### Example: Gating a New Controller (CRD) + +When the feature gate introduces an entirely new CRD and controller, there are additional steps beyond the basic gate check. All three steps below are **mandatory** for any new CRD behind a feature gate. + +#### a. Gate controller registration in `cmd/main.go` + +Wrap the `SetupWithManager` call so the controller only starts when the gate is on: + +```go +if config.DefaultMutableFeatureGate.Enabled(config.MyNewFeature) { + if err = (&controller.MyNewReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "MyNew") + os.Exit(1) + } +} +``` + +#### b. Add a validating webhook for the gated CRD group + +A validating webhook **must** reject CR creation when the gate is off. Without this, users can create resources that no controller will reconcile, leading to silent failures: + +```go +func (v *MyNewValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + if !config.DefaultMutableFeatureGate.Enabled(config.MyNewFeature) { + return nil, fmt.Errorf( + "the MyNewFeature feature is not enabled; "+ + "set --feature-gates=MyNewFeature=true to activate") + } + return nil, nil +} +``` + +#### c. Label the CRD manifests + +Every gated CRD **must** carry maturity annotations and labels in `config/crd/bases/`. These signal to operators and tooling which gate controls the CRD and its current stability level: + +```yaml +metadata: + annotations: + splunk.com/feature-gate: MyNewFeature + splunk.com/feature-stage: Alpha + labels: + splunk.com/feature-stage: alpha +``` + +## Promoting a Gate + +- **Alpha → Beta**: Change `Default: false` to `Default: true` in `featuregates.go`; update the CRD label to `beta` +- **Beta → GA**: Set `LockToDefault: true` in the `FeatureSpec`; update the CRD label to `ga` +- **GA → Removed**: Delete the constant and `FeatureSpec` entry; remove the `if` guard in `cmd/main.go`; remove the CRD annotations/labels and the validating webhook diff --git a/docs/ValidationWebhook.md b/docs/ValidationWebhook.md index a3ae4c2de..f17ad7535 100644 --- a/docs/ValidationWebhook.md +++ b/docs/ValidationWebhook.md @@ -85,23 +85,25 @@ If you prefer not to use cert-manager, you can provide your own TLS certificates #### Option 1: Use the Webhook-Enabled Kustomize Overlay -Deploy using the `config/default-with-webhook` overlay which includes all necessary webhook components: +Deploy using the `config/default-with-webhook` overlay which includes all necessary webhook components and enables the `ValidationWebhook` feature gate automatically: ```bash -# Build and apply the webhook-enabled configuration -kustomize build config/default-with-webhook | kubectl apply -f - +make deploy IMG= ENVIRONMENT=default-with-webhook \ + SPLUNK_GENERAL_TERMS="--accept-sgt-current-at-splunk-com" ``` -#### Option 2: Enable Webhook on Existing Deployment +This uses the same `make deploy` target as the standard deployment, which substitutes the `WATCH_NAMESPACE`, `SPLUNK_ENTERPRISE_IMAGE`, and `SPLUNK_GENERAL_TERMS` placeholder values before running `kustomize build`. -If you already have the operator deployed, you can enable the webhook by setting the `ENABLE_VALIDATION_WEBHOOK` environment variable: +#### Option 2: Enable via Feature Gate on Existing Deployment + +If you already have the operator deployed with the webhook Kubernetes resources (Service, ValidatingWebhookConfiguration, TLS certificates), enable the feature gate by patching the container args: ```bash -kubectl set env deployment/splunk-operator-controller-manager \ - ENABLE_VALIDATION_WEBHOOK=true -n splunk-operator +kubectl patch deployment splunk-operator-controller-manager -n splunk-operator \ + --type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--feature-gates=ValidationWebhook=true"}]' ``` -**Note:** This option also requires the webhook service, ValidatingWebhookConfiguration, and TLS certificates to be deployed. Use Option 1 for a complete deployment. +**Note:** This requires the webhook service, ValidatingWebhookConfiguration, and TLS certificates to already be deployed. Use Option 1 for a complete deployment. #### Option 3: Modify Default Kustomization @@ -119,6 +121,14 @@ Then deploy: make deploy IMG= SPLUNK_GENERAL_TERMS="--accept-sgt-current-at-splunk-com" ``` +### Legacy: ENABLE_VALIDATION_WEBHOOK Environment Variable + +> **Deprecated:** The `ENABLE_VALIDATION_WEBHOOK` environment variable is deprecated and will be removed in a future release. Use the `--feature-gates=ValidationWebhook=true` flag instead. + +For backwards compatibility, setting `ENABLE_VALIDATION_WEBHOOK=true` as an environment variable on the operator container will still enable the validation webhook. The operator logs a deprecation warning when this method is used. + +When both the `--feature-gates=ValidationWebhook=...` CLI flag and the `ENABLE_VALIDATION_WEBHOOK` env var are set, the **CLI flag takes precedence**. The env var is applied at startup before flag parsing, so the CLI value overwrites it. + ## Validated Fields The webhook validates the following spec fields: @@ -290,7 +300,7 @@ kubectl get validatingwebhookconfiguration splunk-operator-validating-webhook-co ```bash kubectl logs -n splunk-operator deployment/splunk-operator-controller-manager | grep -i webhook -# Look for: "Validation webhook enabled via ENABLE_VALIDATION_WEBHOOK=true" +# Look for: "Validation webhook enabled" # Look for: "Starting webhook server" {"port": 9443} ``` @@ -362,7 +372,7 @@ kubectl logs -n splunk-operator deployment/splunk-operator-controller-manager | If you see "Validation webhook disabled" in the logs, ensure: -1. The `ENABLE_VALIDATION_WEBHOOK` environment variable is set to `true` +1. The `--feature-gates=ValidationWebhook=true` flag is set on the operator container args (or the legacy `ENABLE_VALIDATION_WEBHOOK=true` env var is set) 2. You're using the correct kustomize overlay (`config/default-with-webhook`) ## Architecture @@ -457,7 +467,7 @@ var GVR = schema.GroupVersionResource{ // Add to DefaultValidators map var DefaultValidators = map[schema.GroupVersionResource]Validator{ // ... existing validators ... - + GVR: &GenericValidator[*enterpriseApi.]{ ValidateCreateFunc: ValidateCreate, ValidateUpdateFunc: ValidateUpdate, @@ -503,12 +513,7 @@ If your CRD doesn't need context-aware validation, you can omit `ValidateCreateW ## Disabling the Webhook -To disable the webhook after it has been enabled: - -```bash -kubectl set env deployment/splunk-operator-controller-manager \ - ENABLE_VALIDATION_WEBHOOK=false -n splunk-operator -``` +To disable the webhook after it has been enabled, remove the `--feature-gates=ValidationWebhook=true` flag from the container args (or remove the `ENABLE_VALIDATION_WEBHOOK` env var if using the legacy method). Or redeploy using the default kustomization (without webhook): diff --git a/go.mod b/go.mod index 8b5adb30c..0ff959e8b 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/onsi/gomega v1.39.1 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.11.1 github.com/wk8/go-ordered-map/v2 v2.1.7 go.uber.org/zap v1.27.0 @@ -31,6 +32,7 @@ require ( k8s.io/apiextensions-apiserver v0.33.0 k8s.io/apimachinery v0.33.0 k8s.io/client-go v0.33.0 + k8s.io/component-base v0.33.0 k8s.io/kubectl v0.26.2 sigs.k8s.io/controller-runtime v0.21.0 ) @@ -113,7 +115,6 @@ require ( github.com/rs/xid v1.2.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -152,7 +153,6 @@ require ( gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.33.0 // indirect - k8s.io/component-base v0.33.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect diff --git a/pkg/config/featuregates.go b/pkg/config/featuregates.go new file mode 100644 index 000000000..934807511 --- /dev/null +++ b/pkg/config/featuregates.go @@ -0,0 +1,67 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "os" + + "k8s.io/component-base/featuregate" +) + +// Feature gate constants. Add new feature gates here following the checklist +// in docs/FeatureGates.md. +// +// Lifecycle: +// +// Alpha – off by default, opt-in via --feature-gates==true +// Beta – on by default, opt-out via --feature-gates==false +// GA – on, locked; remove the gate in a subsequent release +const ( + // ValidationWebhook gates the centralized validation webhook server. + // When enabled, the operator runs a validating webhook that enforces + // CR schema rules at admission time. + // Replaces the legacy ENABLE_VALIDATION_WEBHOOK env var. + ValidationWebhook featuregate.Feature = "ValidationWebhook" +) + +// defaultFeatureGates is the authoritative registry of all feature gates and +// their default state / maturity. Each entry here automatically becomes +// available via --feature-gates on the operator binary. +var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + ValidationWebhook: {Default: false, PreRelease: featuregate.Alpha}, +} + +var DefaultMutableFeatureGate featuregate.MutableFeatureGate = featuregate.NewFeatureGate() + +func init() { + if err := DefaultMutableFeatureGate.Add(defaultFeatureGates); err != nil { + panic(err) + } + applyLegacyValidationWebhookEnv(DefaultMutableFeatureGate) +} + +// applyLegacyValidationWebhookEnv preserves backwards compatibility for +// deployments using the ENABLE_VALIDATION_WEBHOOK env var instead of +// --feature-gates=ValidationWebhook=true. +// Remove once the ENABLE_VALIDATION_WEBHOOK deprecation period ends. +func applyLegacyValidationWebhookEnv(fg featuregate.MutableFeatureGate) { + if os.Getenv("ENABLE_VALIDATION_WEBHOOK") == "true" { + if err := fg.SetFromMap(map[string]bool{string(ValidationWebhook): true}); err != nil { + fmt.Fprintf(os.Stderr, "WARNING: failed to apply legacy env var ENABLE_VALIDATION_WEBHOOK: %v\n", err) + } + } +} diff --git a/pkg/config/featuregates_test.go b/pkg/config/featuregates_test.go new file mode 100644 index 000000000..03f9d9779 --- /dev/null +++ b/pkg/config/featuregates_test.go @@ -0,0 +1,91 @@ +// Copyright (c) 2018-2026 Splunk Inc. All rights reserved. + +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "testing" + + "k8s.io/component-base/featuregate" +) + +func TestValidationWebhookRegistered(t *testing.T) { + all := DefaultMutableFeatureGate.GetAll() + spec, ok := all[ValidationWebhook] + if !ok { + t.Fatal("ValidationWebhook gate not registered") + } + if spec.Default != false { + t.Errorf("ValidationWebhook default: got %v, want false", spec.Default) + } + if spec.PreRelease != featuregate.Alpha { + t.Errorf("ValidationWebhook prerelease: got %v, want Alpha", spec.PreRelease) + } +} + +func TestValidationWebhookOffByDefault(t *testing.T) { + if DefaultMutableFeatureGate.Enabled(ValidationWebhook) { + t.Error("ValidationWebhook should be disabled by default (Alpha)") + } +} + +func TestLegacyEnvVarEnablesGate(t *testing.T) { + fg := featuregate.NewFeatureGate() + if err := fg.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + ValidationWebhook: {Default: false, PreRelease: featuregate.Alpha}, + }); err != nil { + t.Fatalf("Add: %v", err) + } + + t.Setenv("ENABLE_VALIDATION_WEBHOOK", "true") + applyLegacyValidationWebhookEnv(fg) + + if !fg.Enabled(ValidationWebhook) { + t.Error("ValidationWebhook should be enabled when ENABLE_VALIDATION_WEBHOOK=true") + } +} + +func TestLegacyEnvVarIgnoredWhenUnset(t *testing.T) { + fg := featuregate.NewFeatureGate() + if err := fg.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + ValidationWebhook: {Default: false, PreRelease: featuregate.Alpha}, + }); err != nil { + t.Fatalf("Add: %v", err) + } + + os.Unsetenv("ENABLE_VALIDATION_WEBHOOK") + applyLegacyValidationWebhookEnv(fg) + + if fg.Enabled(ValidationWebhook) { + t.Error("ValidationWebhook should remain disabled when env var is not set") + } +} + +func TestLegacyEnvVarIgnoredWhenNotTrue(t *testing.T) { + fg := featuregate.NewFeatureGate() + if err := fg.Add(map[featuregate.Feature]featuregate.FeatureSpec{ + ValidationWebhook: {Default: false, PreRelease: featuregate.Alpha}, + }); err != nil { + t.Fatalf("Add: %v", err) + } + + t.Setenv("ENABLE_VALIDATION_WEBHOOK", "false") + applyLegacyValidationWebhookEnv(fg) + + if fg.Enabled(ValidationWebhook) { + t.Error("ValidationWebhook should remain disabled when ENABLE_VALIDATION_WEBHOOK=false") + } +} diff --git a/pkg/splunk/enterprise/telemetry.go b/pkg/splunk/enterprise/telemetry.go index a095e1e6e..0339d43f1 100644 --- a/pkg/splunk/enterprise/telemetry.go +++ b/pkg/splunk/enterprise/telemetry.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "time" + enterpriseApiV3 "github.com/splunk/splunk-operator/api/v3" enterpriseApi "github.com/splunk/splunk-operator/api/v4" splclient "github.com/splunk/splunk-operator/pkg/splunk/client" @@ -13,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "time" corev1 "k8s.io/api/core/v1" ) From bfeb9b1f26daf7265f2c84ebf69dc030e1c677f1 Mon Sep 17 00:00:00 2001 From: Igor Grzankowski Date: Mon, 20 Apr 2026 18:55:22 +0200 Subject: [PATCH 14/19] CSPL-4386: Extract inline YAMLs to fixtures (#1801) * Embed values * Embed values * Move fixtures * Fix copyrights --- pkg/splunk/client/enterprise_test.go | 24 +- pkg/splunk/client/test_helpers_test.go | 39 +++ .../testdata/fixtures/get_cluster_info.json | 98 ++++++ .../testdata/fixtures/get_cm_info_empty.json | 16 + .../testdata/fixtures/get_cm_peers.json | 113 +++++++ .../get_indexer_cluster_peer_info.json | 72 ++++ .../get_indexer_cluster_peer_info_empty.json | 16 + .../testdata/fixtures/get_mc_asset_table.json | 122 +++++++ .../get_mc_nav_default_distributed.json | 70 ++++ .../testdata/fixtures/get_server_info.json | 98 ++++++ .../testdata/fixtures/mc_apply_changes.json | 122 +++++++ pkg/splunk/common/urls_testingcode.go | 307 ------------------ pkg/splunk/enterprise/clustermaster_test.go | 32 +- pkg/splunk/enterprise/configuration_test.go | 16 +- pkg/splunk/enterprise/indexercluster_test.go | 26 +- pkg/splunk/enterprise/licensemaster_test.go | 4 +- pkg/splunk/enterprise/test_constants_test.go | 52 +++ .../add_storage_volumes_admin_managed_pv.json | 127 ++++++++ .../add_storage_volumes_custom_storage.json | 96 ++++++ .../fixtures/add_storage_volumes_default.json | 94 ++++++ .../add_storage_volumes_ephemeral_both.json | 64 ++++ .../add_storage_volumes_ephemeral_etc.json | 81 +++++ .../add_storage_volumes_ephemeral_var.json | 81 +++++ .../testdata/fixtures/get_probe.json | 12 + ...et_volume_source_mount_from_configmap.json | 23 ++ .../indexer_cluster_pod_manager_info.json | 100 ++++++ .../indexer_cluster_pod_manager_peer.json | 113 +++++++ .../invalid_peer_in_finish_recycle_info.json | 100 ++++++ .../invalid_peer_in_finish_recycle_peer.json | 113 +++++++ ...nvalid_peer_status_in_scale_down_info.json | 100 ++++++ ...nvalid_peer_status_in_scale_down_peer.json | 113 +++++++ .../testdata/fixtures/quorum_lost_info.json | 27 ++ .../update_status_invalid_response1.json | 113 +++++++ pkg/splunk/enterprise/util.go | 21 -- test/testenv/appframework_utils.go | 2 +- test/testenv/verificationutils.go | 2 +- 36 files changed, 2228 insertions(+), 381 deletions(-) create mode 100644 pkg/splunk/client/test_helpers_test.go create mode 100644 pkg/splunk/client/testdata/fixtures/get_cluster_info.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_cm_info_empty.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_cm_peers.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_indexer_cluster_peer_info.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_indexer_cluster_peer_info_empty.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_mc_asset_table.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_mc_nav_default_distributed.json create mode 100644 pkg/splunk/client/testdata/fixtures/get_server_info.json create mode 100644 pkg/splunk/client/testdata/fixtures/mc_apply_changes.json delete mode 100644 pkg/splunk/common/urls_testingcode.go create mode 100644 pkg/splunk/enterprise/test_constants_test.go create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_admin_managed_pv.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_custom_storage.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_default.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_ephemeral_both.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_ephemeral_etc.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/add_storage_volumes_ephemeral_var.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/get_probe.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/get_volume_source_mount_from_configmap.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/indexer_cluster_pod_manager_info.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/indexer_cluster_pod_manager_peer.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/invalid_peer_in_finish_recycle_info.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/invalid_peer_in_finish_recycle_peer.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/invalid_peer_status_in_scale_down_info.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/invalid_peer_status_in_scale_down_peer.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/quorum_lost_info.json create mode 100644 pkg/splunk/enterprise/testdata/fixtures/update_status_invalid_response1.json diff --git a/pkg/splunk/client/enterprise_test.go b/pkg/splunk/client/enterprise_test.go index 2c902d537..b605663cb 100644 --- a/pkg/splunk/client/enterprise_test.go +++ b/pkg/splunk/client/enterprise_test.go @@ -311,7 +311,7 @@ func TestGetclusterManagerInfo(t *testing.T) { } return nil } - body = splcommon.TestGetCMInfoEmpty + body = loadFixture(t, "get_cm_info_empty.json") splunkClientTester(t, "TestGetclusterManagerInfo", 200, body, wantRequest, test) // test error code @@ -331,7 +331,7 @@ func TestGetIndexerClusterPeerInfo(t *testing.T) { } return nil } - body := splcommon.TestGetIndexerClusterPeerInfo + body := loadFixture(t, "get_indexer_cluster_peer_info.json") splunkClientTester(t, "TestGetIndexerClusterPeerInfo", 200, body, wantRequest, test) // test body with no entries @@ -342,7 +342,7 @@ func TestGetIndexerClusterPeerInfo(t *testing.T) { } return nil } - body = splcommon.TestGetIndexerClusterPeerInfoEmpty + body = loadFixture(t, "get_indexer_cluster_peer_info_empty.json") splunkClientTester(t, "TestGetIndexerClusterPeerInfo", 200, body, wantRequest, test) // test error code @@ -383,7 +383,7 @@ func TestGetClusterManagerPeers(t *testing.T) { } return nil } - body := splcommon.TestGetCMPeers + body := loadFixture(t, "get_cm_peers.json") splunkClientTester(t, "TestGetClusterManagerPeers", 200, body, wantRequest, test) // test error response @@ -433,14 +433,14 @@ func TestAutomateMCApplyChanges(t *testing.T) { var wantRequests []*http.Request wantRequests = []*http.Request(append(wantRequests, request1, request2, request3, request4, request5, request6, request7, request8, request9, request10)) body := []string{ - `{"links":{},"origin":"https://localhost:8089/services/server/info","updated":"2020-09-24T06:38:53+00:00","generator":{"build":"a1a6394cc5ae","version":"8.0.5"},"entry":[{"name":"server-info","id":"https://localhost:8089/services/server/info/server-info","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/server/info/server-info","list":"/services/server/info/server-info"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["*"],"write":[]},"removable":false,"sharing":"system"},"fields":{"required":[],"optional":[],"wildcard":[]},"content":{"activeLicenseGroup":"Trial","activeLicenseSubgroup":"Production","build":"a1a6394cc5ae","cluster_label":["idxc_label"],"cpu_arch":"x86_64","eai:acl":null,"fips_mode":false,"guid":"0F93F33C-4BDA-4A74-AD9F-3FCE26C6AFF0","health_info":"green","health_version":1,"host":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","host_fqdn":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","host_resolved":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","isForwarding":true,"isFree":false,"isTrial":true,"kvStoreStatus":"ready","licenseKeys":["5C52DA5145AD67B8188604C49962D12F2C3B2CF1B82A6878E46F68CA2812807B"],"licenseSignature":"139bf73ec92c84121c79a9b8307a6724","licenseState":"OK","license_labels":["Splunk Enterprise Splunk Analytics for Hadoop Download Trial"],"master_guid":"0F93F33C-4BDA-4A74-AD9F-3FCE26C6AFF0","master_uri":"self","max_users":4294967295,"mode":"normal","numberOfCores":1,"numberOfVirtualCores":2,"os_build":"#1 SMP Thu Sep 3 19:04:44 UTC 2020","os_name":"Linux","os_name_extended":"Linux","os_version":"4.14.193-149.317.amzn2.x86_64","physicalMemoryMB":7764,"product_type":"enterprise","rtsearch_enabled":true,"serverName":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","server_roles":["license_master","cluster_search_head","search_head"],"startup_time":1600928786,"staticAssetId":"CFE3D41EE2CCD1465E8C8453F83E4ECFFF540780B4490E84458DD4A3694CE4D1","version":"8.0.5"}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}`, - splcommon.TestMCApplyChanges, + loadFixture(t, "get_server_info.json"), + loadFixture(t, "mc_apply_changes.json"), "", "", "", - `{"links":{"create":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_new","_reload":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_reload","_acl":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_acl"},"origin":"https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/data/ui/nav","updated":"2020-09-23T23:50:25+00:00","generator":{"build":"a1a6394cc5ae","version":"8.0.5"},"entry":[{"name":"default.distributed","id":"https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","list":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","_reload":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed/_reload","edit":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed"},"author":"nobody","acl":{"app":"splunk_monitoring_console","can_change_perms":true,"can_list":true,"can_share_app":true,"can_share_global":true,"can_share_user":false,"can_write":true,"modifiable":true,"owner":"nobody","perms":{"read":["admin"],"write":["admin"]},"removable":false,"sharing":"app"},"fields":{"required":["eai:data"],"optional":[],"wildcard":[]},"content":{"disabled":false,"eai:acl":null,"eai:appName":"splunk_monitoring_console","eai:data":"