diff --git a/log/handler_glog.go b/log/handler_glog.go
index e3c670c2a7..4694d04a50 100644
--- a/log/handler_glog.go
+++ b/log/handler_glog.go
@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
-	"maps"
 	"regexp"
 	"runtime"
 	"strconv"
@@ -42,18 +41,17 @@ type GlogHandler struct {
 	level    atomic.Int32 // Current log level, atomically accessible
 	override atomic.Bool  // Flag whether overrides are used, atomically accessible
 
-	patterns  []pattern              // Current list of patterns to override with
-	siteCache map[uintptr]slog.Level // Cache of callsite pattern evaluations
-	location  string                 // file:line location where to do a stackdump at
-	lock      sync.RWMutex           // Lock protecting the override pattern list
+	patterns  []pattern    // Current list of patterns to override with
+	siteCache sync.Map     // Cache of callsite pattern evaluations, maps uintptr -> slog.Level
+	location  string       // file:line location where to do a stackdump at
+	lock      sync.RWMutex // Lock protecting the override pattern list
 }
 
 // NewGlogHandler creates a new log handler with filtering functionality similar
 // to Google's glog logger. The returned handler implements Handler.
 func NewGlogHandler(h slog.Handler) *GlogHandler {
 	return &GlogHandler{
-		origin:    h,
-		siteCache: make(map[uintptr]slog.Level),
+		origin: h,
 	}
 }
 
@@ -69,7 +67,7 @@ type pattern struct {
 func (h *GlogHandler) Verbosity(level slog.Level) {
 	h.level.Store(int32(level))
 	// clear the cache to make sure the verbosity is applied correctly.
-	h.siteCache = make(map[uintptr]slog.Level)
+	h.siteCache.Clear()
 }
 
 // Vmodule sets the glog verbosity pattern.
@@ -133,10 +131,9 @@ func (h *GlogHandler) Vmodule(ruleset string) error {
 	}
 	// Swap out the vmodule pattern for the new filter system
 	h.lock.Lock()
-	defer h.lock.Unlock()
-
 	h.patterns = filter
-	h.siteCache = make(map[uintptr]slog.Level)
+	h.lock.Unlock()
+	h.siteCache.Clear()
 	h.override.Store(len(filter) != 0)
 
 	return nil
@@ -152,18 +149,15 @@ func (h *GlogHandler) Enabled(ctx context.Context, lvl slog.Level) bool {
 // WithAttrs implements slog.Handler, returning a new Handler whose attributes
 // consist of both the receiver's attributes and the arguments.
 func (h *GlogHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
+	patterns := []pattern{}
 	h.lock.RLock()
-	siteCache := maps.Clone(h.siteCache)
+	patterns = append(patterns, h.patterns...)
 	h.lock.RUnlock()
 
-	patterns := []pattern{}
-	patterns = append(patterns, h.patterns...)
-
 	res := GlogHandler{
-		origin:    h.origin.WithAttrs(attrs),
-		patterns:  patterns,
-		siteCache: siteCache,
-		location:  h.location,
+		origin:   h.origin.WithAttrs(attrs),
+		patterns: patterns,
+		location: h.location,
 	}
 
 	res.level.Store(h.level.Load())
@@ -183,30 +177,28 @@ func (h *GlogHandler) WithGroup(name string) slog.Handler {
 // local and backtrace filters, finally emitting it if either allow it through.
 func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error {
 	// Check callsite cache for previously calculated log levels
-	h.lock.RLock()
-	lvl, ok := h.siteCache[r.PC]
-	h.lock.RUnlock()
+	lvl, ok := h.siteCache.Load(r.PC)
 
 	// If we didn't cache the callsite yet, calculate it
 	if !ok {
-		h.lock.Lock()
-
 		fs := runtime.CallersFrames([]uintptr{r.PC})
 		frame, _ := fs.Next()
 
+		h.lock.RLock()
 		for _, rule := range h.patterns {
 			if rule.pattern.MatchString(fmt.Sprintf("+%s", frame.File)) {
-				h.siteCache[r.PC], lvl, ok = rule.level, rule.level, true
+				h.siteCache.Store(r.PC, rule.level)
+				lvl, ok = rule.level, true
 			}
 		}
+		h.lock.RUnlock()
 		// If no rule matched, use the default log lvl
 		if !ok {
 			lvl = slog.Level(h.level.Load())
-			h.siteCache[r.PC] = lvl
+			h.siteCache.Store(r.PC, lvl)
 		}
-		h.lock.Unlock()
 	}
-	if lvl <= r.Level {
+	if lvl.(slog.Level) <= r.Level {
 		return h.origin.Handle(context.Background(), r)
 	}
 	return nil