Skip to content

Commit c0ec3eb

Browse files
hessjcghessjcg
committed
fix: Improve Stackdriver error logging in Cloud SQL Proxy
When Stackdriver export fails, extract details from the gRPC error and log them. If the error suggests a permission issue, add a hint suggesting the user check their IAM permissions (Metric Writer and Trace Agent roles).
1 parent e0935a5 commit c0ec3eb

4 files changed

Lines changed: 145 additions & 8 deletions

File tree

‎cmd/root.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ import (
4444
"github.com/spf13/pflag"
4545
"github.com/spf13/viper"
4646
"go.opencensus.io/trace"
47+
"google.golang.org/genproto/googleapis/rpc/errdetails"
48+
"google.golang.org/grpc/codes"
49+
"google.golang.org/grpc/status"
4750
)
4851

4952
var (
@@ -1070,6 +1073,9 @@ func runSignalWrapper(cmd *Command) (err error) {
10701073
sd, err := stackdriver.NewExporter(stackdriver.Options{
10711074
ProjectID: cmd.conf.TelemetryProject,
10721075
MetricPrefix: cmd.conf.TelemetryPrefix,
1076+
OnError: func(err error) {
1077+
cmd.logger.Errorf("Failed to export to Stackdriver: %v", formatStackdriverError(err))
1078+
},
10731079
})
10741080
if err != nil {
10751081
return err
@@ -1299,3 +1305,58 @@ func startHTTPServer(ctx context.Context, l cloudsql.Logger, addr string, mux *h
12991305
l.Errorf("failed to shutdown HTTP server: %v\n", err)
13001306
}
13011307
}
1308+
1309+
func formatStackdriverError(err error) string {
1310+
st, ok := status.FromError(err)
1311+
if !ok {
1312+
return err.Error()
1313+
}
1314+
1315+
// Default message: code and description
1316+
msg := fmt.Sprintf("rpc error: code = %s desc = %s", st.Code(), st.Message())
1317+
1318+
// Try to extract details
1319+
details := st.Details()
1320+
if len(details) > 0 {
1321+
var detailStrings []string
1322+
for _, detail := range details {
1323+
switch d := detail.(type) {
1324+
case *errdetails.ErrorInfo:
1325+
detailStrings = append(detailStrings, fmt.Sprintf("ErrorInfo: Reason=%s, Domain=%s, Metadata=%v", d.Reason, d.Domain, d.Metadata))
1326+
case *errdetails.Help:
1327+
for _, link := range d.Links {
1328+
detailStrings = append(detailStrings, fmt.Sprintf("Help: %s (%s)", link.Description, link.Url))
1329+
}
1330+
default:
1331+
detailStrings = append(detailStrings, fmt.Sprintf("%T: %v", d, d))
1332+
}
1333+
}
1334+
if len(detailStrings) > 0 {
1335+
msg = fmt.Sprintf("%s (details: %s)", msg, strings.Join(detailStrings, "; "))
1336+
}
1337+
}
1338+
1339+
// Add a general hint if we suspect permission issues based on code, message or details
1340+
isPermissionIssue := st.Code() == codes.PermissionDenied ||
1341+
strings.Contains(strings.ToLower(st.Message()), "permission") ||
1342+
strings.Contains(strings.ToLower(st.Message()), "denied")
1343+
1344+
if !isPermissionIssue {
1345+
// Also check details for permission-like reasons
1346+
for _, detail := range details {
1347+
if ei, ok := detail.(*errdetails.ErrorInfo); ok {
1348+
if ei.Reason == "ACCESS_TOKEN_SCOPE_INSUFFICIENT" ||
1349+
strings.Contains(strings.ToLower(ei.Reason), "permission") {
1350+
isPermissionIssue = true
1351+
break
1352+
}
1353+
}
1354+
}
1355+
}
1356+
1357+
if isPermissionIssue {
1358+
msg = fmt.Sprintf("%s - Hint: The Service Account may be missing required IAM permissions. Please ensure it has 'Monitoring Metric Writer' (roles/monitoring.metricWriter) and 'Cloud Trace Agent' (roles/cloudtrace.agent) roles.", msg)
1359+
}
1360+
1361+
return msg
1362+
}

‎cmd/root_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ import (
3131
"github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/internal/proxy"
3232
"github.com/google/go-cmp/cmp"
3333
"github.com/spf13/cobra"
34+
"google.golang.org/genproto/googleapis/rpc/errdetails"
35+
"google.golang.org/grpc/codes"
36+
"google.golang.org/grpc/status"
3437
)
3538

3639
func withDefaults(c *proxy.Config) *proxy.Config {
@@ -1576,3 +1579,76 @@ func TestQuitQuitQuitWithErrors(t *testing.T) {
15761579
t.Fatalf("want = %v, got = %v", errCloseFailed, got)
15771580
}
15781581
}
1582+
1583+
func TestFormatStackdriverError(t *testing.T) {
1584+
tcs := []struct {
1585+
desc string
1586+
err error
1587+
wantSub string
1588+
wantHint bool
1589+
}{
1590+
{
1591+
desc: "generic error",
1592+
err: errors.New("some random error"),
1593+
wantSub: "some random error",
1594+
wantHint: false,
1595+
},
1596+
{
1597+
desc: "gRPC error without details",
1598+
err: status.Error(codes.Internal, "Internal error encountered"),
1599+
wantSub: "rpc error: code = Internal desc = Internal error encountered",
1600+
wantHint: false,
1601+
},
1602+
{
1603+
desc: "gRPC PermissionDenied error",
1604+
err: status.Error(codes.PermissionDenied, "Permission denied"),
1605+
wantSub: "rpc error: code = PermissionDenied desc = Permission denied",
1606+
wantHint: true,
1607+
},
1608+
{
1609+
desc: "gRPC error with ErrorInfo permission reason",
1610+
err: func() error {
1611+
st, _ := status.New(codes.Internal, "Internal error encountered").WithDetails(
1612+
&errdetails.ErrorInfo{
1613+
Reason: "ACCESS_TOKEN_SCOPE_INSUFFICIENT",
1614+
Domain: "googleapis.com",
1615+
},
1616+
)
1617+
return st.Err()
1618+
}(),
1619+
wantSub: "rpc error: code = Internal desc = Internal error encountered",
1620+
wantHint: true,
1621+
},
1622+
{
1623+
desc: "gRPC error with Help link",
1624+
err: func() error {
1625+
st, _ := status.New(codes.Aborted, "Aborted").WithDetails(
1626+
&errdetails.Help{
1627+
Links: []*errdetails.Help_Link{
1628+
{
1629+
Description: "See docs",
1630+
Url: "http://docs",
1631+
},
1632+
},
1633+
},
1634+
)
1635+
return st.Err()
1636+
}(),
1637+
wantSub: "Help: See docs (http://docs)",
1638+
wantHint: false,
1639+
},
1640+
}
1641+
1642+
for _, tc := range tcs {
1643+
t.Run(tc.desc, func(t *testing.T) {
1644+
got := formatStackdriverError(tc.err)
1645+
if !strings.Contains(got, tc.wantSub) {
1646+
t.Errorf("formatStackdriverError(%v) = %q, want to contain %q", tc.err, got, tc.wantSub)
1647+
}
1648+
hasHint := strings.Contains(got, "Hint: The Service Account may be missing required IAM permissions")
1649+
if hasHint != tc.wantHint {
1650+
t.Errorf("formatStackdriverError(%v) = %q, hasHint = %v, wantHint = %v", tc.err, got, hasHint, tc.wantHint)
1651+
}
1652+
})
1653+
}
1654+
}

‎go.mod‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ require (
1919
golang.org/x/oauth2 v0.36.0
2020
golang.org/x/sys v0.47.0
2121
google.golang.org/api v0.289.0
22+
google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a
23+
google.golang.org/grpc v1.82.1
2224
gopkg.in/natefinch/lumberjack.v2 v2.2.1
2325
)
2426

@@ -79,9 +81,7 @@ require (
7981
golang.org/x/sync v0.22.0 // indirect
8082
golang.org/x/text v0.40.0 // indirect
8183
golang.org/x/time v0.15.0 // indirect
82-
google.golang.org/genproto v0.0.0-20260720211330-0afa2a65878a // indirect
83-
google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect
84-
google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect
85-
google.golang.org/grpc v1.82.1 // indirect
84+
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect
85+
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
8686
google.golang.org/protobuf v1.36.11 // indirect
8787
)

‎go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -640,10 +640,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc
640640
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
641641
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
642642
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
643-
google.golang.org/genproto v0.0.0-20260720211330-0afa2a65878a h1:MVNwR9RFj7qfpMtNK71pq97FgrLG0lVHZh+VbM2LZeI=
644-
google.golang.org/genproto v0.0.0-20260720211330-0afa2a65878a/go.mod h1:0qnvndM9dUhat9AtF1jqYN6WZ+tMxEAFImo3WNvUX7w=
645-
google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a h1:97PfJ4tCxY5C7NzzgGqQEMZmXbISdvSArNNEOoUGKBg=
646-
google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a/go.mod h1:1brfde68Npq6+WA75c1EHWPijZEG1kMus61ygPZfn4A=
643+
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
644+
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
645+
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU=
646+
google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA=
647647
google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a h1:qI/YMH1ep2qQtqcp00gMQyoU7mjvbhg88GJKCvfoLj0=
648648
google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
649649
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

0 commit comments

Comments
 (0)