[experimental] Sync Replay (etaging)

cmd/baton-github/main.go

@@ -14,5 +14,8 @@ var version = "dev"
 
 func main() {
 	ctx := context.Background()
-	config.RunConnector(ctx, "baton-github", version, cfg.Config, connector.NewLambdaConnector, connectorrunner.WithSessionStoreEnabled())
+	config.RunConnector(ctx, "baton-github", version, cfg.Config, connector.NewLambdaConnector,
+		connectorrunner.WithSessionStoreEnabled(),
+		connectorrunner.WithKeepPreviousSyncC1Z(),
+	)
 }

go.mod

@@ -2,6 +2,8 @@ module github.com/conductorone/baton-github
 
 go 1.25.2
 
+replace github.com/conductorone/baton-sdk => /Users/matt.kaniaris/work/baton-sdk-2
+
 require (
 	github.com/conductorone/baton-sdk v0.17.0
 	github.com/deckarep/golang-set/v2 v2.9.0

pkg/config/config.go

@@ -94,6 +94,13 @@ var Config = field.NewConfiguration(
 		syncSecrets,
 		omitArchivedRepositories,
 		directCollaboratorsOnly,
+		// Default to a small worker pool: the source-cache warm path fans
+		// member pages out via SpawnCursors, and cold syncs parallelize
+		// per-resource work. 4 keeps sustained request rate well under
+		// GitHub's secondary (points-per-minute) limits; the SDK limiter
+		// backs off on 429/secondary responses regardless. --workers/
+		// BATON_WORKERS still override.
+		field.WithConnectorDefault(field.WorkerCountField, 4),
 	},
 	field.WithConnectorDisplayName("GitHub v2"),
 	field.WithHelpUrl("/docs/baton/github-v2"),

pkg/connector/connector.go

@@ -3,13 +3,17 @@ package connector
 import (
 	"context"
 	"crypto/rsa"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/hex"
 	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -127,11 +131,18 @@ type GitHub struct {
 	omitArchivedRepositories bool
 	directCollaboratorsOnly  bool
 	enterprises              []string
+	// authScope is a stable, non-secret identity for the configured
+	// credential, mixed into source-cache scope signatures so two
+	// different credentials (with different member visibility) never
+	// share cached scopes. PATs use a token digest; GitHub Apps use the
+	// stable installation identity, which survives hourly installation
+	// token rotation.
+	authScope string
 }
 
 func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 {
 	resourceSyncers := []connectorbuilder.ResourceSyncerV2{
-		OrgBuilder(gh.client, gh.appClient, gh.orgCache, gh.orgs, gh.syncSecrets),
+		OrgBuilder(gh.client, gh.appClient, gh.orgCache, gh.orgs, gh.syncSecrets, gh.authScope),
 		TeamBuilder(gh.client, gh.orgCache, gh.directCollaboratorsOnly),
 		UserBuilder(gh.client, gh.graphqlClient, gh.orgCache, gh.orgs, gh.customClient, gh.enterprises),
 		RepositoryBuilder(gh.client, gh.orgCache, gh.omitArchivedRepositories, gh.directCollaboratorsOnly),
@@ -198,6 +209,17 @@ func (gh *GitHub) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) {
 	}, nil
 }
 
+// allowNonAdminOrgs reports whether the org-admin requirement is relaxed
+// (BATON_GITHUB_ALLOW_NON_ADMIN=true). EXPERIMENTAL escape hatch for
+// read-only sync testing with a member-level token: org member listings
+// are fully readable by any org member, but admin-only surfaces
+// (invitations, org roles, secrets) and provisioning will fail or come
+// back partial. Do not use in production.
+func allowNonAdminOrgs() bool {
+	v, _ := strconv.ParseBool(os.Getenv("BATON_GITHUB_ALLOW_NON_ADMIN"))
+	return v
+}
+
 // Validate hits the GitHub API to validate that the configured credentials are still valid.
 func (gh *GitHub) Validate(ctx context.Context) (annotations.Annotations, error) {
 	if gh.appClient != nil {
@@ -229,7 +251,7 @@ func (gh *GitHub) Validate(ctx context.Context) (annotations.Annotations, error)
 		}
 
 		// Only sync orgs that we are an admin for
-		if strings.ToLower(membership.GetRole()) != orgRoleAdmin {
+		if strings.ToLower(membership.GetRole()) != orgRoleAdmin && !allowNonAdminOrgs() {
 			if filterOrgs {
 				err := fmt.Errorf("access token must be an admin on the %s organization", o)
 				return nil, uhttp.WrapErrors(codes.PermissionDenied, "github-connector: credentials validation failed", err)
@@ -254,7 +276,7 @@ func (gh *GitHub) Validate(ctx context.Context) (annotations.Annotations, error)
 				zap.Error(err))
 		}
 	}
-	return nil, nil
+	return sourceCacheCapabilityAnnotations(), nil
 }
 
 func (gh *GitHub) validateAppCredentials(ctx context.Context) (annotations.Annotations, error) {
@@ -279,7 +301,13 @@ func (gh *GitHub) validateAppCredentials(ctx context.Context) (annotations.Annot
 		}
 	}
 
-	return nil, nil
+	return sourceCacheCapabilityAnnotations(), nil
+}
+
+func sourceCacheCapabilityAnnotations() annotations.Annotations {
+	return annotations.New(v2.SourceCacheCapability_builder{
+		Mode: v2.SourceCacheCapability_MODE_READ_WRITE,
+	}.Build())
 }
 
 // newGitHubClient returns a new GitHub API client authenticated with an access token via oauth2.
@@ -346,9 +374,20 @@ func newWithGithubPAT(ctx context.Context, ghc *cfg.Github) (*GitHub, error) {
 		syncSecrets:              ghc.SyncSecrets,
 		omitArchivedRepositories: ghc.OmitArchivedRepositories,
 		directCollaboratorsOnly:  ghc.DirectCollaboratorsOnly,
+		authScope:                patAuthScope(ghc.Token),
 	}, nil
 }
 
+// patAuthScope derives a stable, non-secret identity from a personal
+// access token for source-cache scope isolation. The digest is truncated
+// (it only needs to distinguish credentials, not resist collision
+// attacks) and never logged alongside anything that would identify it as
+// token-derived material.
+func patAuthScope(token string) string {
+	sum := sha256.Sum256([]byte("baton-github-auth-scope|" + token))
+	return "pat:" + hex.EncodeToString(sum[:8])
+}
+
 func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) {
 	jwttoken, err := getJWTToken(ghc.AppId, string(ghc.AppPrivatekeyPath))
 	if err != nil {
@@ -433,6 +472,10 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) {
 		syncSecrets:              ghc.SyncSecrets,
 		omitArchivedRepositories: ghc.OmitArchivedRepositories,
 		directCollaboratorsOnly:  ghc.DirectCollaboratorsOnly,
+		// The installation identity is stable across the hourly
+		// installation-token rotation, so cached scopes survive rotation
+		// (a per-token digest would go cold every hour).
+		authScope: fmt.Sprintf("app:%s:install:%d", ghc.AppId, installation.GetID()),
 	}
 	return gh, nil
 }