diff --git a/.github/workflows/go-cover.yml b/.github/workflows/go-cover.yml index 326ee2e34dc0..18c4b7bbf818 100644 --- a/.github/workflows/go-cover.yml +++ b/.github/workflows/go-cover.yml @@ -1,79 +1,94 @@ +name: Hermetic Go tests with coverage and race detector + on: + push: + branches: [main] + paths: + - "**/*.go" + - "go.mod" + - "go.sum" + - ".github/workflows/go-cover.yml" + - "codecov.yml" + pull_request: + paths: + - "**/*.go" + - "go.mod" + - "go.sum" + - ".github/workflows/go-cover.yml" + - "codecov.yml" schedule: - cron: "0 5 * * 1,4" workflow_dispatch: -name: Go tests with coverage and race detector + +permissions: + contents: read + +concurrency: + group: go-cover-${{ github.ref }} + cancel-in-progress: true + jobs: test: runs-on: ubuntu-22.04-large steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Login to GCR - uses: docker/login-action@v3 - with: - registry: gcr.io - username: _json_key - password: ${{ secrets.RILL_BINARY_SA }} - - name: Restore DuckDB extensions - id: restore-duckdb-extensions - uses: actions/cache/restore@v5 - with: - path: ~/.duckdb/extensions - key: ${{ runner.os }}-cache-duckdb-extensions - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.26.5 - - name: Go test with coverage - run: |- - # Build list of packages to include in coverage, excluding generated code in 'proto/gen' - PACKAGES=$(go list ./... | grep -v 'proto/gen/' | tr '\n' ',' | sed -e 's/,$//' | sed -e 's/github.com\/rilldata\/rill/./g') - # Run tests with coverage output - go test ./... -timeout 60m -v -race -covermode=atomic -coverprofile=coverage.out -coverpkg=$PACKAGES - env: - RILL_RUNTIME_TEST_MODE: expensive - RILL_RUNTIME_DRUID_TEST_DSN: ${{ secrets.RILL_RUNTIME_DRUID_TEST_DSN }} - RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON }} - RILL_RUNTIME_SNOWFLAKE_TEST_DSN: ${{ secrets.RILL_RUNTIME_SNOWFLAKE_TEST_DSN }} - RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON }} - RILL_RUNTIME_GCS_TEST_HMAC_KEY: ${{ secrets.RILL_RUNTIME_GCS_TEST_HMAC_KEY }} - RILL_RUNTIME_GCS_TEST_HMAC_SECRET: ${{ secrets.RILL_RUNTIME_GCS_TEST_HMAC_SECRET }} - RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} - RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} - RILL_RUNTIME_AZURE_TEST_CONNECTION_STRING: ${{ secrets.RILL_RUNTIME_AZURE_TEST_CONNECTION_STRING }} - RILL_RUNTIME_ATHENA_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} - RILL_RUNTIME_ATHENA_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} - RILL_RUNTIME_REDSHIFT_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} - RILL_RUNTIME_REDSHIFT_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} - RILL_RUNTIME_MOTHERDUCK_TEST_PATH: ${{ secrets.RILL_RUNTIME_MOTHERDUCK_TEST_PATH }} - RILL_RUNTIME_MOTHERDUCK_TEST_TOKEN: ${{ secrets.RILL_RUNTIME_MOTHERDUCK_TEST_TOKEN }} - RILL_RUNTIME_DATABRICKS_TEST_DSN: ${{ secrets.RILL_RUNTIME_DATABRICKS_TEST_DSN }} - RILL_TEST_GH_TOKEN: ${{ secrets.RILL_TEST_GH_TOKEN }} - RILL_ADMIN_TEST_GITHUB_APP_ID: ${{ secrets.RILL_ADMIN_TEST_GITHUB_APP_ID }} - RILL_ADMIN_TEST_GITHUB_APP_PRIVATE_KEY: ${{ secrets.RILL_ADMIN_TEST_GITHUB_APP_PRIVATE_KEY }} - RILL_ADMIN_TEST_GITHUB_MANAGED_ACCOUNT: ${{ secrets.RILL_ADMIN_TEST_GITHUB_MANAGED_ACCOUNT }} - RILL_RUNTIME_OPENAI_TEST_API_KEY: ${{ secrets.RILL_RUNTIME_OPENAI_TEST_API_KEY }} - RILL_RUNTIME_GEMINI_TEST_API_KEY: ${{ secrets.RILL_RUNTIME_GEMINI_TEST_API_KEY }} - - name: Save DuckDB extensions - uses: actions/cache/save@v5 - with: - path: ~/.duckdb/extensions - key: ${{ steps.restore-duckdb-extensions.outputs.cache-primary-key }} - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.out - flags: go - - name: Notify Slack - uses: ravsamhq/notify-slack-action@v2 - if: always() - with: - status: ${{ job.status }} - notification_title: "{workflow} has {status_message}" - message_format: "{emoji} *{workflow}*" - footer: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}>" - notify_when: failure - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TEAM_PLATFORM }} + - name: Check out code + uses: actions/checkout@v4 + + - name: Restore DuckDB extensions + id: restore-duckdb-extensions + uses: actions/cache/restore@v5 + with: + path: ~/.duckdb/extensions + key: ${{ runner.os }}-cache-duckdb-extensions + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.26.5 + cache: true + + - name: Run hermetic race tests with coverage + shell: bash + run: |- + set -euo pipefail + PACKAGES="$(go list ./admin/... ./cli/... ./runtime/... ./scripts/... \ + | grep -Ev '/proto/gen/|/node_modules/' \ + | paste -sd, -)" + go test -short -race -shuffle=on -timeout=30m \ + -covermode=atomic \ + -coverpkg="$PACKAGES" \ + -coverprofile=coverage.out \ + ./admin/... ./cli/... ./runtime/... ./scripts/... + # Each test binary emits the same coverpkg block set. Merge identical + # blocks before artifact upload so the profile stays compact. + awk 'NR == 1 { print; next } { key = $1 " " $2; count[key] += $3 } END { for (key in count) print key, count[key] }' \ + coverage.out > coverage.compact.out + mv coverage.compact.out coverage.out + go tool cover -func=coverage.out | tee coverage-summary.txt + + - name: Save DuckDB extensions + if: always() + uses: actions/cache/save@v5 + with: + path: ~/.duckdb/extensions + key: ${{ steps.restore-duckdb-extensions.outputs.cache-primary-key }} + + - name: Preserve diagnostic coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: go-coverage-${{ github.sha }} + path: | + coverage.out + coverage-summary.txt + if-no-files-found: ignore + + - name: Upload hermetic Go coverage to Codecov + if: success() + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.out + disable_search: true + flags: go-hermetic + fail_ci_if_error: true diff --git a/.github/workflows/go-live-contract.yml b/.github/workflows/go-live-contract.yml new file mode 100644 index 000000000000..fb85a41fb728 --- /dev/null +++ b/.github/workflows/go-live-contract.yml @@ -0,0 +1,56 @@ +name: Scheduled Go live-provider contracts + +on: + schedule: + - cron: "0 5 * * 1,4" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-22.04-large + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Log in to GCR + uses: docker/login-action@v3 + with: + registry: gcr.io + username: _json_key + password: ${{ secrets.RILL_BINARY_SA }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.26.5 + cache: true + + - name: Run live-provider contracts + run: go test ./admin/... ./cli/... ./runtime/... ./scripts/... -timeout 60m -v + env: + RILL_RUNTIME_TEST_MODE: expensive + RILL_RUNTIME_DRUID_TEST_DSN: ${{ secrets.RILL_RUNTIME_DRUID_TEST_DSN }} + RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON }} + RILL_RUNTIME_SNOWFLAKE_TEST_DSN: ${{ secrets.RILL_RUNTIME_SNOWFLAKE_TEST_DSN }} + RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON }} + RILL_RUNTIME_GCS_TEST_HMAC_KEY: ${{ secrets.RILL_RUNTIME_GCS_TEST_HMAC_KEY }} + RILL_RUNTIME_GCS_TEST_HMAC_SECRET: ${{ secrets.RILL_RUNTIME_GCS_TEST_HMAC_SECRET }} + RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} + RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} + RILL_RUNTIME_AZURE_TEST_CONNECTION_STRING: ${{ secrets.RILL_RUNTIME_AZURE_TEST_CONNECTION_STRING }} + RILL_RUNTIME_ATHENA_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} + RILL_RUNTIME_ATHENA_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} + RILL_RUNTIME_REDSHIFT_TEST_AWS_ACCESS_KEY_ID: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID }} + RILL_RUNTIME_REDSHIFT_TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY }} + RILL_RUNTIME_MOTHERDUCK_TEST_PATH: ${{ secrets.RILL_RUNTIME_MOTHERDUCK_TEST_PATH }} + RILL_RUNTIME_MOTHERDUCK_TEST_TOKEN: ${{ secrets.RILL_RUNTIME_MOTHERDUCK_TEST_TOKEN }} + RILL_RUNTIME_DATABRICKS_TEST_DSN: ${{ secrets.RILL_RUNTIME_DATABRICKS_TEST_DSN }} + RILL_TEST_GH_TOKEN: ${{ secrets.RILL_TEST_GH_TOKEN }} + RILL_ADMIN_TEST_GITHUB_APP_ID: ${{ secrets.RILL_ADMIN_TEST_GITHUB_APP_ID }} + RILL_ADMIN_TEST_GITHUB_APP_PRIVATE_KEY: ${{ secrets.RILL_ADMIN_TEST_GITHUB_APP_PRIVATE_KEY }} + RILL_ADMIN_TEST_GITHUB_MANAGED_ACCOUNT: ${{ secrets.RILL_ADMIN_TEST_GITHUB_MANAGED_ACCOUNT }} + RILL_RUNTIME_OPENAI_TEST_API_KEY: ${{ secrets.RILL_RUNTIME_OPENAI_TEST_API_KEY }} + RILL_RUNTIME_GEMINI_TEST_API_KEY: ${{ secrets.RILL_RUNTIME_GEMINI_TEST_API_KEY }} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000000..41e857d64d01 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,53 @@ +coverage: + status: + default_rules: + flag_coverage_not_uploaded_behavior: exclude + project: + # Each suite has a comparable baseline; do not aggregate unrelated or + # historically different reports into Codecov's implicit project status. + default: false + go-hermetic: + flags: [go-hermetic] + # Bootstrap the first hermetic baseline explicitly because main only + # has the incomparable legacy `go` report. Switch to auto after merge. + target: 34.5% + threshold: 0.1% + web-admin: + flags: [web-admin] + target: auto + threshold: 1% + web-common: + flags: [web-common] + target: auto + threshold: 1% + patch: + default: + target: 80% + threshold: 5% + +flags: + # Keep this separate from the legacy `go` flag, whose history includes live + # provider contracts and is not comparable to the hermetic PR suite. + go-hermetic: + paths: + - admin/ + - cli/ + - runtime/ + - scripts/ + carryforward: false + web-admin: + paths: + - web-admin/src/ + carryforward: false + web-common: + paths: + - web-common/src/ + carryforward: false + +ignore: + - "**/proto/gen/**" + - "admin/testadmin/**" + - "runtime/testruntime/**" + - "web-admin/src/client/gen/**" + - "web-admin/src/runtime-client/**/gen/**" + - "web-common/src/lib/i18n/gen/**" diff --git a/runtime/controller_test.go b/runtime/controller_test.go index 747e6721072e..6b109803fe41 100644 --- a/runtime/controller_test.go +++ b/runtime/controller_test.go @@ -12,6 +12,7 @@ import ( "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/parser" "github.com/rilldata/rill/runtime/testruntime" + "github.com/rilldata/rill/runtime/testruntime/testmode" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" ) @@ -1424,6 +1425,8 @@ func newExplore(metricsVew string, measures, dims []string) (*runtimev1.Explore, } func TestDedicatedConnector(t *testing.T) { + // This contract requires live S3 and GCS credentials, so hermetic short-mode coverage skips it. + testmode.Expensive(t) // Acquire the connectors for the runtime instance. vars := make(map[string]string) diff --git a/runtime/drivers/druid/druid_test.go b/runtime/drivers/druid/druid_test.go index f266e42b134a..6e202e51d73d 100644 --- a/runtime/drivers/druid/druid_test.go +++ b/runtime/drivers/druid/druid_test.go @@ -9,6 +9,7 @@ import ( "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/storage" "github.com/rilldata/rill/runtime/testruntime" + "github.com/rilldata/rill/runtime/testruntime/testmode" "github.com/stretchr/testify/require" "go.uber.org/zap" ) @@ -37,6 +38,8 @@ func TestScan(t *testing.T) { } func acquireTestDruid(t *testing.T) (drivers.Handle, drivers.OLAPStore) { + // Druid has no local fixture in this package; keep the real-cluster contract out of hermetic short runs. + testmode.Expensive(t) cfg := testruntime.AcquireConnector(t, "druid") conn, err := drivers.Open("druid", "", "default", cfg, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) diff --git a/runtime/testruntime/connectors.go b/runtime/testruntime/connectors.go index 362f95640623..d298e2d63415 100644 --- a/runtime/testruntime/connectors.go +++ b/runtime/testruntime/connectors.go @@ -67,6 +67,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // - BigQuery Read Session User // The test dataset is pre-populated with tables defined in testdata/init_data/bigquery_init_data.sql. "bigquery": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) gac := os.Getenv("RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON") require.NotEmpty(t, gac, "Bigquery RILL_RUNTIME_BIGQUERY_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON not configured") @@ -75,6 +76,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // Snowflake connector connects to a real snowflake cloud using dsn in RILL_RUNTIME_SNOWFLAKE_TEST_DSN // The test dataset is pre-populated with tables defined in testdata/init_data/snowflake_init_data.sql: "snowflake": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) dsn := os.Getenv("RILL_RUNTIME_SNOWFLAKE_TEST_DSN") require.NotEmpty(t, dsn, "RILL_RUNTIME_SNOWFLAKE_TEST_DSN not configured") @@ -83,6 +85,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // Databricks connector connects to a real Databricks SQL warehouse using dsn in RILL_RUNTIME_DATABRICKS_TEST_DSN. // The test dataset is pre-populated with tables defined in testdata/init_data/databricks_init_data.sql. "databricks": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) dsn := os.Getenv("RILL_RUNTIME_DATABRICKS_TEST_DSN") require.NotEmpty(t, dsn, "RILL_RUNTIME_DATABRICKS_TEST_DSN not configured") @@ -100,6 +103,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ }, // gcs connector uses an actual gcs bucket with data populated from testdata/init_data/azure. "gcs": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) gac := os.Getenv("RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON") require.NotEmpty(t, gac, "GCS RILL_RUNTIME_GCS_TEST_GOOGLE_APPLICATION_CREDENTIALS_JSON not configured") @@ -108,6 +112,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ } }, "gcs_s3_compat": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) hmacKey := os.Getenv("RILL_RUNTIME_GCS_TEST_HMAC_KEY") hmacSecret := os.Getenv("RILL_RUNTIME_GCS_TEST_HMAC_SECRET") @@ -121,6 +126,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ }, // S3 connector uses an actual S3 bucket with data populated from testdata/init_data/azure. "s3": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) accessKeyID := os.Getenv("RILL_RUNTIME_S3_TEST_AWS_ACCESS_KEY_ID") secretAccessKey := os.Getenv("RILL_RUNTIME_S3_TEST_AWS_SECRET_ACCESS_KEY") @@ -135,6 +141,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // The test dataset is pre-populated with table definitions in testdata/init_data/athena_init_data.sql, // and the actual data is stored on S3, which matches the data in testdata/init_data/azure/parquet_test. "athena": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) accessKeyID := os.Getenv("RILL_RUNTIME_ATHENA_TEST_AWS_ACCESS_KEY_ID") secretAccessKey := os.Getenv("RILL_RUNTIME_ATHENA_TEST_AWS_SECRET_ACCESS_KEY") @@ -148,6 +155,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // Redshift connector connects to an actual Redshift Serverless Service. // The test dataset is pre-populated with table definitions in testdata/init_data/redshift_init_data.sql, "redshift": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) accessKeyID := os.Getenv("RILL_RUNTIME_REDSHIFT_TEST_AWS_ACCESS_KEY_ID") secretAccessKey := os.Getenv("RILL_RUNTIME_REDSHIFT_TEST_AWS_SECRET_ACCESS_KEY") @@ -161,6 +169,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ // druid connects to a real Druid cluster using the connection string in RILL_RUNTIME_DRUID_TEST_DSN. // This usually uses the master.in cluster. "druid": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) dsn := os.Getenv("RILL_RUNTIME_DRUID_TEST_DSN") require.NotEmpty(t, dsn, "Druid test DSN not configured") @@ -271,6 +280,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ } }, "azure_cloud": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) apiKey := os.Getenv("RILL_RUNTIME_AZURE_TEST_CONNECTION_STRING") require.NotEmpty(t, apiKey) @@ -327,18 +337,21 @@ var Connectors = map[string]ConnectorAcquireFunc{ } }, "openai": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) apiKey := os.Getenv("RILL_RUNTIME_OPENAI_TEST_API_KEY") require.NotEmpty(t, apiKey) return map[string]string{"api_key": apiKey} }, "claude": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) apiKey := os.Getenv("RILL_RUNTIME_CLAUDE_TEST_API_KEY") require.NotEmpty(t, apiKey) return map[string]string{"api_key": apiKey} }, "gemini": func(t TestingT) map[string]string { + testmode.Expensive(t) loadDotEnv(t) apiKey := os.Getenv("RILL_RUNTIME_GEMINI_TEST_API_KEY") require.NotEmpty(t, apiKey)