From e9a584b8d09a0bf64b9cdfb57d78a5d5033f81b8 Mon Sep 17 00:00:00 2001 From: Udit Gaurav Date: Thu, 20 Aug 2026 14:56:22 +0530 Subject: [PATCH] feat: [CHAOS-12655]: Add RT (Resilience Testing) CLI support for load testing capabilities Signed-off-by: Udit Gaurav --- cmd/harness/main-harness.go | 2 + modules/rt/apitest_test.go | 113 +++ modules/rt/composite.go | 69 ++ modules/rt/composite_test.go | 140 +++ modules/rt/filters.go | 31 + modules/rt/filters_test.go | 76 ++ modules/rt/rt.go | 67 ++ modules/rt/run.go | 158 ++++ modules/rt/run_test.go | 293 ++++++ modules/rt/script.go | 102 +++ modules/rt/script_test.go | 221 +++++ modules/rt/template.go | 67 ++ modules/rt/template_test.go | 117 +++ modules/rt/usage.go | 110 +++ modules/rt/usage_test.go | 226 +++++ modules/rt/watch.go | 175 ++++ modules/rt/watch_test.go | 338 +++++++ modules/rt/window.go | 59 ++ modules/rt/window_test.go | 146 +++ pkg/spec/rt.spec.yaml | 1655 ++++++++++++++++++++++++++++++++++ 20 files changed, 4165 insertions(+) create mode 100644 modules/rt/apitest_test.go create mode 100644 modules/rt/composite.go create mode 100644 modules/rt/composite_test.go create mode 100644 modules/rt/filters.go create mode 100644 modules/rt/filters_test.go create mode 100644 modules/rt/rt.go create mode 100644 modules/rt/run.go create mode 100644 modules/rt/run_test.go create mode 100644 modules/rt/script.go create mode 100644 modules/rt/script_test.go create mode 100644 modules/rt/template.go create mode 100644 modules/rt/template_test.go create mode 100644 modules/rt/usage.go create mode 100644 modules/rt/usage_test.go create mode 100644 modules/rt/watch.go create mode 100644 modules/rt/watch_test.go create mode 100644 modules/rt/window.go create mode 100644 modules/rt/window_test.go create mode 100644 pkg/spec/rt.spec.yaml diff --git a/cmd/harness/main-harness.go b/cmd/harness/main-harness.go index 69ce414..fb3100e 100644 --- a/cmd/harness/main-harness.go +++ b/cmd/harness/main-harness.go @@ -17,6 +17,7 @@ import ( "github.com/harness/cli/modules/gitops" "github.com/harness/cli/modules/iacm" "github.com/harness/cli/modules/pipeline" + "github.com/harness/cli/modules/rt" "github.com/harness/cli/pkg/console" "github.com/harness/cli/pkg/hbase" "github.com/harness/cli/pkg/registry" @@ -49,6 +50,7 @@ func main() { pipeline.ModuleInit(reg.Module("pipeline")) // har is an external module (external_binary: harness-har) — ModuleInit is not loaded here. iacm.ModuleInit(reg.Module("iacm")) + rt.ModuleInit(reg.Module("rt")) rootcmd.MaybeCheckSpecs(reg) root := &cobra.Command{ diff --git a/modules/rt/apitest_test.go b/modules/rt/apitest_test.go new file mode 100644 index 0000000..a8f7bca --- /dev/null +++ b/modules/rt/apitest_test.go @@ -0,0 +1,113 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "testing" + + "github.com/harness/cli/pkg/auth" + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/registry" +) + +// The remaining handlers reach the API through client.New(ctx), so covering them +// means standing a server up in front of a Ctx and recording what was asked of it. + +// One inbound request, so a test can assert on the route taken, not just the result. +type call struct { + method string + path string + query url.Values + body map[string]any +} + +// Sent through as-is rather than JSON-encoded, for the one route that answers YAML. +type rawResponse string + +// An unmapped path answers 404: what the real API does, and what the best-effort +// readers here have to survive. +func apiCtx(t *testing.T, routes map[string]any) (*cmdctx.Ctx, *[]call) { + t.Helper() + + calls := &[]call{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c := call{method: r.Method, path: r.URL.Path, query: r.URL.Query()} + _ = json.NewDecoder(r.Body).Decode(&c.body) + *calls = append(*calls, c) + + resp, ok := routes[r.URL.Path] + if !ok { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + fmt.Fprintf(w, `{"message":"no such route %s"}`, r.URL.Path) + return + } + if raw, isRaw := resp.(rawResponse); isRaw { + w.Header().Set("Content-Type", "application/yaml") + fmt.Fprint(w, string(raw)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + })) + t.Cleanup(srv.Close) + + // A fresh registry per test doubles as a check that ModuleInit wires every id. + reg := registry.New() + ModuleInit(reg.Module("rt")) + + ctx := &cmdctx.Ctx{ + Context: context.Background(), + Auth: &auth.ResolvedAuth{ + APIUrl: srv.URL, + AccountID: "acct", + OrgID: "eng", + ProjectID: "payments", + PATToken: "pat.test", + AuthType: auth.AuthTypePAT, + }, + FlagValues: map[string]any{}, + Resolver: reg, + } + // Render to a file so endpoint tests do not print over the test output. + ctx.FormatFlags.OutFile = filepath.Join(t.TempDir(), "out") + return ctx, calls +} + +func api(format string, args ...any) string { + return basePath + fmt.Sprintf(format, args...) +} + +func itemsPage(items ...map[string]any) map[string]any { + page := make([]any, 0, len(items)) + for _, item := range items { + page = append(page, item) + } + return map[string]any{"items": page} +} + +// The runs-of-a-load-test response; the identity picker reads only the identities. +func runList(identities ...string) map[string]any { + items := make([]map[string]any, 0, len(identities)) + for _, id := range identities { + items = append(items, map[string]any{"identity": id}) + } + return itemsPage(items...) +} + +func findCall(calls *[]call, method, path string) (call, bool) { + for _, c := range *calls { + if c.method == method && c.path == path { + return c, true + } + } + return call{}, false +} diff --git a/modules/rt/composite.go b/modules/rt/composite.go new file mode 100644 index 0000000..8989b67 --- /dev/null +++ b/modules/rt/composite.go @@ -0,0 +1,69 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/harness/cli/pkg/cmdctx" +) + +const compositeBodyFnID = "composite_body" + +// A composite becomes a pipeline, and pipeline identifiers are stricter than load test +// ones — notably hyphens are fine in a load test identity and rejected here. +var ( + compositeIdentifierPattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_$]*$`) + compositeIdentifierFix = strings.NewReplacer("-", "_", ".", "_", " ", "_", "/", "_") +) + +const compositeIdentifierMaxLen = 128 + +// Not in the spec so the identifier can be checked here: the server's own rejection +// arrives as an HTTP 500 quoting a pipeline rule, naming no argument. +func compositeBody(ctx *cmdctx.Ctx) (any, error) { + if err := checkCompositeIdentifier(ctx.Id); err != nil { + return nil, err + } + name := cmdctx.GetString(ctx.FlagValues, "name") + if name == "" { + name = ctx.Id + } + return map[string]any{ + "identifier": ctx.Id, + "name": name, + "description": cmdctx.GetString(ctx.FlagValues, "description"), + "objective": cmdctx.GetString(ctx.FlagValues, "objective"), + "loadTest": map[string]any{ + "loadTestRef": cmdctx.GetString(ctx.FlagValues, "loadtest"), + }, + "probe": map[string]any{ + "identity": cmdctx.GetString(ctx.FlagValues, "probe"), + "infraReference": cmdctx.GetString(ctx.FlagValues, "probe-infra"), + "duration": cmdctx.GetString(ctx.FlagValues, "probe-duration"), + }, + }, nil +} + +func checkCompositeIdentifier(identifier string) error { + if identifier == "" { + return errors.New("create composite_loadtest requires an ") + } + if len(identifier) > compositeIdentifierMaxLen { + return fmt.Errorf("%q is %d characters: a pipeline identifier is capped at %d", + identifier, len(identifier), compositeIdentifierMaxLen) + } + if compositeIdentifierPattern.MatchString(identifier) { + return nil + } + const problem = "%q is not a valid pipeline identifier: it has to start with a letter and hold only letters, digits, underscores or dollar signs. A composite load test is a pipeline, so it will not take the hyphens a load test identity does" + // Only suggest a rewrite that passes: swapping separators cannot fix a leading digit. + if fixed := compositeIdentifierFix.Replace(identifier); compositeIdentifierPattern.MatchString(fixed) { + return fmt.Errorf(problem+"; try %q", identifier, fixed) + } + return fmt.Errorf(problem, identifier) +} diff --git a/modules/rt/composite_test.go b/modules/rt/composite_test.go new file mode 100644 index 0000000..a1ba204 --- /dev/null +++ b/modules/rt/composite_test.go @@ -0,0 +1,140 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "reflect" + "strings" + "testing" + + "github.com/harness/cli/pkg/cmdctx" +) + +func TestCheckCompositeIdentifier(t *testing.T) { + cases := []struct { + name string + identifier string + wantErr string // substring; empty means the identifier is accepted + }{ + {name: "plain", identifier: "checkout_load"}, + {name: "digits and dollar", identifier: "svc$2_peak"}, + {name: "single letter", identifier: "a"}, + {name: "mixed case", identifier: "CheckoutLoad"}, + + {name: "empty", identifier: "", wantErr: "requires an "}, + {name: "too long", identifier: strings.Repeat("a", 129), wantErr: "capped at 128"}, + + // The defect: load test identities take hyphens, pipeline identifiers do not. + {name: "hyphen suggests underscore", identifier: "checkout-load", wantErr: `try "checkout_load"`}, + {name: "dot suggests underscore", identifier: "checkout.load", wantErr: `try "checkout_load"`}, + {name: "space suggests underscore", identifier: "checkout load", wantErr: `try "checkout_load"`}, + {name: "slash suggests underscore", identifier: "checkout/load", wantErr: `try "checkout_load"`}, + + // A rewrite is only offered when it would actually be accepted. + {name: "leading digit", identifier: "2checkout", wantErr: "has to start with a letter"}, + {name: "leading underscore", identifier: "_checkout", wantErr: "has to start with a letter"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := checkCompositeIdentifier(tc.identifier) + + if tc.wantErr == "" { + if err != nil { + t.Fatalf("identifier %q: unexpected error: %v", tc.identifier, err) + } + return + } + if err == nil { + t.Fatalf("identifier %q: expected an error containing %q, got none", tc.identifier, tc.wantErr) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("identifier %q: error %q does not contain %q", tc.identifier, err, tc.wantErr) + } + }) + } +} + +func TestCheckCompositeIdentifierOnlySuggestsValidRewrites(t *testing.T) { + for _, identifier := range []string{"2-checkout", "-checkout", "...", "$checkout"} { + err := checkCompositeIdentifier(identifier) + if err == nil { + t.Fatalf("identifier %q: expected rejection", identifier) + } + if strings.Contains(err.Error(), "try ") { + t.Fatalf("identifier %q: offered a rewrite that would also be rejected: %v", identifier, err) + } + } +} + +// The identifier is checked before the body is built, so a bad one never reaches the server. +func TestCompositeBodyRejectsBadIdentifier(t *testing.T) { + for _, identifier := range []string{"", "checkout-load", "2checkout"} { + if _, err := compositeBody(&cmdctx.Ctx{Id: identifier, FlagValues: map[string]any{}}); err == nil { + t.Fatalf("identifier %q: expected rejection", identifier) + } + } +} + +func TestCompositeBody(t *testing.T) { + body, err := compositeBody(&cmdctx.Ctx{ + Id: "checkout_load", + FlagValues: map[string]any{ + "name": "Checkout peak", + "description": "Black Friday rehearsal", + "objective": "hold p95 under 400ms", + "loadtest": "checkout-peak", + "probe": "cart_latency", + "probe-infra": "prod_k8s", + "probe-duration": "10m", + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + want := map[string]any{ + "identifier": "checkout_load", + "name": "Checkout peak", + "description": "Black Friday rehearsal", + "objective": "hold p95 under 400ms", + "loadTest": map[string]any{"loadTestRef": "checkout-peak"}, + "probe": map[string]any{ + "identity": "cart_latency", + "infraReference": "prod_k8s", + "duration": "10m", + }, + } + if !reflect.DeepEqual(body, want) { + t.Fatalf("body mismatch:\n got %#v\nwant %#v", body, want) + } +} + +// Every key stays present when its flag is unset — the route reads an absent probe +// block as no probe at all, rather than as one waiting on a runtime input. +func TestCompositeBodyDefaultsNameToIdentifierAndKeepsEmptyKeys(t *testing.T) { + body, err := compositeBody(&cmdctx.Ctx{ + Id: "checkout_load", + FlagValues: map[string]any{"loadtest": "checkout-peak", "probe": "cart_latency"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + want := map[string]any{ + "identifier": "checkout_load", + "name": "checkout_load", + "description": "", + "objective": "", + "loadTest": map[string]any{"loadTestRef": "checkout-peak"}, + "probe": map[string]any{ + "identity": "cart_latency", + "infraReference": "", + "duration": "", + }, + } + if !reflect.DeepEqual(body, want) { + t.Fatalf("body mismatch:\n got %#v\nwant %#v", body, want) + } +} diff --git a/modules/rt/filters.go b/modules/rt/filters.go new file mode 100644 index 0000000..69cbda1 --- /dev/null +++ b/modules/rt/filters.go @@ -0,0 +1,31 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "strings" + + "github.com/harness/cli/pkg/cmdctx" +) + +const loadTestFilterParamsID = "loadtest_filters" + +// Not a query_params expression: a list flag formats as the literal "[]" when unset, which +// the route reads as a tag nothing carries and answers with an empty page. +func loadTestFilterParams(ctx *cmdctx.Ctx) (map[string]string, error) { + tags := make([]string, 0, 4) + for _, tag := range cmdctx.GetStringSlice(ctx.FlagValues, "tag") { + // Splitting here keeps --tag a,b and --tag a --tag b the same request. + for _, part := range strings.Split(tag, ",") { + if part = strings.TrimSpace(part); part != "" { + tags = append(tags, part) + } + } + } + if len(tags) == 0 { + // Absent rather than empty: an empty filter is not a filter. + return nil, nil + } + return map[string]string{"tags": strings.Join(tags, ",")}, nil +} diff --git a/modules/rt/filters_test.go b/modules/rt/filters_test.go new file mode 100644 index 0000000..28d082f --- /dev/null +++ b/modules/rt/filters_test.go @@ -0,0 +1,76 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "testing" + + "github.com/harness/cli/pkg/cmdctx" +) + +func filtersFor(t *testing.T, tags any) map[string]string { + t.Helper() + flags := map[string]any{} + if tags != nil { + flags["tag"] = tags + } + qp, err := loadTestFilterParams(&cmdctx.Ctx{FlagValues: flags}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return qp +} + +func TestLoadTestFiltersOmitsAnEmptyTagList(t *testing.T) { + for name, tags := range map[string]any{ + "flag absent": nil, + "empty slice": []string{}, + "blank value": []string{""}, + "only spaces": []string{" "}, + "empty parts": []string{",", " , "}, + } { + t.Run(name, func(t *testing.T) { + if qp := filtersFor(t, tags); len(qp) != 0 { + t.Errorf("got %v, want no tags param at all", qp) + } + }) + } +} + +func TestLoadTestFiltersJoinsWithCommas(t *testing.T) { + cases := map[string]struct { + tags []string + want string + }{ + "one": {[]string{"smoke"}, "smoke"}, + "repeated flag": {[]string{"smoke", "nightly"}, "smoke,nightly"}, + "comma in one": {[]string{"smoke,nightly"}, "smoke,nightly"}, + "mixed spellings": {[]string{"smoke,nightly", "weekly"}, "smoke,nightly,weekly"}, + "padding trimmed": {[]string{" smoke , nightly "}, "smoke,nightly"}, + "blanks dropped": {[]string{"smoke", "", "nightly"}, "smoke,nightly"}, + "order is kept": {[]string{"b", "a"}, "b,a"}, + "duplicates kept": {[]string{"smoke", "smoke"}, "smoke,smoke"}, + "inner space kept": {[]string{"needs review"}, "needs review"}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + qp := filtersFor(t, tc.tags) + if qp["tags"] != tc.want { + t.Errorf("tags = %q, want %q", qp["tags"], tc.want) + } + if len(qp) != 1 { + t.Errorf("got %v, want only the tags param", qp) + } + }) + } +} + +func TestLoadTestFiltersSpellingsAgree(t *testing.T) { + repeated := filtersFor(t, []string{"smoke", "nightly"}) + comma := filtersFor(t, []string{"smoke,nightly"}) + if repeated["tags"] != comma["tags"] { + t.Errorf("--tag smoke --tag nightly gave %q but --tag smoke,nightly gave %q", + repeated["tags"], comma["tags"]) + } +} diff --git a/modules/rt/rt.go b/modules/rt/rt.go new file mode 100644 index 0000000..33a12ab --- /dev/null +++ b/modules/rt/rt.go @@ -0,0 +1,67 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package rt implements the handlers behind the Resilience Testing commands. +// Load testing is the first feature area under RT; chaos is expected next. +// Plain HTTP calls live in rt.spec.yaml — only what the spec cannot express is here. +package rt + +import ( + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/registry" +) + +// The load test manager API root. A second RT feature area would have its own root. +const basePath = "/gateway/loadTest/manager/api/v1" + +// ModuleInit registers the RT handlers. Commands are declared in rt.spec.yaml. +func ModuleInit(reg registry.ModuleRegistrar) { + reg.RegisterBodyFn(startRunBodyFnID, startRunBody) + reg.RegisterBodyFn(encodeScriptBodyFnID, encodeScriptBody) + reg.RegisterBodyFn(compositeBodyFnID, compositeBody) + reg.RegisterFollowFn(watchFollowFnID, watchFollowFn) + reg.RegisterWorkflow(rerunWorkflowID, rerunWorkflow) + reg.RegisterWorkflow(exportTemplateYamlWorkflowID, exportTemplateYaml) + reg.RegisterTextFormatter(formatScriptID, formatScript) + reg.RegisterListTransformFn(usageReportRowsID, usageReportRows) + reg.RegisterQueryParamsFn(usageWindowParamsID, usageWindowParams) + reg.RegisterQueryParamsFn(loadTestFilterParamsID, loadTestFilterParams) + reg.RegisterFlagResolveFn(resolveScriptRevisionID, resolveScriptRevision) +} + +// accountIdentifier is added by the client on the way out; org and project are not. +func scopeParams(ctx *cmdctx.Ctx) map[string]string { + qp := map[string]string{} + if ctx.Auth == nil { + return qp + } + if ctx.Auth.OrgID != "" { + qp["organizationIdentifier"] = ctx.Auth.OrgID + } + if ctx.Auth.ProjectID != "" { + qp["projectIdentifier"] = ctx.Auth.ProjectID + } + return qp +} + +func asMap(v any) map[string]any { + m, _ := v.(map[string]any) + return m +} + +func stringField(m map[string]any, key string) string { + if m == nil { + return "" + } + s, _ := m[key].(string) + return s +} + +// JSON decoding gives every number as a float64, so this covers counts and rates alike. +func floatField(m map[string]any, key string) float64 { + if m == nil { + return 0 + } + f, _ := m[key].(float64) + return f +} diff --git a/modules/rt/run.go b/modules/rt/run.go new file mode 100644 index 0000000..116307a --- /dev/null +++ b/modules/rt/run.go @@ -0,0 +1,158 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "errors" + "fmt" + "math/rand/v2" + "net/url" + "regexp" + "sort" + + "github.com/harness/cli/pkg/client" + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/registry" + "github.com/harness/cli/pkg/spec" +) + +const ( + startRunBodyFnID = "start_run" + rerunWorkflowID = "rerun" + + // Matches the console's suffix, so runs started from either place sort together. + runSuffixLen = 3 + runSuffixAlphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + + runIdentityAttempts = 20 + runIdentityScan = 100 +) + +// The run API sometimes reports a load test as an internal UUID instead of its name. +var uuidPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// Not in the spec: the server wants a client-supplied identity, and overrides as an array. +func startRunBody(ctx *cmdctx.Ctx) (any, error) { + if ctx.Id == "" { + return nil, errors.New("execute loadtest requires a ") + } + identity, err := newRunIdentity(ctx, ctx.Id) + if err != nil { + return nil, err + } + body := map[string]any{"identity": identity} + if name := cmdctx.GetString(ctx.FlagValues, "name"); name != "" { + body["name"] = name + } + if values := setArgsToValues(ctx.SetArgs); len(values) > 0 { + body["values"] = values + } + return body, nil +} + +// Sorted so the body is stable across runs. +func setArgsToValues(set map[string]string) []map[string]string { + if len(set) == 0 { + return nil + } + names := make([]string, 0, len(set)) + for name := range set { + names = append(names, name) + } + sort.Strings(names) + values := make([]map[string]string, 0, len(names)) + for _, name := range names { + values = append(values, map[string]string{"name": name, "value": set[name]}) + } + return values +} + +// Picks a free suffix up front rather than retrying after the server rejects a duplicate. +func newRunIdentity(ctx *cmdctx.Ctx, loadTestID string) (string, error) { + taken := existingRunIdentities(ctx, loadTestID) + for range runIdentityAttempts { + candidate := loadTestID + "-" + randomSuffix() + if !taken[candidate] { + return candidate, nil + } + } + return "", fmt.Errorf("could not find an unused run identity for load test %q", loadTestID) +} + +func randomSuffix() string { + b := make([]byte, runSuffixLen) + for i := range b { + b[i] = runSuffixAlphabet[rand.IntN(len(runSuffixAlphabet))] + } + return string(b) +} + +// Best effort: on failure the caller falls back to an unchecked random suffix. +func existingRunIdentities(ctx *cmdctx.Ctx, loadTestID string) map[string]bool { + taken := map[string]bool{} + qp := scopeParams(ctx) + qp["limit"] = fmt.Sprintf("%d", runIdentityScan) + resp, _, err := client.New(ctx).Get(basePath+"/load-tests/"+url.PathEscape(loadTestID)+"/runs", qp) + if err != nil { + return taken + } + items, _ := asMap(resp)["items"].([]any) + for _, item := range items { + if id := stringField(asMap(item), "identity"); id != "" { + taken[id] = true + } + } + return taken +} + +// There is no rerun route, so read the previous run for its load test and start a new one. +func rerunWorkflow(ctx *cmdctx.Ctx) error { + if ctx.Id == "" { + return errors.New("execute loadtest_run:rerun requires a ") + } + prevID := ctx.Id + + prev, _, err := client.New(ctx).Get(basePath+"/runs/"+url.PathEscape(prevID), scopeParams(ctx)) + if err != nil { + return fmt.Errorf("reading run %q: %w", prevID, err) + } + parent := stringField(asMap(prev), "loadTestIdentity") + switch { + case parent == "": + return fmt.Errorf("run %q does not record which load test it belongs to, so it cannot be rerun", prevID) + case uuidPattern.MatchString(parent): + return fmt.Errorf("run %q reports its load test as an internal id (%s) rather than a name; start a new run with: harness execute loadtest ", prevID, parent) + } + + // startRunBody reads the load test from ctx.Id, so point it at the parent and put it back. + ctx.Id = parent + defer func() { ctx.Id = prevID }() + if ctx.FlagValues == nil { + ctx.FlagValues = map[string]any{} + } + if cmdctx.GetString(ctx.FlagValues, "name") == "" { + ctx.FlagValues["name"] = "Rerun of " + prevID + } + + ep := &spec.EndpointSpec{ + Method: "POST", + Path: basePath + "/load-tests/{{ctx.id}}/runs", + BodyFn: "rt:" + startRunBodyFnID, + ItemExpr: "it", + QueryParams: scopeQueryExprs, + } + result, err := registry.RunEndpoint(ctx, ep) + if err != nil { + return fmt.Errorf("starting a rerun of %q: %w", prevID, err) + } + if cmdctx.GetBool(ctx.FlagValues, "follow") { + return watchFollowFn(ctx, result) + } + return nil +} + +var scopeQueryExprs = map[string]string{ + "organizationIdentifier": `auth.org != "" ? auth.org : nil`, + "projectIdentifier": `auth.project != "" ? auth.project : nil`, +} diff --git a/modules/rt/run_test.go b/modules/rt/run_test.go new file mode 100644 index 0000000..ba6ec4f --- /dev/null +++ b/modules/rt/run_test.go @@ -0,0 +1,293 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "strings" + "testing" + + "github.com/harness/cli/pkg/auth" + "github.com/harness/cli/pkg/cmdctx" +) + +func TestSetArgsToValues(t *testing.T) { + got := setArgsToValues(map[string]string{ + "users": "500", + "duration": "10m", + "host": "api.example.com", + }) + want := []map[string]string{ + {"name": "duration", "value": "10m"}, + {"name": "host", "value": "api.example.com"}, + {"name": "users", "value": "500"}, + } + if len(got) != len(want) { + t.Fatalf("got %d values, want %d", len(got), len(want)) + } + for i := range want { + if got[i]["name"] != want[i]["name"] || got[i]["value"] != want[i]["value"] { + t.Errorf("value %d = %v, want %v", i, got[i], want[i]) + } + } +} + +func TestSetArgsToValuesEmpty(t *testing.T) { + if got := setArgsToValues(nil); got != nil { + t.Errorf("got %v, want nothing sent when no --set is given", got) + } + if got := setArgsToValues(map[string]string{}); got != nil { + t.Errorf("got %v, want nothing sent for an empty set", got) + } +} + +func TestRandomSuffix(t *testing.T) { + seen := map[string]bool{} + for range 200 { + s := randomSuffix() + if len(s) != runSuffixLen { + t.Fatalf("suffix %q is %d characters, want %d", s, len(s), runSuffixLen) + } + for _, r := range s { + if !strings.ContainsRune(runSuffixAlphabet, r) { + t.Fatalf("suffix %q holds %q, which is outside the console's alphabet", s, r) + } + } + seen[s] = true + } + // 36^3 possibilities, so 200 draws collapsing to a handful means the draw is broken. + if len(seen) < 100 { + t.Errorf("200 draws produced only %d distinct suffixes", len(seen)) + } +} + +func TestStartRunBodyNeedsID(t *testing.T) { + _, err := startRunBody(&cmdctx.Ctx{}) + if err == nil || !strings.Contains(err.Error(), "") { + t.Fatalf("expected the missing-id message, got %v", err) + } +} + +func TestStartRunBodyPicksAnUntakenIdentity(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/load-tests/checkout/runs"): runList("checkout-aaa", "checkout-bbb"), + }) + ctx.Id = "checkout" + + body, err := startRunBody(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + identity, _ := asMap(body)["identity"].(string) + if identity == "checkout-aaa" || identity == "checkout-bbb" { + t.Errorf("identity %q is already taken", identity) + } + if !strings.HasPrefix(identity, "checkout-") || len(identity) != len("checkout-")+runSuffixLen { + t.Errorf("identity = %q, want checkout- and %d more characters", identity, runSuffixLen) + } + + // Without the limit the scan sees a default page and calls a used suffix free. + c, ok := findCall(calls, "GET", api("/load-tests/checkout/runs")) + if !ok { + t.Fatal("the existing runs were never read") + } + if c.query.Get("limit") == "" { + t.Error("the run scan should ask for a page large enough to be worth reading") + } +} + +func TestStartRunBodySurvivesAnUnreadableRunList(t *testing.T) { + ctx, _ := apiCtx(t, nil) // every route 404s + ctx.Id = "checkout" + + body, err := startRunBody(ctx) + if err != nil { + t.Fatalf("a failed lookup should not stop a run: %v", err) + } + if identity, _ := asMap(body)["identity"].(string); !strings.HasPrefix(identity, "checkout-") { + t.Errorf("identity = %q, want one derived from the load test anyway", identity) + } +} + +func TestStartRunBodyCarriesNameAndOverrides(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{api("/load-tests/checkout/runs"): runList()}) + ctx.Id = "checkout" + ctx.FlagValues["name"] = "peak traffic" + ctx.SetArgs = map[string]string{"users": "500"} + + body, err := startRunBody(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + m := asMap(body) + if m["name"] != "peak traffic" { + t.Errorf("name = %v, want it carried through", m["name"]) + } + values, _ := m["values"].([]map[string]string) + if len(values) != 1 || values[0]["name"] != "users" || values[0]["value"] != "500" { + t.Errorf("values = %v, want the one --set override", m["values"]) + } +} + +func TestRerunNeedsARunID(t *testing.T) { + ctx, _ := apiCtx(t, nil) + if err := rerunWorkflow(ctx); err == nil || !strings.Contains(err.Error(), "") { + t.Fatalf("expected the missing-id message, got %v", err) + } +} + +func TestRerunStartsAFreshRunOfTheSameLoadTest(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{"identity": "checkout-aaa", "loadTestIdentity": "checkout"}, + api("/load-tests/checkout/runs"): map[string]any{"identity": "checkout-bbb"}, + }) + ctx.Id = "checkout-aaa" + + if err := rerunWorkflow(ctx); err != nil { + t.Fatalf("unexpected error: %v", err) + } + post, ok := findCall(calls, "POST", api("/load-tests/checkout/runs")) + if !ok { + t.Fatal("no run was started against the parent load test") + } + // Unnamed, a rerun is indistinguishable from the original in a list. + if name, _ := post.body["name"].(string); !strings.Contains(name, "checkout-aaa") { + t.Errorf("name = %q, want the run it reruns named", name) + } + // Callers reuse ctx after a workflow returns. + if ctx.Id != "checkout-aaa" { + t.Errorf("ctx.Id = %q, want the previous run restored", ctx.Id) + } +} + +func TestRerunKeepsAnExplicitName(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{"loadTestIdentity": "checkout"}, + api("/load-tests/checkout/runs"): map[string]any{"identity": "checkout-bbb"}, + }) + ctx.Id = "checkout-aaa" + ctx.FlagValues["name"] = "friday peak" + + if err := rerunWorkflow(ctx); err != nil { + t.Fatalf("unexpected error: %v", err) + } + post, _ := findCall(calls, "POST", api("/load-tests/checkout/runs")) + if post.body["name"] != "friday peak" { + t.Errorf("name = %v, want --name to win over the default", post.body["name"]) + } +} + +func TestRerunRefusesAnInternalParentID(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{ + "loadTestIdentity": "1b4e28ba-2fa1-11d2-883f-0016d3cca427", + }, + }) + ctx.Id = "checkout-aaa" + + err := rerunWorkflow(ctx) + if err == nil { + t.Fatal("expected an internal parent id to be refused") + } + if !strings.Contains(err.Error(), "harness execute loadtest") { + t.Errorf("error %q should name the command that does work", err) + } + if len(*calls) != 1 { + t.Errorf("made %d requests, want the rerun refused before starting anything", len(*calls)) + } +} + +func TestRerunRefusesARunWithNoParent(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{"identity": "checkout-aaa"}, + }) + ctx.Id = "checkout-aaa" + + err := rerunWorkflow(ctx) + if err == nil || !strings.Contains(err.Error(), "cannot be rerun") { + t.Fatalf("expected a run with no recorded parent to be refused, got %v", err) + } +} + +func TestRerunReportsAnUnreadablePreviousRun(t *testing.T) { + ctx, _ := apiCtx(t, nil) + ctx.Id = "checkout-aaa" + + err := rerunWorkflow(ctx) + if err == nil || !strings.Contains(err.Error(), "checkout-aaa") { + t.Fatalf("expected the unreadable run named in the error, got %v", err) + } +} + +func TestUUIDPattern(t *testing.T) { + for _, id := range []string{ + "1b4e28ba-2fa1-11d2-883f-0016d3cca427", + "1B4E28BA-2FA1-11D2-883F-0016D3CCA427", + } { + if !uuidPattern.MatchString(id) { + t.Errorf("%q should be recognised as an internal id", id) + } + } + for _, id := range []string{ + "checkout-load", + "checkout", + "1b4e28ba-2fa1-11d2-883f", + "1b4e28ba2fa111d2883f0016d3cca427", + "", + } { + if uuidPattern.MatchString(id) { + t.Errorf("%q is a name, not an internal id", id) + } + } +} + +func TestScopeParams(t *testing.T) { + got := scopeParams(&cmdctx.Ctx{Auth: &auth.ResolvedAuth{ + AccountID: "acct", OrgID: "eng", ProjectID: "payments", + }}) + if got["organizationIdentifier"] != "eng" || got["projectIdentifier"] != "payments" { + t.Errorf("got %v, want the org and project carried through", got) + } + if _, present := got["accountIdentifier"]; present { + t.Error("accountIdentifier is the client's to add") + } + + got = scopeParams(&cmdctx.Ctx{Auth: &auth.ResolvedAuth{AccountID: "acct", OrgID: "eng"}}) + if got["organizationIdentifier"] != "eng" { + t.Errorf("got %v, want the org", got) + } + if _, present := got["projectIdentifier"]; present { + t.Error("an account-level scope should not send an empty project") + } + + if got := scopeParams(&cmdctx.Ctx{}); len(got) != 0 { + t.Errorf("got %v, want nothing when there is no auth", got) + } +} + +func TestResponseFieldHelpers(t *testing.T) { + m := map[string]any{"name": "checkout", "rps": 12.5, "count": float64(7)} + if stringField(m, "name") != "checkout" { + t.Error("stringField should read a string") + } + if floatField(m, "rps") != 12.5 || floatField(m, "count") != 7 { + t.Error("floatField should read both rates and counts") + } + + // Wrong type, absent field and nil object all mean "not reported", not a panic. + if stringField(m, "rps") != "" || stringField(m, "absent") != "" || stringField(nil, "name") != "" { + t.Error("stringField should be empty for anything it cannot read") + } + if floatField(m, "name") != 0 || floatField(m, "absent") != 0 || floatField(nil, "rps") != 0 { + t.Error("floatField should be zero for anything it cannot read") + } + + if asMap(map[string]any{"a": 1})["a"] != 1 { + t.Error("asMap should pass an object through") + } + for _, v := range []any{nil, "text", 42, []any{1}} { + if asMap(v) != nil { + t.Errorf("asMap(%v) should be nil for a non-object", v) + } + } +} diff --git a/modules/rt/script.go b/modules/rt/script.go new file mode 100644 index 0000000..3b85af3 --- /dev/null +++ b/modules/rt/script.go @@ -0,0 +1,102 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "net/url" + "os" + "strconv" + "strings" + + "github.com/harness/cli/pkg/client" + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/console" +) + +const ( + encodeScriptBodyFnID = "encode_script" + formatScriptID = "format_script" + resolveScriptRevisionID = "resolve_script_revision" + + revisionScan = 100 +) + +// The API stores scripts base64-encoded, which the framework's -f handling cannot produce. +func encodeScriptBody(ctx *cmdctx.Ctx) (any, error) { + content, err := cmdctx.SlurpInputFile(ctx.FlagValues) + if err != nil { + return nil, err + } + if strings.TrimSpace(content) == "" { + return nil, fmt.Errorf("%s is empty, so there is nothing to upload", cmdctx.GetString(ctx.FlagValues, "file")) + } + body := map[string]any{ + "scriptContent": base64.StdEncoding.EncodeToString([]byte(content)), + } + if description := cmdctx.GetString(ctx.FlagValues, "description"); description != "" { + body["description"] = description + } + return body, nil +} + +// Writes the script back exactly as uploaded, so "get loadtest_script -o x.jmx" is a download. +func formatScript(w io.Writer, d cmdctx.DataAccessor) error { + encoded := d.GetString("it.scriptContent") + if encoded == "" { + return errors.New("this load test has no stored script: tests that run from a container image carry their script inside the image") + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return fmt.Errorf("decoding the stored script: %w", err) + } + // Bundles are zip archives; painting one onto a terminal is never what was wanted. + if d.GetBool("it.isBundle") && w == os.Stdout && console.IsStdoutTTY() { + return fmt.Errorf("this revision is %s: save it with -o bundle.zip, or redirect stdout", bundleKind(d)) + } + _, err = w.Write(decoded) + return err +} + +func bundleKind(d cmdctx.DataAccessor) string { + if main := d.GetString("it.bundleMainFile"); main != "" { + return "a zip workspace, with " + main + " as its main plan" + } + return "a zip workspace" +} + +// The console numbers revisions 1, 2, 3, but the route keys on the identity — look a bare number up. +func resolveScriptRevision(ctx *cmdctx.Ctx, raw string) (string, error) { + number, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + // Not a number, so it is already an identifier. + return raw, nil + } + if ctx.Id == "" { + return "", errors.New("a load test id is needed to look up a revision by number") + } + + qp := scopeParams(ctx) + qp["limit"] = strconv.Itoa(revisionScan) + resp, _, err := client.New(ctx).Get(basePath+"/load-tests/"+url.PathEscape(ctx.Id)+"/script/revisions", qp) + if err != nil { + return "", fmt.Errorf("reading the script revisions of load test %q: %w", ctx.Id, err) + } + + items, _ := asMap(resp)["items"].([]any) + for _, item := range items { + m := asMap(item) + if int(floatField(m, "revisionNumber")) != number { + continue + } + if identity := stringField(m, "identity"); identity != "" { + return identity, nil + } + } + return "", fmt.Errorf("load test %q has no script revision %d among its most recent %d; list them with: harness list loadtest_script:revisions %s", + ctx.Id, number, revisionScan, ctx.Id) +} diff --git a/modules/rt/script_test.go b/modules/rt/script_test.go new file mode 100644 index 0000000..773b905 --- /dev/null +++ b/modules/rt/script_test.go @@ -0,0 +1,221 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "bytes" + "encoding/base64" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/harness/cli/pkg/cmdctx" +) + +// A DataAccessor over a flat map: the formatter reads two paths and nothing nested. +type fakeData map[string]any + +func (f fakeData) GetString(path string) string { s, _ := f[path].(string); return s } +func (f fakeData) GetBool(path string) bool { b, _ := f[path].(bool); return b } +func (f fakeData) GetInt64(path string) int64 { i, _ := f[path].(int64); return i } +func (f fakeData) GetTs(path string) string { return f.GetString(path) } +func (f fakeData) GetData() any { return map[string]any(f) } +func (f fakeData) GetSlice(path string) []any { s, _ := f[path].([]any); return s } + +func TestFormatScript(t *testing.T) { + const script = "openapi: 3.0.0\nplan: checkout\n" + var buf bytes.Buffer + err := formatScript(&buf, fakeData{"it.scriptContent": base64.StdEncoding.EncodeToString([]byte(script))}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if buf.String() != script { + t.Fatalf("got %q, want the script back unchanged", buf.String()) + } +} + +func TestFormatScriptEmpty(t *testing.T) { + err := formatScript(&bytes.Buffer{}, fakeData{}) + if err == nil || !strings.Contains(err.Error(), "container image") { + t.Fatalf("expected the container-image explanation, got %v", err) + } +} + +func TestFormatScriptBadBase64(t *testing.T) { + err := formatScript(&bytes.Buffer{}, fakeData{"it.scriptContent": "not base64!!"}) + if err == nil || !strings.Contains(err.Error(), "decoding") { + t.Fatalf("expected a decode error, got %v", err) + } +} + +func TestFormatScriptBundleToFile(t *testing.T) { + const archive = "PK\x03\x04zip bytes" + var buf bytes.Buffer + d := fakeData{ + "it.scriptContent": base64.StdEncoding.EncodeToString([]byte(archive)), + "it.isBundle": true, + "it.bundleMainFile": "checkout.jmx", + } + if err := formatScript(&buf, d); err != nil { + t.Fatalf("a bundle written to a file is fine, got %v", err) + } + if buf.String() != archive { + t.Fatalf("got %q, want the archive intact", buf.String()) + } +} + +func TestBundleKind(t *testing.T) { + got := bundleKind(fakeData{"it.bundleMainFile": "checkout.jmx"}) + if !strings.Contains(got, "checkout.jmx") { + t.Errorf("got %q, want the main plan named", got) + } + if got := bundleKind(fakeData{}); got != "a zip workspace" { + t.Errorf("got %q, want the bare description when no main plan is recorded", got) + } +} + +func TestEncodeScriptBody(t *testing.T) { + const script = "plan: checkout\n" + path := filepath.Join(t.TempDir(), "checkout.jmx") + if err := os.WriteFile(path, []byte(script), 0o600); err != nil { + t.Fatal(err) + } + + body, err := encodeScriptBody(&cmdctx.Ctx{FlagValues: map[string]any{"file": path}}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + m, ok := body.(map[string]any) + if !ok { + t.Fatalf("body is %T, want a map", body) + } + if m["scriptContent"] != base64.StdEncoding.EncodeToString([]byte(script)) { + t.Errorf("scriptContent = %v, want the encoded file", m["scriptContent"]) + } + if _, present := m["description"]; present { + t.Error("an absent --description should be left out, not sent empty") + } +} + +func TestEncodeScriptBodyWithDescription(t *testing.T) { + path := filepath.Join(t.TempDir(), "checkout.jmx") + if err := os.WriteFile(path, []byte("plan: checkout\n"), 0o600); err != nil { + t.Fatal(err) + } + body, err := encodeScriptBody(&cmdctx.Ctx{FlagValues: map[string]any{ + "file": path, "description": "peak traffic", + }}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if m, _ := body.(map[string]any); m["description"] != "peak traffic" { + t.Errorf("description = %v, want it carried through", m["description"]) + } +} + +func TestEncodeScriptBodyEmptyFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "empty.jmx") + if err := os.WriteFile(path, []byte(" \n\t\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := encodeScriptBody(&cmdctx.Ctx{FlagValues: map[string]any{"file": path}}) + if err == nil || !strings.Contains(err.Error(), "nothing to upload") { + t.Fatalf("expected a refusal to upload an empty script, got %v", err) + } + if !strings.Contains(err.Error(), path) { + t.Errorf("expected the path in %q", err) + } +} + +func TestEncodeScriptBodyMissingFlag(t *testing.T) { + if _, err := encodeScriptBody(&cmdctx.Ctx{FlagValues: map[string]any{}}); err == nil { + t.Fatal("expected an error when -f is absent") + } +} + +func TestResolveScriptRevisionByNumber(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/load-tests/checkout/script/revisions"): itemsPage( + map[string]any{"revisionNumber": float64(1), "identity": "rev-one"}, + map[string]any{"revisionNumber": float64(2), "identity": "rev-two"}, + ), + }) + ctx.Id = "checkout" + + got, err := resolveScriptRevision(ctx, "2") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "rev-two" { + t.Errorf("got %q, want the identity of revision 2", got) + } + if _, ok := findCall(calls, "GET", api("/load-tests/checkout/script/revisions")); !ok { + t.Error("the revisions of the load test were never read") + } +} + +func TestResolveScriptRevisionTrimsTheNumber(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{ + api("/load-tests/checkout/script/revisions"): itemsPage( + map[string]any{"revisionNumber": float64(2), "identity": "rev-two"}, + ), + }) + ctx.Id = "checkout" + + if got, err := resolveScriptRevision(ctx, " 2 "); err != nil || got != "rev-two" { + t.Errorf("got %q, %v; want rev-two", got, err) + } +} + +func TestResolveScriptRevisionPassesIdentifiersThrough(t *testing.T) { + ctx, calls := apiCtx(t, nil) + ctx.Id = "checkout" + + for _, raw := range []string{"rev-two", "abc123", "2.0", "v2"} { + got, err := resolveScriptRevision(ctx, raw) + if err != nil || got != raw { + t.Errorf("resolveScriptRevision(%q) = %q, %v; want it passed through", raw, got, err) + } + } + if len(*calls) != 0 { + t.Errorf("made %d requests, want an identifier resolved without asking the API", len(*calls)) + } +} + +func TestResolveScriptRevisionNeedsALoadTest(t *testing.T) { + ctx, _ := apiCtx(t, nil) + + _, err := resolveScriptRevision(ctx, "2") + if err == nil || !strings.Contains(err.Error(), "load test id") { + t.Fatalf("expected the missing load test explained, got %v", err) + } +} + +func TestResolveScriptRevisionUnknownNumber(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{ + api("/load-tests/checkout/script/revisions"): itemsPage( + map[string]any{"revisionNumber": float64(1), "identity": "rev-one"}, + ), + }) + ctx.Id = "checkout" + + _, err := resolveScriptRevision(ctx, "7") + if err == nil { + t.Fatal("expected an unknown revision number to be refused") + } + if !strings.Contains(err.Error(), "list loadtest_script:revisions") { + t.Errorf("error %q should point at the command that lists them", err) + } +} + +func TestResolveScriptRevisionReportsAnUnreadableList(t *testing.T) { + ctx, _ := apiCtx(t, nil) + ctx.Id = "checkout" + + _, err := resolveScriptRevision(ctx, "2") + if err == nil || !strings.Contains(err.Error(), "checkout") { + t.Fatalf("expected the load test named in the error, got %v", err) + } +} diff --git a/modules/rt/template.go b/modules/rt/template.go new file mode 100644 index 0000000..8448e84 --- /dev/null +++ b/modules/rt/template.go @@ -0,0 +1,67 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/url" + + "github.com/harness/cli/pkg/client" + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/format" +) + +const exportTemplateYamlWorkflowID = "export_template_yaml" + +// A workflow, not an endpoint: the route answers with YAML, which the shared client's +// unconditional json.Unmarshal cannot decode, so this goes through client.DoRaw. +func exportTemplateYaml(ctx *cmdctx.Ctx) error { + if ctx.Id == "" { + return errors.New("get loadtest_template:yaml requires a ") + } + + qp := scopeParams(ctx) + if revision := cmdctx.GetString(ctx.FlagValues, "revision"); revision != "" { + qp["revision"] = revision + } + if hub := cmdctx.GetString(ctx.FlagValues, "hub"); hub != "" { + qp["hubIdentity"] = hub + } + + resp, err := client.New(ctx).DoRaw(client.Request{ + Method: "GET", + Path: basePath + "/load-test-templates/" + url.PathEscape(ctx.Id) + "/yaml", + QueryParams: qp, + }) + if err != nil { + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading the exported template: %w", err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("API error %d: %s", resp.StatusCode, client.APIErrorMessage(resp.StatusCode, body)) + } + if len(bytes.TrimSpace(body)) == 0 { + return fmt.Errorf("template %q exported as an empty document", ctx.Id) + } + + w, closeW, err := format.OpenWriter(ctx.FormatFlags.OutFile) + if err != nil { + return err + } + defer closeW() + + if !bytes.HasSuffix(body, []byte("\n")) { + body = append(body, '\n') + } + _, err = w.Write(body) + return err +} diff --git a/modules/rt/template_test.go b/modules/rt/template_test.go new file mode 100644 index 0000000..32c9d46 --- /dev/null +++ b/modules/rt/template_test.go @@ -0,0 +1,117 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// Returns what was written, since the point of the command is the document on disk. +func exportedYaml(t *testing.T, id string, routes map[string]any) (string, *[]call) { + t.Helper() + ctx, calls := apiCtx(t, routes) + ctx.Id = id + ctx.FormatFlags.OutFile = filepath.Join(t.TempDir(), "template.yaml") + + if err := exportTemplateYaml(ctx); err != nil { + t.Fatalf("unexpected error: %v", err) + } + written, err := os.ReadFile(ctx.FormatFlags.OutFile) + if err != nil { + t.Fatalf("nothing was written: %v", err) + } + return string(written), calls +} + +func TestExportTemplateYaml(t *testing.T) { + const doc = "apiVersion: v1\nkind: LoadTestTemplate\nname: checkout\n" + got, _ := exportedYaml(t, "checkout", map[string]any{ + api("/load-test-templates/checkout/yaml"): rawResponse(doc), + }) + if got != doc { + t.Errorf("got %q, want the document unchanged", got) + } +} + +func TestExportTemplateYamlEndsWithANewline(t *testing.T) { + got, _ := exportedYaml(t, "checkout", map[string]any{ + api("/load-test-templates/checkout/yaml"): rawResponse("name: checkout"), + }) + if got != "name: checkout\n" { + t.Errorf("got %q, want a trailing newline added", got) + } +} + +func TestExportTemplateYamlPassesRevisionAndHub(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/load-test-templates/checkout/yaml"): rawResponse("name: checkout\n"), + }) + ctx.Id = "checkout" + ctx.FormatFlags.OutFile = filepath.Join(t.TempDir(), "template.yaml") + ctx.FlagValues["revision"] = "3" + ctx.FlagValues["hub"] = "chaoshub198x" + + if err := exportTemplateYaml(ctx); err != nil { + t.Fatalf("unexpected error: %v", err) + } + c, ok := findCall(calls, "GET", api("/load-test-templates/checkout/yaml")) + if !ok { + t.Fatal("the template was never read") + } + if c.query.Get("revision") != "3" { + t.Errorf("revision = %q, want --revision carried through", c.query.Get("revision")) + } + if c.query.Get("hubIdentity") != "chaoshub198x" { + t.Errorf("hubIdentity = %q, want --hub carried through", c.query.Get("hubIdentity")) + } + // An absent flag must be omitted, not sent empty: "" is a hub name to the route. + if _, present := c.query["scope"]; present { + t.Error("only the flags that were given belong in the request") + } +} + +func TestExportTemplateYamlOmitsAbsentFlags(t *testing.T) { + _, calls := exportedYaml(t, "checkout", map[string]any{ + api("/load-test-templates/checkout/yaml"): rawResponse("name: checkout\n"), + }) + c, _ := findCall(calls, "GET", api("/load-test-templates/checkout/yaml")) + for _, key := range []string{"revision", "hubIdentity"} { + if _, present := c.query[key]; present { + t.Errorf("%s was sent empty; an absent flag should be left out", key) + } + } +} + +func TestExportTemplateYamlNeedsAnID(t *testing.T) { + ctx, _ := apiCtx(t, nil) + err := exportTemplateYaml(ctx) + if err == nil || !strings.Contains(err.Error(), "") { + t.Fatalf("expected the missing-id message, got %v", err) + } +} + +func TestExportTemplateYamlRefusesAnEmptyDocument(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{ + api("/load-test-templates/checkout/yaml"): rawResponse(" \n\t\n"), + }) + ctx.Id = "checkout" + + err := exportTemplateYaml(ctx) + if err == nil || !strings.Contains(err.Error(), "empty document") { + t.Fatalf("expected an empty export to be refused, got %v", err) + } +} + +func TestExportTemplateYamlReportsAnAPIError(t *testing.T) { + ctx, _ := apiCtx(t, nil) // every route 404s + ctx.Id = "checkout" + + err := exportTemplateYaml(ctx) + if err == nil || !strings.Contains(err.Error(), "404") { + t.Fatalf("expected the status reported, got %v", err) + } +} diff --git a/modules/rt/usage.go b/modules/rt/usage.go new file mode 100644 index 0000000..991fd13 --- /dev/null +++ b/modules/rt/usage.go @@ -0,0 +1,110 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "fmt" + "strings" + "unicode" + + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/spec" +) + +const usageReportRowsID = "usage_report_rows" + +// The route answers with a bare matrix, not objects; reshaping into rows is what buys --format. +// The trailing TOTAL row is kept — it is the account figure the report exists to give. +func usageReportRows(_ *cmdctx.Ctx, data any) ([]any, []spec.FieldDef, cmdctx.PageMeta, error) { + matrix, ok := data.([]any) + if !ok { + return nil, nil, cmdctx.PageMeta{}, fmt.Errorf("the usage report came back as %T, not the expected matrix of rows", data) + } + // The service always writes a header, so no rows means an empty report, not an unreadable shape. + if len(matrix) == 0 { + return nil, nil, cmdctx.PageMeta{}, nil + } + + ids, fields := usageReportFields(matrix[0]) + rows := make([]any, 0, len(matrix)-1) + for _, raw := range matrix[1:] { + cells, _ := raw.([]any) + row := make(map[string]any, len(ids)) + for i, id := range ids { + // A short row leaves trailing columns unset: "not reported" rather than zero. + if i < len(cells) { + row[id] = cells[i] + } + } + rows = append(rows, row) + } + return rows, fields, cmdctx.PageMeta{}, nil +} + +// Keeps the service's own heading as the label so exported CSV matches the API, +// while --columns matches on the snake_case id used everywhere else in the CLI. +func usageReportFields(header any) ([]string, []spec.FieldDef) { + cells, _ := header.([]any) + ids := make([]string, 0, len(cells)) + fields := make([]spec.FieldDef, 0, len(cells)) + used := make(map[string]bool, len(cells)) + + for i, cell := range cells { + label, _ := cell.(string) + id := snakeCase(label) + if id == "" { + id = fmt.Sprintf("col_%d", i) + } + if used[id] { + base := id + for suffix := 2; ; suffix++ { + id = fmt.Sprintf("%s_%d", base, suffix) + if !used[id] { + break + } + } + } + used[id] = true + ids = append(ids, id) + fields = append(fields, spec.FieldDef{ + ID: id, + Label: label, + Expr: fmt.Sprintf("it[%q]", id), + }) + } + return ids, fields +} + +// ServiceID becomes service_id, VUSeconds becomes vu_seconds: runs of capitals stay one word. +func snakeCase(s string) string { + var b strings.Builder + runes := []rune(s) + for i, r := range runes { + switch { + case r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r): + case b.Len() > 0 && !strings.HasSuffix(b.String(), "_"): + b.WriteByte('_') + continue + default: + continue + } + // A capital opens a word after a lowercase or digit, or when it ends a run before one. + if unicode.IsUpper(r) && i > 0 && b.Len() > 0 && !strings.HasSuffix(b.String(), "_") { + previous := runes[i-1] + startsWord := unicode.IsLower(previous) || unicode.IsDigit(previous) + endsAcronym := unicode.IsUpper(previous) && i+1 < len(runes) && + unicode.IsLower(runes[i+1]) && !pluralSuffix(runes, i+1) + if startsWord || endsAcronym { + b.WriteByte('_') + } + } + b.WriteRune(unicode.ToLower(r)) + } + return strings.Trim(b.String(), "_") +} + +// A lone "s" closing a capital run, as in VUs — without this it splits into v_us. +func pluralSuffix(runes []rune, i int) bool { + return runes[i] == 's' && (i+1 == len(runes) || !unicode.IsLower(runes[i+1])) +} diff --git a/modules/rt/usage_test.go b/modules/rt/usage_test.go new file mode 100644 index 0000000..fe86694 --- /dev/null +++ b/modules/rt/usage_test.go @@ -0,0 +1,226 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/exprenv" + "github.com/harness/cli/pkg/format" + "github.com/harness/cli/pkg/registry" + "github.com/harness/cli/pkg/spec" +) + +// reportHeader is the header the service writes, as it arrives after JSON decoding. +var reportHeader = []any{"ServiceID", "Org", "Project", "RunCount", "PeakUsers", "MaxWorkers", "DurationSec", "VUSeconds", "Complexity"} + +func TestUsageReportRows(t *testing.T) { + data := []any{ + reportHeader, + []any{"checkout", "eng", "payments", "12", "500", "4", "3600", "1800000", "42.50"}, + []any{"search", "eng", "discovery", "3", "100", "1", "900", "90000", "7.25"}, + []any{"TOTAL", "", "", "", "", "", "", "", "50"}, + } + + items, fields, meta, err := usageReportRows(nil, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if meta.Notice != "" { + t.Errorf("expected no paging notice, got %q", meta.Notice) + } + + // The header is columns, not a row; every other row is an item, TOTAL included. + if len(items) != 3 { + t.Fatalf("expected 3 rows, got %d", len(items)) + } + if len(fields) != len(reportHeader) { + t.Fatalf("expected %d fields, got %d", len(reportHeader), len(fields)) + } + + // --columns matches the id, while the printed header keeps the service's wording. + wantIDs := []string{"service_id", "org", "project", "run_count", "peak_users", "max_workers", "duration_sec", "vu_seconds", "complexity"} + for i, want := range wantIDs { + if fields[i].ID != want { + t.Errorf("field %d: id = %q, want %q", i, fields[i].ID, want) + } + if label, _ := reportHeader[i].(string); fields[i].Label != label { + t.Errorf("field %d: label = %q, want the service's %q", i, fields[i].Label, label) + } + if want := `it["` + wantIDs[i] + `"]`; fields[i].Expr != want { + t.Errorf("field %d: expr = %q, want %q", i, fields[i].Expr, want) + } + } + + first, ok := items[0].(map[string]any) + if !ok { + t.Fatalf("row 0 is %T, want map[string]any", items[0]) + } + if first["service_id"] != "checkout" || first["complexity"] != "42.50" { + t.Errorf("row 0 = %v, want checkout/42.50", first) + } + + // The account total is the figure the report exists to report. + last, _ := items[2].(map[string]any) + if last["service_id"] != "TOTAL" || last["complexity"] != "50" { + t.Errorf("last row = %v, want the TOTAL row", last) + } +} + +func TestUsageReportRendersThroughTheListPipeline(t *testing.T) { + data := []any{ + reportHeader, + []any{"checkout", "eng", "payments", "12", "500", "4", "3600", "1800000", "42.50"}, + []any{"TOTAL", "", "", "", "", "", "", "", "50"}, + } + items, fields, meta, err := usageReportRows(nil, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + for _, formatName := range []string{"csv", "table", "tsv"} { + t.Run(formatName, func(t *testing.T) { + out := filepath.Join(t.TempDir(), "report") + err := format.FormatArrayOutput( + cmdctx.FormatFlags{Format: formatName, OutFile: out}, + false, items, "it", + &spec.TableSpec{Columns: registry.FieldsToTableColumns(fields)}, + fields, exprenv.Make(&cmdctx.Ctx{}), &meta, + ) + if err != nil { + t.Fatalf("rendering as %s: %v", formatName, err) + } + rendered, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + got := string(rendered) + + // The header keeps the service's own wording rather than the ids. + if !strings.Contains(got, "ServiceID") || !strings.Contains(got, "VUSeconds") { + t.Errorf("as %s, got:\n%s\nwant the service's headings", formatName, got) + } + // Every cell has to survive the round trip through the expressions. + for _, want := range []string{"checkout", "payments", "1800000", "42.50", "TOTAL"} { + if !strings.Contains(got, want) { + t.Errorf("as %s, got:\n%s\nwant it to contain %q", formatName, got, want) + } + } + }) + } +} + +func TestUsageReportRowsEmpty(t *testing.T) { + items, fields, _, err := usageReportRows(nil, []any{}) + if err != nil { + t.Fatalf("an empty report is not an error, got: %v", err) + } + if len(items) != 0 || len(fields) != 0 { + t.Fatalf("expected nothing to render, got %d items and %d fields", len(items), len(fields)) + } +} + +func TestUsageReportRowsHeaderOnly(t *testing.T) { + items, fields, _, err := usageReportRows(nil, []any{reportHeader}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(items) != 0 { + t.Errorf("expected no rows, got %d", len(items)) + } + if len(fields) != len(reportHeader) { + t.Errorf("expected the columns to survive, got %d", len(fields)) + } +} + +func TestUsageReportRowsNotAMatrix(t *testing.T) { + for _, data := range []any{map[string]any{"rows": 1}, "text", nil} { + if _, _, _, err := usageReportRows(nil, data); err == nil { + t.Errorf("data %v (%T): expected an error", data, data) + } + } +} + +func TestUsageReportRowsShortRow(t *testing.T) { + data := []any{ + []any{"ServiceID", "Org", "Complexity"}, + []any{"checkout", "eng"}, + } + items, _, _, err := usageReportRows(nil, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + row, _ := items[0].(map[string]any) + if _, present := row["complexity"]; present { + t.Errorf("missing cell should be absent, got %v", row["complexity"]) + } + if row["org"] != "eng" { + t.Errorf("org = %v, want eng", row["org"]) + } +} + +func TestUsageReportRowsDuplicateHeaders(t *testing.T) { + data := []any{ + []any{"Org", "Org", "Org"}, + []any{"a", "b", "c"}, + } + items, fields, _, err := usageReportRows(nil, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := []string{"org", "org_2", "org_3"} + for i, id := range want { + if fields[i].ID != id { + t.Errorf("field %d: id = %q, want %q", i, fields[i].ID, id) + } + } + row, _ := items[0].(map[string]any) + if row["org"] != "a" || row["org_2"] != "b" || row["org_3"] != "c" { + t.Errorf("row = %v, want all three columns kept", row) + } +} + +func TestUsageReportRowsUnnamedColumn(t *testing.T) { + data := []any{[]any{"Org", "", nil}} + _, fields, _, err := usageReportRows(nil, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for i, want := range []string{"org", "col_1", "col_2"} { + if fields[i].ID != want { + t.Errorf("field %d: id = %q, want %q", i, fields[i].ID, want) + } + } +} + +func TestSnakeCase(t *testing.T) { + cases := map[string]string{ + "ServiceID": "service_id", + "VUSeconds": "vu_seconds", + "RunCount": "run_count", + "MaxWorkers": "max_workers", + "DurationSec": "duration_sec", + "Org": "org", + "Complexity": "complexity", + "P95Latency": "p95_latency", + "already_ok": "already_ok", + "peak-users": "peak_users", + "ID": "id", + "": "", + + // A capital run closed by a plural s is one word, not two. + "Total VUs": "total_vus", + "VUsCount": "vus_count", + "APIs": "apis", + } + for in, want := range cases { + if got := snakeCase(in); got != want { + t.Errorf("snakeCase(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/modules/rt/watch.go b/modules/rt/watch.go new file mode 100644 index 0000000..54bc621 --- /dev/null +++ b/modules/rt/watch.go @@ -0,0 +1,175 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "strings" + "time" + + "github.com/harness/cli/pkg/client" + "github.com/harness/cli/pkg/cmdctx" +) + +const watchFollowFnID = "watch" + +// Runs report metrics about this often; polling faster mostly repeats a line. +const defaultPollInterval = 10 * time.Second + +const ( + runStatusStopped = "Stopped" + runStatusFinished = "Finished" + runStatusFailed = "Failed" +) + +func isTerminalStatus(status string) bool { + return status == runStatusStopped || status == runStatusFinished || status == runStatusFailed +} + +// Polls until the run is terminal, exiting non-zero on failure so a gating pipeline step fails too. +// The timeline goes to stderr, so "--follow --format json > run.json" still yields one document. +func watchFollowFn(ctx *cmdctx.Ctx, result any) error { + identity := runToWatch(ctx, result) + if identity == "" { + return errors.New("--follow: the response did not name a run to watch") + } + interval, err := pollInterval(ctx) + if err != nil { + return err + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + var previous, status string + for { + run, err := readRun(ctx, identity) + if err != nil { + // A deadline can land mid-request; without this it surfaces as a raw transport error. + if stopped := stoppedWatching(ctx, identity, status); stopped != nil { + return stopped + } + return err + } + status = stringField(run, "status") + + if line := progressLine(run); line != previous { + fmt.Fprintln(os.Stderr, line) + previous = line + } + if isTerminalStatus(status) { + return terminalError(run, identity) + } + + select { + case <-ctx.Context.Done(): + return stoppedWatching(ctx, identity, status) + case <-ticker.C: + } + } +} + +// Starting or reading a run names it in the response; stopping one answers with a bare ack. +func runToWatch(ctx *cmdctx.Ctx, result any) string { + if id := stringField(asMap(result), "identity"); id != "" { + return id + } + return ctx.Id +} + +func pollInterval(ctx *cmdctx.Ctx) (time.Duration, error) { + raw := cmdctx.GetString(ctx.FlagValues, "interval") + if raw == "" { + return defaultPollInterval, nil + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("--interval %q is not a duration such as 5s or 2m", raw) + } + if d <= 0 { + return 0, fmt.Errorf("--interval must be greater than zero, got %s", d) + } + return d, nil +} + +func readRun(ctx *cmdctx.Ctx, identity string) (map[string]any, error) { + resp, _, err := client.New(ctx).Get(basePath+"/runs/"+url.PathEscape(identity), scopeParams(ctx)) + if err != nil { + return nil, err + } + return asMap(resp), nil +} + +// Returns nil while the context is live. Watching is a read, so the wording says the run was left alone. +func stoppedWatching(ctx *cmdctx.Ctx, identity, status string) error { + cause := context.Cause(ctx.Context) + if cause == nil { + return nil + } + still := "" + if status != "" { + still = ", it was still " + status + } + reason := "watching was interrupted" + if cmdctx.IsTimeout(cause) { + reason = cause.Error() + } + return fmt.Errorf("stopped watching run %s%s: %s. The run is unaffected and can be followed again with: harness get loadtest_run %s --follow", + identity, still, reason, identity) +} + +// One line per poll, so a watch reads as a timeline. Unmeasured fields are omitted, not zeroed. +func progressLine(run map[string]any) string { + status := stringField(run, "status") + targetUsers := floatField(run, "targetUsers") + metrics := asMap(run["lastMetrics"]) + if len(metrics) == 0 { + return fmt.Sprintf("%-9s users=%.0f", status, targetUsers) + } + + var line strings.Builder + fmt.Fprintf(&line, "%-9s users=", status) + // The ramp-up gap is worth watching, but not every tool reports the current count. + if current := floatField(metrics, "currentUsers"); current > 0 { + fmt.Fprintf(&line, "%.0f/%.0f", current, targetUsers) + } else { + fmt.Fprintf(&line, "%.0f", targetUsers) + } + + fmt.Fprintf(&line, " rps=%.1f requests=%.0f failures=%.0f errors=%.2f%%", + floatField(metrics, "totalRps"), floatField(metrics, "totalRequests"), + floatField(metrics, "totalFailures"), floatField(metrics, "errorRate")) + fmt.Fprintf(&line, " avg=%.0fms p50=%.0fms p95=%.0fms p99=%.0fms", + floatField(metrics, "avgResponseMs"), floatField(metrics, "p50ResponseMs"), + floatField(metrics, "p95ResponseMs"), floatField(metrics, "p99ResponseMs")) + + // Latency is time to first byte, which only JMeter measures; "lat-p99=0ms" would claim an instant reply. + if hasLatency(metrics) { + fmt.Fprintf(&line, " lat-avg=%.0fms lat-p50=%.0fms lat-p95=%.0fms lat-p99=%.0fms", + floatField(metrics, "avgLatencyMs"), floatField(metrics, "p50LatencyMs"), + floatField(metrics, "p95LatencyMs"), floatField(metrics, "p99LatencyMs")) + } + + return line.String() +} + +func hasLatency(metrics map[string]any) bool { + return floatField(metrics, "avgLatencyMs") > 0 || floatField(metrics, "p50LatencyMs") > 0 || + floatField(metrics, "p95LatencyMs") > 0 || floatField(metrics, "p99LatencyMs") > 0 +} + +// Stopped and finished are both outcomes the caller asked for, so only Failed exits non-zero. +func terminalError(run map[string]any, identity string) error { + if stringField(run, "status") != runStatusFailed { + return nil + } + if msg := stringField(run, "errorMessage"); msg != "" { + return fmt.Errorf("run %s failed: %s", identity, msg) + } + return fmt.Errorf("run %s failed", identity) +} diff --git a/modules/rt/watch_test.go b/modules/rt/watch_test.go new file mode 100644 index 0000000..f716283 --- /dev/null +++ b/modules/rt/watch_test.go @@ -0,0 +1,338 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/harness/cli/pkg/cmdctx" +) + +func TestIsTerminalStatus(t *testing.T) { + for _, status := range []string{runStatusStopped, runStatusFinished, runStatusFailed} { + if !isTerminalStatus(status) { + t.Errorf("%q should end a watch", status) + } + } + for _, status := range []string{"Queued", "Initializing", "Running", ""} { + if isTerminalStatus(status) { + t.Errorf("%q should keep a watch going", status) + } + } +} + +func TestPollInterval(t *testing.T) { + cases := []struct { + name string + raw any + want time.Duration + wantErr string + }{ + {name: "absent", want: defaultPollInterval}, + {name: "empty", raw: "", want: defaultPollInterval}, + {name: "seconds", raw: "5s", want: 5 * time.Second}, + {name: "minutes", raw: "2m", want: 2 * time.Minute}, + {name: "not a duration", raw: "5", wantErr: "not a duration"}, + {name: "words", raw: "quick", wantErr: "not a duration"}, + {name: "zero", raw: "0s", wantErr: "greater than zero"}, + {name: "negative", raw: "-5s", wantErr: "greater than zero"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fv := map[string]any{} + if tc.raw != nil { + fv["interval"] = tc.raw + } + got, err := pollInterval(&cmdctx.Ctx{FlagValues: fv}) + + if tc.wantErr != "" { + if err == nil { + t.Fatalf("expected an error containing %q, got %s", tc.wantErr, got) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error %q does not contain %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Fatalf("got %s, want %s", got, tc.want) + } + }) + } +} + +func TestRunToWatch(t *testing.T) { + ctx := &cmdctx.Ctx{Id: "from-argv"} + if got := runToWatch(ctx, map[string]any{"identity": "from-response"}); got != "from-response" { + t.Errorf("got %q, want the identity in the response", got) + } + for _, result := range []any{nil, map[string]any{}, map[string]any{"identity": ""}, "ok"} { + if got := runToWatch(ctx, result); got != "from-argv" { + t.Errorf("result %v: got %q, want the id from the command line", result, got) + } + } + if got := runToWatch(&cmdctx.Ctx{}, nil); got != "" { + t.Errorf("with nothing to watch, got %q, want empty", got) + } +} + +func TestTerminalError(t *testing.T) { + failed := map[string]any{"status": runStatusFailed, "errorMessage": "target unreachable"} + err := terminalError(failed, "run-1") + if err == nil || !strings.Contains(err.Error(), "target unreachable") { + t.Fatalf("expected the server's reason, got %v", err) + } + if !strings.Contains(err.Error(), "run-1") { + t.Errorf("expected the run id in %q", err) + } + + err = terminalError(map[string]any{"status": runStatusFailed}, "run-2") + if err == nil || !strings.Contains(err.Error(), "run-2 failed") { + t.Fatalf("expected a bare failure, got %v", err) + } + + for _, status := range []string{runStatusFinished, runStatusStopped} { + if err := terminalError(map[string]any{"status": status}, "run-3"); err != nil { + t.Errorf("%q should exit clean, got %v", status, err) + } + } +} + +func TestProgressLineWithoutMetrics(t *testing.T) { + got := progressLine(map[string]any{"status": "Queued", "targetUsers": float64(50)}) + if !strings.Contains(got, "Queued") || !strings.Contains(got, "users=50") { + t.Fatalf("got %q, want the status and the target", got) + } + // Nothing has been measured yet, so no measurements should be claimed. + if strings.Contains(got, "rps=") { + t.Errorf("got %q, want no metrics before any arrive", got) + } +} + +func TestProgressLineWithMetrics(t *testing.T) { + run := map[string]any{ + "status": "Running", + "targetUsers": float64(100), + "lastMetrics": map[string]any{ + "currentUsers": float64(40), + "totalRps": 12.34, + "totalRequests": float64(5000), + "totalFailures": float64(7), + "errorRate": 0.14, + "avgResponseMs": float64(120), + "p50ResponseMs": float64(100), + "p95ResponseMs": float64(300), + "p99ResponseMs": float64(800), + }, + } + got := progressLine(run) + // During ramp-up the gap between current and target is the thing worth watching. + for _, want := range []string{"Running", "users=40/100", "rps=12.3", "requests=5000", "failures=7", "errors=0.14%", "avg=120ms", "p99=800ms"} { + if !strings.Contains(got, want) { + t.Errorf("got %q, want it to contain %q", got, want) + } + } + // Latency is time to first byte, which only JMeter measures. + if strings.Contains(got, "lat-") { + t.Errorf("got %q, want no latency block when the tool does not measure it", got) + } +} + +func TestProgressLineWithoutCurrentUsers(t *testing.T) { + run := map[string]any{ + "status": "Running", + "targetUsers": float64(100), + "lastMetrics": map[string]any{"totalRps": 5.0}, + } + got := progressLine(run) + if strings.Contains(got, "0/100") { + t.Fatalf("got %q, want the target alone rather than a zero current count", got) + } + if !strings.Contains(got, "users=100") { + t.Fatalf("got %q, want users=100", got) + } +} + +func TestProgressLineWithLatency(t *testing.T) { + run := map[string]any{ + "status": "Running", + "targetUsers": float64(10), + "lastMetrics": map[string]any{ + "totalRps": 1.0, + "avgLatencyMs": float64(15), + "p99LatencyMs": float64(90), + "p50LatencyMs": float64(10), + "p95LatencyMs": float64(60), + "avgResponseMs": float64(120), + }, + } + got := progressLine(run) + for _, want := range []string{"lat-avg=15ms", "lat-p50=10ms", "lat-p95=60ms", "lat-p99=90ms"} { + if !strings.Contains(got, want) { + t.Errorf("got %q, want it to contain %q", got, want) + } + } +} + +func TestHasLatency(t *testing.T) { + if hasLatency(map[string]any{"totalRps": 5.0}) { + t.Error("a tool that reports no latency should not get a latency block") + } + if hasLatency(map[string]any{"avgLatencyMs": float64(0), "p99LatencyMs": float64(0)}) { + t.Error("all-zero latency is the server omitting it, not an instant reply") + } + if !hasLatency(map[string]any{"p95LatencyMs": float64(60)}) { + t.Error("one measured percentile is enough to show the block") + } +} + +func TestStoppedWatchingSaysTheRunIsUnaffected(t *testing.T) { + ctx, _ := apiCtx(t, nil) + inner, cancel := context.WithCancelCause(ctx.Context) + cancel(errors.New("interrupted")) + ctx.Context = inner + + err := stoppedWatching(ctx, "checkout-aaa", "Running") + if err == nil { + t.Fatal("expected giving up to be reported") + } + for _, want := range []string{"checkout-aaa", "still Running", "unaffected", "--follow"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("message %q should mention %q", err, want) + } + } +} + +func TestStoppedWatchingUsesTheTimeoutWording(t *testing.T) { + ctx, _ := apiCtx(t, nil) + inner, cancel := context.WithCancelCause(ctx.Context) + cancel(&cmdctx.TimeoutError{Secs: 30}) + ctx.Context = inner + + err := stoppedWatching(ctx, "checkout-aaa", "Running") + if err == nil || !strings.Contains(err.Error(), "timed out after 30s") { + t.Fatalf("expected the timeout named, got %v", err) + } +} + +func TestStoppedWatchingOmitsAnUnknownStatus(t *testing.T) { + ctx, _ := apiCtx(t, nil) + inner, cancel := context.WithCancelCause(ctx.Context) + cancel(errors.New("interrupted")) + ctx.Context = inner + + err := stoppedWatching(ctx, "checkout-aaa", "") + if err == nil || strings.Contains(err.Error(), "still") { + t.Fatalf("expected no claim about the status, got %v", err) + } +} + +func TestStoppedWatchingIsSilentWhileLive(t *testing.T) { + ctx, _ := apiCtx(t, nil) + if err := stoppedWatching(ctx, "checkout-aaa", "Running"); err != nil { + t.Fatalf("got %v, want nothing while the watch is still live", err) + } +} + +func TestWatchFollowFnNeedsARunToWatch(t *testing.T) { + ctx, _ := apiCtx(t, nil) + err := watchFollowFn(ctx, map[string]any{"acknowledged": true}) + if err == nil || !strings.Contains(err.Error(), "did not name a run") { + t.Fatalf("expected the missing run reported, got %v", err) + } +} + +func TestWatchFollowFnRejectsABadInterval(t *testing.T) { + ctx, _ := apiCtx(t, nil) + ctx.FlagValues["interval"] = "soon" + err := watchFollowFn(ctx, map[string]any{"identity": "checkout-aaa"}) + if err == nil || !strings.Contains(err.Error(), "--interval") { + t.Fatalf("expected the interval rejected before polling, got %v", err) + } +} + +func TestWatchFollowFnStopsAtATerminalStatus(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{ + "identity": "checkout-aaa", "status": runStatusFinished, + }, + }) + ctx.FlagValues["interval"] = "1h" // never elapses; the watch must not wait for it + + if err := watchFollowFn(ctx, map[string]any{"identity": "checkout-aaa"}); err != nil { + t.Fatalf("a finished run should exit clean, got %v", err) + } + if len(*calls) != 1 { + t.Errorf("polled %d times, want the watch to end on the first read", len(*calls)) + } +} + +func TestWatchFollowFnFailsOnAFailedRun(t *testing.T) { + ctx, _ := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{ + "identity": "checkout-aaa", "status": runStatusFailed, + "errorMessage": "the host refused every connection", + }, + }) + + err := watchFollowFn(ctx, map[string]any{"identity": "checkout-aaa"}) + if err == nil { + t.Fatal("expected a failed run to exit non-zero") + } + if !strings.Contains(err.Error(), "the host refused every connection") { + t.Errorf("error %q should carry what the server said", err) + } +} + +func TestWatchFollowFnFallsBackToTheCommandLineID(t *testing.T) { + ctx, calls := apiCtx(t, map[string]any{ + api("/runs/checkout-aaa"): map[string]any{"status": runStatusStopped}, + }) + ctx.Id = "checkout-aaa" + + if err := watchFollowFn(ctx, map[string]any{"acknowledged": true}); err != nil { + t.Fatalf("a stopped run should exit clean, got %v", err) + } + if _, ok := findCall(calls, "GET", api("/runs/checkout-aaa")); !ok { + t.Error("the watch never read the run named on the command line") + } +} + +func TestWatchFollowFnSurfacesAReadFailure(t *testing.T) { + ctx, _ := apiCtx(t, nil) // every route 404s + err := watchFollowFn(ctx, map[string]any{"identity": "checkout-aaa"}) + if err == nil { + t.Fatal("expected an unreadable run to end the watch") + } + if strings.Contains(err.Error(), "unaffected") { + t.Errorf("error %q reads as giving up, but the watch was never interrupted", err) + } +} + +func TestWatchFollowFnReportsATimeoutDuringARead(t *testing.T) { + ctx, _ := apiCtx(t, nil) + inner, cancel := context.WithCancelCause(ctx.Context) + cancel(&cmdctx.TimeoutError{Secs: 5}) + ctx.Context = inner + + err := watchFollowFn(ctx, map[string]any{"identity": "checkout-aaa"}) + if err == nil { + t.Fatal("expected the timeout to end the watch") + } + for _, want := range []string{"timed out after 5s", "checkout-aaa", "unaffected", "--follow"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("message %q should mention %q", err, want) + } + } + if strings.Contains(err.Error(), "API request failed") { + t.Errorf("message %q is the raw transport error, not the interrupted-watch one", err) + } +} diff --git a/modules/rt/window.go b/modules/rt/window.go new file mode 100644 index 0000000..f17a9cf --- /dev/null +++ b/modules/rt/window.go @@ -0,0 +1,59 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "fmt" + "strconv" + "time" + + "github.com/harness/cli/pkg/cmdctx" + "github.com/harness/cli/pkg/exprenv/exprfuncs" +) + +const usageWindowParamsID = "usage_window" + +// The console opens on the last 30 days, so the bare command agrees with what people see. +const defaultUsageWindow = 30 * 24 * time.Hour + +// Both bounds are required — the route parses them unchecked and answers 400 on an empty +// string rather than defaulting. The spec has no way to say "now", so the window is built here. +func usageWindowParams(ctx *cmdctx.Ctx) (map[string]string, error) { + // End first, so the default span hangs off it: otherwise --to alone would imply a start + // after its own end, and "usage up to June" would be refused as inverted. + end, err := usageBound(ctx, "to", time.Now()) + if err != nil { + return nil, err + } + start, err := usageBound(ctx, "from", time.UnixMilli(end).Add(-defaultUsageWindow)) + if err != nil { + return nil, err + } + // The server's own inverted-window error does not say which flag was wrong. + if end < start { + return nil, fmt.Errorf("--to is earlier than --from, so the window is empty: %s to %s", + time.UnixMilli(start).Format(time.RFC3339), time.UnixMilli(end).Format(time.RFC3339)) + } + return map[string]string{ + "startTime": strconv.FormatInt(start, 10), + "endTime": strconv.FormatInt(end, 10), + }, nil +} + +// Parses through the spec's own parseDateMs, so the accepted formats match the flag descriptions. +func usageBound(ctx *cmdctx.Ctx, flag string, fallback time.Time) (int64, error) { + raw := cmdctx.GetString(ctx.FlagValues, flag) + if raw == "" { + return fallback.UnixMilli(), nil + } + parsed := exprfuncs.ParseDateMs(raw) + if parsed == "" { + return 0, fmt.Errorf("--%s %q is not a date: use a span such as 30d or 2w, a date such as 2026-01-01, or unix millis", flag, raw) + } + ms, err := strconv.ParseInt(parsed, 10, 64) + if err != nil { + return 0, fmt.Errorf("--%s %q is not a date: %w", flag, raw, err) + } + return ms, nil +} diff --git a/modules/rt/window_test.go b/modules/rt/window_test.go new file mode 100644 index 0000000..e65ef3b --- /dev/null +++ b/modules/rt/window_test.go @@ -0,0 +1,146 @@ +// Copyright © 2026 Harness Inc. +// SPDX-License-Identifier: Apache-2.0 + +package rt + +import ( + "strconv" + "strings" + "testing" + "time" + + "github.com/harness/cli/pkg/cmdctx" +) + +// Returns both bounds parsed, since every assertion here is about the instants. +func windowFor(t *testing.T, flags map[string]any) (time.Time, time.Time) { + t.Helper() + qp, err := usageWindowParams(&cmdctx.Ctx{FlagValues: flags}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + start, err := strconv.ParseInt(qp["startTime"], 10, 64) + if err != nil { + t.Fatalf("startTime %q is not millis: %v", qp["startTime"], err) + } + end, err := strconv.ParseInt(qp["endTime"], 10, 64) + if err != nil { + t.Fatalf("endTime %q is not millis: %v", qp["endTime"], err) + } + return time.UnixMilli(start), time.UnixMilli(end) +} + +func TestUsageWindowSendsBothBounds(t *testing.T) { + qp, err := usageWindowParams(&cmdctx.Ctx{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, key := range []string{"startTime", "endTime"} { + if qp[key] == "" { + t.Errorf("%s is missing; the route rejects a half-open window", key) + } + } + if len(qp) != 2 { + t.Errorf("got %v, want only the two bounds", qp) + } +} + +func TestUsageWindowDefaultsToThirtyDays(t *testing.T) { + before := time.Now() + start, end := windowFor(t, nil) + after := time.Now() + + if end.Before(before.Add(-time.Second)) || end.After(after.Add(time.Second)) { + t.Errorf("end = %s, want roughly now", end) + } + if span := end.Sub(start); span < defaultUsageWindow-time.Minute || span > defaultUsageWindow+time.Minute { + t.Errorf("window spans %s, want %s", span, defaultUsageWindow) + } +} + +func TestUsageWindowFromOnlyEndsNow(t *testing.T) { + before := time.Now() + start, end := windowFor(t, map[string]any{"from": "7d"}) + after := time.Now() + + if span := end.Sub(start); span < 7*24*time.Hour-time.Minute || span > 7*24*time.Hour+time.Minute { + t.Errorf("window spans %s, want 7 days", span) + } + if end.Before(before.Add(-time.Second)) || end.After(after.Add(time.Second)) { + t.Errorf("end = %s, want roughly now", end) + } +} + +func TestUsageWindowToOnlyEndsAtTheGivenDate(t *testing.T) { + start, end := windowFor(t, map[string]any{"to": "2026-06-01"}) + + // A bare date is midnight UTC, so the window is the same whoever runs the command. + wantEnd := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + if !end.Equal(wantEnd) { + t.Errorf("end = %s, want the given %s", end.UTC(), wantEnd) + } + if span := end.Sub(start); span != defaultUsageWindow { + t.Errorf("window spans %s, want the default %s ending at --to", span, defaultUsageWindow) + } + if start.After(end) { + t.Errorf("start %s is after end %s", start, end) + } +} + +func TestUsageWindowExplicitDates(t *testing.T) { + start, end := windowFor(t, map[string]any{"from": "2026-01-01", "to": "2026-02-01"}) + if start.Year() != 2026 || start.Month() != time.January { + t.Errorf("start = %s, want 2026-01-01", start) + } + if end.Month() != time.February { + t.Errorf("end = %s, want 2026-02-01", end) + } +} + +func TestUsageWindowAcceptsUnixMillis(t *testing.T) { + want := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + start, _ := windowFor(t, map[string]any{"from": strconv.FormatInt(want.UnixMilli(), 10)}) + if !start.Equal(want) { + t.Errorf("start = %s, want %s", start, want) + } +} + +func TestUsageWindowRejectsInverted(t *testing.T) { + _, err := usageWindowParams(&cmdctx.Ctx{FlagValues: map[string]any{ + "from": "2026-06-01", "to": "2026-01-01", + }}) + if err == nil { + t.Fatal("expected an inverted window to be refused") + } + // The message has to name both ends, since the flags are easy to mix up. + for _, want := range []string{"--to", "--from", "2026-06-01", "2026-01-01"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q should mention %q", err, want) + } + } +} + +func TestUsageWindowAllowsAnInstant(t *testing.T) { + start, end := windowFor(t, map[string]any{"from": "2026-01-01", "to": "2026-01-01"}) + if !start.Equal(end) { + t.Errorf("start %s and end %s should be the same instant", start, end) + } +} + +func TestUsageWindowRejectsUnparseableBounds(t *testing.T) { + for _, tc := range []struct{ flag, value string }{ + {"from", "nonsense"}, + {"to", "nonsense"}, + {"from", "30 days ago"}, + } { + _, err := usageWindowParams(&cmdctx.Ctx{FlagValues: map[string]any{tc.flag: tc.value}}) + if err == nil { + t.Errorf("--%s %q: expected an error", tc.flag, tc.value) + continue + } + // The flag description is the only other place these formats appear. + if !strings.Contains(err.Error(), "--"+tc.flag) || !strings.Contains(err.Error(), "30d") { + t.Errorf("--%s %q: error %q should name the flag and the accepted formats", tc.flag, tc.value, err) + } + } +} diff --git a/pkg/spec/rt.spec.yaml b/pkg/spec/rt.spec.yaml new file mode 100644 index 0000000..76cbf6d --- /dev/null +++ b/pkg/spec/rt.spec.yaml @@ -0,0 +1,1655 @@ +spec_version: 1 +module_type: builtin +module_desc: Resilience Testing — RT load tests, runs, templates, scripts, and usage +help_text: | + ## Resilience Testing (rt) + + Resilience Testing (RT) is how Harness verifies the way a service holds up + under stress. Load testing is the feature area available today: run JMeter, + Locust, and k6 RT load tests against your services. Later RT areas add their + own nouns under this same module, so `rt` is the module and `loadtest` is + what you name on the command line. + + ### Domain Model + + A **loadtest** is a reusable RT load test definition: which tool to run, + which script to run, which infrastructure and environment to run it on, and + the tunables (target users, duration) it accepts. Creating one does not run + anything. + + `create loadtest --set toolType=K6 --set infraIdentifier= --set + environmentIdentifier=` builds the definition from key=value pairs, or + pass a whole body with `-f test.yaml`. Use one or the other: when `-f` is + given it supplies the entire body and `--set` is ignored. + + Two variants create a load test from something you already have: + `create loadtest:from_json -f spec.json` takes a declarative endpoint + spec instead of a tool script, and `create loadtest:from_template + --template ` instantiates a template. A template-backed test is + either a REFERENCE (a pointer — the template stays the source of truth) or a + LOCAL copy. Use `execute loadtest:sync ` to re-pull a REFERENCE test + after its template changes; `templateUpdateAvailable` on the test tells you + when that is worth doing. + + Export a test with `get loadtest --yaml -o test.yaml`, edit it, and + apply it back with `update loadtest -f test.yaml`. + + `list loadtest:variables ` shows the variables and runtime inputs the + test accepts — the inputs you can override at run time. + + A **loadtest_run** is one execution of an RT load test. + `execute loadtest ` starts a run and prints its identity; add `--follow` + to stay attached and stream status and metrics until the run reaches a + terminal state, exiting non-zero if it fails. Supply run-time values with + `--set targetUsers=50`. + + Inspect runs with `list loadtest_run` (across the scope) or + `list loadtest_run ` (for one test), and + `get loadtest_run `. While a run is live you can retarget it with + `update loadtest_run --set targetUsers=200` and end it early with + `execute loadtest_run:stop `. + + `execute loadtest_run:rerun ` starts a fresh run of the same load + test with the same shape. + + Results come from `get loadtest_run:summary ` for the headline + numbers, `get loadtest_run:metrics --view timeseries|scatter|aggregate` + for the metric series, and `get loadtest_run:graph ` for the + render-ready chart payload. + + A **loadtest_template** is a shareable, versioned RT load test definition. + Templates live in hubs; `list loadtest_template` spans every hub in the + scope unless you narrow it with `--hub`, and the HUB column names the hub + each template came from. Every other template command targets one hub, so + pass `--hub` when the template is not in the default (unhubbed) one. + + Templates are revisioned: `create loadtest_template:revision --revision + ` adds a revision, `list loadtest_template:revision ` lists them, + and `get loadtest_template:revision /` fetches one. + `get loadtest_template:yaml ` writes the template as a YAML document. + + A **loadtest_script** is the script attached to an RT load test, with its own + revision history. `get loadtest_script ` shows the current + script, `list loadtest_script ` lists revisions, and + `update loadtest_script -f script.jmx` uploads a new one. + + A **composite_loadtest** is a pipeline that runs RT load tests and chaos + probes together as stages. Identifiers must not contain hyphens — use + underscores. + + **loadtest_usage** reports RT load testing consumption for the account. It is + account-wide and ignores `--org` / `--project`. + + ### Nouns + + {{nouns}} + +nouns: + - noun: loadtest + short_desc: "RT load test definitions — the tool, script, infrastructure, and tunables for a test." + noun_aliases: [loadtests] + fields: + - id: identity + label: ID + expr: it.identity + - id: name + expr: it.name + mutable_path: name + - id: tool_type + label: Tool + expr: it.toolType + - id: environment + expr: it.environmentIdentifier + mutable_path: environmentIdentifier + - id: infra + label: Infra + expr: it.infraIdentifier + mutable_path: infraIdentifier + - id: target_type + expr: it.targetType + mutable_path: targetType + - id: last_status + label: Last Run + expr: 'it.recentRuns == nil || len(it.recentRuns) == 0 ? "" : it.recentRuns[0].status' + - id: last_executed + expr: it.lastExecuted + - id: target_users + label: Users + expr: it.targetUsers + align: right + - id: duration + label: Duration (s) + expr: it.durationSeconds + align: right + - id: max_duration + label: Max Duration (s) + expr: it.maxDurationSec + align: right + - id: script_source + expr: it.scriptSource + mutable_path: scriptSource + - id: cleanup_policy + expr: it.cleanupPolicy + mutable_path: cleanupPolicy + - id: import_type + expr: it.importType + - id: template + expr: 'it.templateReference == nil ? "" : it.templateReference.identity' + - id: template_revision + expr: 'it.templateReference == nil ? "" : it.templateReference.revision' + - id: template_hub + expr: 'it.templateReference == nil ? "" : it.templateReference.hubIdentity' + - id: update_available + label: Update Available + expr: it.templateUpdateAvailable + - id: tags + expr: it.tags + field_type: set + mutable_path: tags + - id: service_references + label: Services + expr: it.serviceReferences + field_type: set + mutable_path: serviceReferences + - id: description + expr: it.description + width_max: 60 + mutable_path: description + - id: created_at + expr: it.createdAt + - id: created_by + expr: it.createdBy + - id: updated_at + expr: it.updatedAt + - id: updated_by + expr: it.updatedBy + - id: script + expr: it.scriptContent + field_type: multiline_text + + - noun: loadtest_run + short_desc: "RT load test runs — one execution of an RT load test, with its live and final metrics." + noun_aliases: [loadtest_runs] + fields: + - id: identity + label: ID + expr: it.identity + - id: name + expr: it.name + - id: loadtest + label: Load Test + expr: it.loadTestIdentity + - id: status + expr: it.status + - id: run_sequence + label: "#" + expr: it.runSequence + align: right + - id: tool_type + label: Tool + expr: it.toolType + - id: target_users + label: Users + expr: it.targetUsers + align: right + - id: spawn_rate + expr: it.spawnRate + align: right + - id: duration + label: Duration (s) + expr: it.durationSeconds + align: right + - id: ramp_up + label: Ramp Up (s) + expr: it.rampUpTimeSec + align: right + - id: worker_count + label: Workers + expr: it.workerCount + align: right + - id: environment + expr: it.environmentIdentifier + - id: infra + label: Infra + expr: it.infraIdentifier + - id: target_type + expr: it.targetType + - id: started_at + expr: it.startedAt + - id: finished_at + expr: it.finishedAt + - id: rps + label: RPS + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.totalRps' + align: right + - id: requests + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.totalRequests' + align: right + - id: failures + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.totalFailures' + align: right + - id: error_rate + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.errorRate' + align: right + - id: avg_response_ms + label: Avg (ms) + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.avgResponseMs' + align: right + - id: p95_response_ms + label: P95 (ms) + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.p95ResponseMs' + align: right + - id: p99_response_ms + label: P99 (ms) + expr: 'it.lastMetrics == nil ? nil : it.lastMetrics.p99ResponseMs' + align: right + - id: error_code + expr: it.errorCode + - id: error_message + expr: it.errorMessage + width_max: 80 + - id: created_at + expr: it.createdAt + - id: created_by + expr: it.createdBy + + - noun: loadtest_template + short_desc: "RT load test templates — shareable, revisioned definitions held in hubs." + noun_aliases: [loadtest_templates] + fields: + - id: identity + label: ID + expr: it.identity + - id: name + expr: it.name + mutable_path: name + - id: revision + expr: it.revision + - id: hub + label: Hub + expr: it.hubIdentity + - id: tool_type + label: Tool + expr: it.toolType + - id: infra_type + expr: it.infraType + mutable_path: infraType + - id: script_source + expr: it.scriptSource + mutable_path: scriptSource + - id: tags + expr: it.tags + field_type: set + mutable_path: tags + - id: description + expr: it.description + width_max: 60 + mutable_path: description + - id: created_at + expr: it.createdAt + - id: created_by + expr: it.createdBy + - id: updated_at + expr: it.updatedAt + - id: updated_by + expr: it.updatedBy + - id: script + expr: it.scriptContent + field_type: multiline_text + + - noun: loadtest_script + short_desc: "RT load test scripts — the script attached to an RT load test, and its revisions." + noun_aliases: [loadtest_scripts] + fields: + - id: identity + label: ID + expr: it.identity + - id: revision_number + label: Rev + expr: it.revisionNumber + align: right + - id: loadtest + label: Load Test + expr: it.loadTestIdentity + - id: description + expr: it.description + width_max: 60 + - id: is_bundle + label: Bundle + expr: it.isBundle + - id: bundle_main_file + expr: it.bundleMainFile + - id: created_at + expr: it.createdAt + - id: created_by + expr: it.createdBy + - id: script + expr: 'coalesce(it.scriptContent, it.bundleMainContent)' + field_type: multiline_text + + - noun: composite_loadtest + short_desc: "Composite RT load tests — pipelines that run RT load tests and chaos probes as stages." + noun_aliases: [composite_loadtests] + fields: + - id: pipeline_identifier + label: ID + expr: it.pipelineIdentifier + - id: name + expr: it.name + - id: loadtest_count + label: Load Tests + expr: it.loadTestCount + align: right + - id: probe_count + label: Probes + expr: it.probeCount + align: right + - id: stage_names + label: Stages + expr: it.stageNames + field_type: set + - id: tags + expr: it.tags + field_type: tags + - id: description + expr: it.description + width_max: 60 + # Epoch millis: without epochMs the table prints a float64 as 1.787e+12. + - id: created_at + expr: epochMs(it.createdAt) + field_type: ts + - id: last_updated_at + expr: epochMs(it.lastUpdatedAt) + field_type: ts + + - noun: loadtest_variable + short_desc: "Variables and runtime inputs an RT load test or template accepts." + noun_aliases: [loadtest_variables] + fields: + - id: name + expr: it.name + - id: value + expr: it.value + - id: type + expr: it.type + - id: required + expr: it.required + - id: category + expr: it.category + - id: default + expr: it.default + - id: allowed_values + expr: it.allowedValues + field_type: set + - id: description + expr: it.description + width_max: 60 + + - noun: loadtest_run_summary + short_desc: "Headline result of a finished or running RT load test run." + fields: + - id: run_id + label: Run + expr: it.runId + - id: run_name + expr: it.runName + - id: status + expr: it.status + - id: duration + expr: it.duration + - id: total_requests + expr: it.totalRequests + align: right + - id: rps + label: RPS + expr: it.requestsPerSec + align: right + - id: error_rate + label: Error Rate (%) + expr: it.errorRate + align: right + - id: avg_response_s + label: Avg (s) + expr: it.avgResponseTime + align: right + - id: p50_response_ms + label: P50 (ms) + expr: it.p50ResponseMs + align: right + - id: p95_response_ms + label: P95 (ms) + expr: it.p95ResponseMs + align: right + - id: p99_response_ms + label: P99 (ms) + expr: it.p99ResponseMs + align: right + - id: peak_vus + label: Peak VUs + expr: it.peakVUs + align: right + - id: iterations + expr: it.iterationsCompleted + align: right + - id: target_users + label: Target Users + expr: it.targetUsers + align: right + - id: spawn_rate + expr: it.spawnRate + align: right + - id: duration_seconds + label: Duration (s) + expr: it.durationSeconds + align: right + - id: started_at + expr: it.startedAt + - id: finished_at + expr: it.finishedAt + + - noun: loadtest_metric + short_desc: "One time-series sample taken during an RT load test run." + noun_aliases: [loadtest_metrics] + fields: + - id: timestamp + expr: it.timestamp + - id: rps + label: RPS + expr: it.totalRps + align: right + - id: users + expr: it.currentUsers + align: right + - id: p50_response_ms + label: P50 (ms) + expr: it.p50ResponseMs + align: right + - id: p95_response_ms + label: P95 (ms) + expr: it.p95ResponseMs + align: right + - id: p99_response_ms + label: P99 (ms) + expr: it.p99ResponseMs + align: right + - id: error_rate + label: Error Rate (%) + expr: it.errorRate + align: right + + - noun: loadtest_graph_point + short_desc: "One plotting sample from an RT load test run — the minimal series behind the run chart." + noun_aliases: [loadtest_graph_points] + fields: + - id: timestamp + expr: epochMs(it.timestamp) + - id: users + expr: it.users + align: right + - id: rps + label: RPS + expr: it.requestsPerSec + align: right + - id: errors_per_sec + label: Errors/s + expr: it.errorsPerSec + align: right + - id: avg_response_s + label: Avg (s) + expr: it.avgResponseTime + align: right + + - noun: loadtest_request + short_desc: "One sampled request from an RT load test run, for response-time distribution." + noun_aliases: [loadtest_requests] + fields: + - id: timestamp + expr: it.timestamp + - id: method + expr: it.method + - id: endpoint + expr: it.endpoint + - id: response_time_ms + label: Response (ms) + expr: it.responseTimeMs + align: right + - id: success + expr: it.success + + - noun: loadtest_endpoint_stat + short_desc: "Per-endpoint request statistics aggregated across an RT load test run." + noun_aliases: [loadtest_endpoint_stats] + fields: + - id: method + expr: it.method + - id: endpoint + expr: it.endpoint + - id: total_requests + expr: it.totalRequests + align: right + - id: total_failures + expr: it.totalFailures + align: right + - id: error_rate + label: Error Rate (%) + expr: it.errorRate + align: right + - id: avg_response_ms + label: Avg (ms) + expr: it.avgResponseTimeMs + align: right + - id: min_response_ms + label: Min (ms) + expr: it.minResponseTimeMs + align: right + - id: max_response_ms + label: Max (ms) + expr: it.maxResponseTimeMs + align: right + - id: p50_response_ms + label: P50 (ms) + expr: it.p50ResponseMs + align: right + - id: rps + label: RPS + expr: it.avgRps + align: right + + - noun: loadtest_usage + short_desc: "Per-service RT load testing consumption. Account-wide; ignores --org and --project." + noun_aliases: [loadtest_usages] + fields: + - id: service + label: Service + expr: it.serviceId + - id: org + expr: it.orgId + - id: project + expr: it.projectId + - id: run_count + label: Runs + expr: it.runCount + align: right + - id: peak_users + expr: it.peakUsers + align: right + - id: max_worker_count + label: Max Workers + expr: it.maxWorkerCount + align: right + - id: total_duration_sec + label: Duration (s) + expr: it.totalDurationSec + align: right + - id: total_vu_seconds + label: VU Seconds + expr: it.totalVuSeconds + align: right + - id: tool_types + label: Tools + expr: it.toolTypes + field_type: set + - id: infra_types + label: Infra + expr: it.infraTypes + field_type: set + - id: complexity + expr: it.complexity + align: right + + - noun: loadtest_usage_total + short_desc: "Account-wide RT load testing consumption total and the weighting behind it." + fields: + - id: account + expr: it.accountId + - id: total_usage + label: Total Usage + expr: it.totalUsage + align: right + - id: service_count + label: Services + expr: it.serviceCount + align: right + - id: base_service_weight + expr: 'it.ratioConfig == nil ? nil : it.ratioConfig.baseServiceWeight' + align: right + - id: metric_weights + expr: 'it.ratioConfig == nil ? nil : formatMetadata(it.ratioConfig.metricWeights)' + - id: metric_thresholds + expr: 'it.ratioConfig == nil ? nil : formatMetadata(it.ratioConfig.metricThresholds)' + - id: updated_at + expr: 'it.ratioConfig == nil ? nil : epochMs(it.ratioConfig.updatedAt)' + - id: updated_by + expr: 'it.ratioConfig == nil ? nil : it.ratioConfig.updatedBy' + +commands: + # ---------------------------------------------------------------- loadtest + + - command: list loadtest + verb: list + noun: loadtest + short: "List load tests: harness list loadtest [--search ] [--tool JMeter|Locust|K6] [--status ]" + handler_type: endpoint + flags: + - name: search + description: Filter RT load tests by name + - name: tool + description: Filter by tool type + completion_values: [JMeter, Locust, K6] + - name: environment + description: Filter by environment identifier + - name: status + description: Filter by last run status + completion_values: [Pending, Running, Stopping, Stopped, Finished, Failed] + - name: tag + description: Filter by tag (repeatable) + is_multi: true + - name: sort + description: Field to sort by + - name: sort-ascending + description: Sort ascending instead of descending + is_bool: true + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests + items_expr: it.items + get_id_expr: it.identity + completion: + id_expr: it.identity + name_expr: it.name + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + search: flags.search + toolType: flags.tool + environmentIdentifier: flags.environment + status: flags.status + sortField: flags.sort + sortAscending: 'flags["sort-ascending"] ? "true" : nil' + # A repeatable flag cannot go through query_params: it formats as "[]", not joined. + query_params_fn: loadtest_filters + paging: + paging_strategy: page_index + countable: true + page_index_param: page + page_size_param: limit + page_size_default: 15 + page_size_max: 100 + page_base: 0 + total_expr: it.pagination.totalItems + columns: [identity, name, tool_type, environment, last_status, last_executed] + + - command: get loadtest + verb: get + noun: loadtest + short: "Get a load test: harness get loadtest [--yaml]" + id_label: "" + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}} + item_expr: it + yaml_pick_expr: it + yaml_exclude: + - uniqueId + - parentUniqueId + - accountIdentifier + - organizationIdentifier + - projectIdentifier + - createdAt + - createdBy + - updatedAt + - updatedBy + - createdByUserDetails + - updatedByUserDetails + - recentRuns + - lastExecuted + - latestRevisionIdentifier + - templateUpdateAvailable + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + + - command: create loadtest + verb: create + noun: loadtest + requires_id: true + short: "Create a load test: harness create loadtest --set toolType=K6 --set infraIdentifier= --set environmentIdentifier=, or -f test.yaml" + flags_builtin: + set: true + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-tests + file_body: optional + create_strategy: set-fields + create_body_init: + identity: ctx.id + name: coalesce(ctx.setArgs.name, ctx.id) + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + text_footer: "\nStart it with: harness execute loadtest {{it.identity}}\n" + + - command: create loadtest:from_json + verb: create + noun: loadtest + noun_variant: from_json + requires_id: true + short: "Create a load test from a JSON endpoint spec: harness create loadtest:from_json -f spec.json --set infraIdentifier= --set environmentIdentifier=" + flags_builtin: + set: true + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-tests/from-json + file_body: optional + create_strategy: set-fields + create_body_init: + identity: ctx.id + name: coalesce(ctx.setArgs.name, ctx.id) + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + text_footer: "\nStart it with: harness execute loadtest {{it.identity}}\n" + + - command: create loadtest:from_template + verb: create + noun: loadtest + noun_variant: from_template + requires_id: true + short: "Create a load test from a template: harness create loadtest:from_template --template [--revision ] [--import-type REFERENCE|LOCAL]" + flags_builtin: + set: true + flags: + - name: template + description: Identity of the template to instantiate + required: true + completion_noun: loadtest_template + - name: revision + description: Template revision to pin (defaults to the latest) + - name: hub + description: Hub holding the template + - name: import-type + description: REFERENCE keeps the template as the source of truth; LOCAL copies it + default: REFERENCE + completion_values: [REFERENCE, LOCAL] + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-tests/from-template + file_body: optional + create_strategy: set-fields + create_body_init: + identity: ctx.id + name: coalesce(ctx.setArgs.name, ctx.id) + importType: flags["import-type"] + templateReference.identity: flags.template + templateReference.revision: 'flags.revision != "" ? flags.revision : nil' + templateReference.hubIdentity: 'flags.hub != "" ? flags.hub : nil' + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + text_footer: "\nStart it with: harness execute loadtest {{it.identity}}\n" + + - command: update loadtest + verb: update + noun: loadtest + short: "Update a load test: harness update loadtest --set name=, or -f test.yaml" + id_label: "" + flags_builtin: + set: true + del: true + handler_type: endpoint + endpoint: + method: PUT + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}} + file_body: optional + update_strategy: get-then-put + update_body_pick: it + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + + - command: update loadtest:json_spec + verb: update + noun: loadtest + noun_variant: json_spec + short: "Replace a load test's JSON endpoint spec: harness update loadtest:json_spec -f spec.json" + id_label: "" + handler_type: endpoint + endpoint: + method: PUT + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/json-script + file_body: required + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + no_fields: true + text_header: "\nUpdated the JSON spec for load test {{ctx.id}}\n" + + - command: delete loadtest + verb: delete + noun: loadtest + short: "Delete a load test: harness delete loadtest " + id_label: "" + confirm_mode: prompt + handler_type: endpoint + endpoint: + method: DELETE + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}} + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + no_fields: true + text_header: "\nDeleted load test {{ctx.id}}\n" + + - command: list loadtest:variables + verb: list + noun: loadtest + noun_variant: variables + short: "Show the variables and runtime inputs a load test accepts: harness list loadtest:variables " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest + fields_noun: loadtest_variable + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.parentId}}/variables + items_expr: 'concat(it.inputs ?? [], it.variables ?? [])' + # Variables are not addressable resources — there is no "get one" form. + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + columns: [name, value, type, required, description] + + - command: execute loadtest:sync + verb: execute + noun: loadtest + noun_variant: sync + short: "Re-pull a template-backed load test from its template: harness execute loadtest:sync " + id_label: "" + completion_noun: loadtest + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/sync-template + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + text_header: "\nSynced load test {{ctx.id}} from its template\n" + + - command: execute loadtest + verb: execute + noun: loadtest + short: "Start a load test run: harness execute loadtest [--set targetUsers=50] [--follow]" + id_label: "" + completion_noun: loadtest + fields_noun: loadtest_run + follow_fn: watch + flags_builtin: + set: true + flags: + - name: name + description: Name for this run (defaults to the generated run identity) + - name: follow + description: Stream status and metrics until the run reaches a terminal state + is_bool: true + - name: interval + description: How often to poll while following, such as 5s or 2m (default 10s) + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/runs + # The server wants a client-supplied identity, and overrides as an array not an object. + body_fn: start_run + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + text_footer: "\nWatch it with: harness get loadtest_run {{it.identity}} --follow\n" + + # ------------------------------------------------------------ loadtest_run + + - command: list loadtest_run + verb: list + noun: loadtest_run + short: "List load test runs: harness list loadtest_run [] [--status Running]" + parentid_label: "[]" + completion_noun: loadtest + handler_type: endpoint + flags: + - name: search + description: Filter runs by name + - name: status + description: Filter by run status + completion_values: [Pending, Running, Stopping, Stopped, Finished, Failed] + - name: sort + description: Field to sort by + - name: sort-ascending + description: Sort ascending instead of descending + is_bool: true + endpoint: + # With a load test id the runs come from that test, without one from the whole scope. + path: '{{ctx.parentId != "" ? "/gateway/loadTest/manager/api/v1/load-tests/" + ctx.parentId + "/runs" : "/gateway/loadTest/manager/api/v1/runs"}}' + items_expr: it.items + get_id_expr: it.identity + completion: + id_expr: it.identity + name_expr: it.name + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + search: flags.search + status: flags.status + sortField: flags.sort + sortAscending: 'flags["sort-ascending"] ? "true" : nil' + paging: + paging_strategy: page_index + countable: true + page_index_param: page + page_size_param: limit + page_size_default: 15 + page_size_max: 100 + page_base: 0 + total_expr: it.pagination.totalItems + columns: [identity, loadtest, status, run_sequence, target_users, duration, started_at] + + - command: get loadtest_run + verb: get + noun: loadtest_run + short: "Get a load test run: harness get loadtest_run [--follow]" + id_label: "" + follow_fn: watch + handler_type: endpoint + flags: + - name: follow + description: Stream status and metrics until the run reaches a terminal state + is_bool: true + - name: interval + description: How often to poll while following, such as 5s or 2m (default 10s) + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.id}} + item_expr: it + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + + - command: update loadtest_run + verb: update + noun: loadtest_run + short: "Retarget a running load test: harness update loadtest_run --users 200 [--spawn-rate 5]" + id_label: "" + handler_type: endpoint + flags: + - name: users + description: New target number of virtual users + - name: spawn-rate + description: New spawn rate, in users per second + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.id}}/update + # This endpoint takes the scope in the body, not just the query string. + body_params: + identity: ctx.id + accountIdentifier: auth.account + organizationIdentifier: auth.org + projectIdentifier: auth.project + targetUsers: 'flags.users != "" ? int(flags.users) : nil' + spawnRate: 'flags["spawn-rate"] != "" ? float(flags["spawn-rate"]) : nil' + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + no_fields: true + text_header: "\nUpdated run {{ctx.id}}\n" + + - command: execute loadtest_run:stop + verb: execute + noun: loadtest_run + noun_variant: stop + short: "Stop a running load test: harness execute loadtest_run:stop [--follow]" + id_label: "" + completion_noun: loadtest_run + follow_fn: watch + handler_type: endpoint + flags: + - name: follow + description: Stay attached until the run finishes stopping + is_bool: true + - name: interval + description: How often to poll while following, such as 5s or 2m (default 10s) + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.id}}/stop + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + # The endpoint acknowledges the request; it does not return the run. + no_fields: true + text_header: "\nRequested stop for run {{ctx.id}}\n" + + - command: execute loadtest_run:rerun + verb: execute + noun: loadtest_run + noun_variant: rerun + short: "Run a load test again with the same shape: harness execute loadtest_run:rerun [--follow]" + id_label: "" + completion_noun: loadtest_run + handler_type: workflow + workflow_id: rerun + flags: + - name: follow + description: Stream status and metrics until the new run reaches a terminal state + is_bool: true + - name: interval + description: How often to poll while following, such as 5s or 2m (default 10s) + + - command: get loadtest_run:summary + verb: get + noun: loadtest_run + noun_variant: summary + short: "Show the headline results of a load test run: harness get loadtest_run:summary " + id_label: "" + completion_noun: loadtest_run + fields_noun: loadtest_run_summary + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.id}}/summary + item_expr: it + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + + - command: list loadtest_run:metrics + verb: list + noun: loadtest_run + noun_variant: metrics + short: "Show the metric series for a load test run: harness list loadtest_run:metrics " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_run + fields_noun: loadtest_metric + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.parentId}}/metrics/timeseries + items_expr: it.dataPoints + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + + - command: list loadtest_run:graph + verb: list + noun: loadtest_run + noun_variant: graph + short: "Show the plotting series behind a run chart: harness list loadtest_run:graph " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_run + fields_noun: loadtest_graph_point + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.parentId}}/graph + items_expr: it.dataPoints + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + + - command: list loadtest_run:requests + verb: list + noun: loadtest_run + noun_variant: requests + short: "Show sampled requests from a load test run: harness list loadtest_run:requests " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_run + fields_noun: loadtest_request + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.parentId}}/metrics/scatter + items_expr: it.dataPoints + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + + - command: list loadtest_run:endpoints + verb: list + noun: loadtest_run + noun_variant: endpoints + short: "Break a load test run down by endpoint: harness list loadtest_run:endpoints " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_run + fields_noun: loadtest_endpoint_stat + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/runs/{{ctx.parentId}}/metrics/aggregate + items_expr: it.endpointStats + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + + # --------------------------------------------------------- loadtest_script + + - command: get loadtest_script + verb: get + noun: loadtest_script + short: "Show the script attached to a load test: harness get loadtest_script " + id_label: "" + completion_noun: loadtest + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/script + item_expr: it + # The API returns scriptContent base64-encoded; decode it for display. + text_formatter: format_script + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + + - command: update loadtest_script + verb: update + noun: loadtest_script + short: "Replace the script on a load test: harness update loadtest_script -f test.jmx" + id_label: "" + completion_noun: loadtest + handler_type: endpoint + flags: + # Not file_body: the API stores scripts base64-encoded, so encode_script reads this flag. + - name: file + short: f + description: Script file to upload, or - to read it from stdin + required: true + - name: description + description: Description recorded against the new revision + endpoint: + method: PUT + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/script + # scriptContent must be base64; the raw file is read and encoded in Go. + body_fn: encode_script + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + no_fields: true + text_header: "\nUpdated the script on load test {{ctx.id}}\n" + + - command: list loadtest_script:revisions + verb: list + noun: loadtest_script + noun_variant: revisions + short: "List the script revisions of a load test: harness list loadtest_script:revisions " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest + handler_type: endpoint + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.parentId}}/script/revisions + # A bare array rather than a page, so there is no envelope to read a total from. + items_expr: it + get_id_expr: it.identity + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + paging: + paging_strategy: flat_list + columns: [revision_number, identity, description, is_bundle, created_at, created_by] + + - command: get loadtest_script:revision + verb: get + noun: loadtest_script + noun_variant: revision + short: "Show one script revision: harness get loadtest_script:revision --revision 3" + id_label: "" + completion_noun: loadtest + handler_type: endpoint + flags: + - name: revision + description: Revision number, or the revision identifier + required: true + # The endpoint keys on the identifier, but people think in numbers, so look it up. + flag_resolve_fn: resolve_script_revision + endpoint: + path: /gateway/loadTest/manager/api/v1/load-tests/{{ctx.id}}/script/revisions/{{flags.revision}} + item_expr: it + text_formatter: format_script + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + + # ------------------------------------------------------- loadtest_template + + - command: list loadtest_template + verb: list + noun: loadtest_template + short: "List load test templates: harness list loadtest_template [--hub ] [--tool JMeter|Locust|K6]" + handler_type: endpoint + flags: + - name: hub + description: Narrow to one hub (default spans every hub in the scope) + - name: search + description: Filter templates by name + - name: tool + description: Filter by tool type + completion_values: [JMeter, Locust, K6] + - name: infra-type + description: Filter by infrastructure type + completion_values: [kubernetes, linux] + - name: sort + description: Field to sort by + - name: sort-ascending + description: Sort ascending instead of descending + is_bool: true + endpoint: + path: /gateway/loadTest/manager/api/v1/load-test-templates + items_expr: it.items + get_id_expr: it.identity + completion: + id_expr: it.identity + name_expr: it.name + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + search: flags.search + toolType: flags.tool + infraType: flags["infra-type"] + sortField: flags.sort + sortAscending: 'flags["sort-ascending"] ? "true" : nil' + paging: + paging_strategy: page_index + countable: true + page_index_param: page + page_size_param: limit + page_size_default: 15 + page_size_max: 100 + page_base: 0 + total_expr: it.pagination.totalItems + columns: [identity, name, hub, tool_type, infra_type, revision, updated_at] + + - command: get loadtest_template + verb: get + noun: loadtest_template + short: "Get a load test template: harness get loadtest_template [--hub ]" + id_label: "" + handler_type: endpoint + flags: + - name: hub + description: Hub holding the template + endpoint: + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}} + item_expr: it + yaml_pick_expr: it + yaml_exclude: + - uniqueId + - parentUniqueId + - accountIdentifier + - organizationIdentifier + - projectIdentifier + - createdAt + - createdBy + - updatedAt + - updatedBy + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + + - command: create loadtest_template + verb: create + noun: loadtest_template + requires_id: true + short: "Create a load test template: harness create loadtest_template --set toolType=K6 --set revision=v1, or -f template.yaml" + flags_builtin: + set: true + flags: + - name: hub + description: Hub to create the template in + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-test-templates + file_body: optional + create_strategy: set-fields + create_body_init: + identity: ctx.id + name: coalesce(ctx.setArgs.name, ctx.id) + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + item_expr: it + + - command: update loadtest_template + verb: update + noun: loadtest_template + short: "Update a load test template: harness update loadtest_template --set name=, or -f template.yaml" + id_label: "" + flags_builtin: + set: true + del: true + flags: + - name: hub + description: Hub holding the template + handler_type: endpoint + endpoint: + method: PUT + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}} + file_body: optional + update_strategy: get-then-put + update_body_pick: it + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + item_expr: it + + - command: delete loadtest_template + verb: delete + noun: loadtest_template + short: "Delete a load test template: harness delete loadtest_template [--hub ]" + id_label: "" + confirm_mode: prompt + handler_type: endpoint + flags: + - name: hub + description: Hub holding the template + endpoint: + method: DELETE + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}} + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + no_fields: true + text_header: "\nDeleted load test template {{ctx.id}}\n" + + - command: list loadtest_template:revisions + verb: list + noun: loadtest_template + noun_variant: revisions + short: "List the revisions of a template: harness list loadtest_template:revisions " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_template + handler_type: endpoint + flags: + - name: hub + description: Hub holding the template + endpoint: + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.parentId}}/revisions + # The route answers with a bare array of revisions rather than a page. + items_expr: it + get_id_expr: it.revision + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + paging: + paging_strategy: flat_list + columns: [revision, name, tool_type, infra_type, updated_at, updated_by] + + - command: create loadtest_template:revision + verb: create + noun: loadtest_template + noun_variant: revision + requires_id: true + short: "Add a revision to a template: harness create loadtest_template:revision --revision v2, or -f template.yaml" + completion_noun: loadtest_template + flags_builtin: + set: true + flags: + - name: revision + description: Identifier for the new revision + required: true + - name: hub + description: Hub holding the template + handler_type: endpoint + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}}/revisions + file_body: optional + create_strategy: set-fields + create_body_init: + identity: ctx.id + revision: flags.revision + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + item_expr: it + + - command: get loadtest_template:revision + verb: get + noun: loadtest_template + noun_variant: revision + short: "Get one revision of a template: harness get loadtest_template:revision --revision v2" + id_label: "" + completion_noun: loadtest_template + handler_type: endpoint + flags: + - name: revision + description: Revision to fetch + required: true + - name: hub + description: Hub holding the template + endpoint: + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}}/revisions/{{flags.revision}} + item_expr: it + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + + - command: delete loadtest_template:revision + verb: delete + noun: loadtest_template + noun_variant: revision + short: "Delete one revision of a template: harness delete loadtest_template:revision --revision v2" + id_label: "" + completion_noun: loadtest_template + confirm_mode: prompt + handler_type: endpoint + flags: + - name: revision + description: Revision to delete + required: true + - name: hub + description: Hub holding the template + endpoint: + method: DELETE + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.id}}/revisions/{{flags.revision}} + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + hubIdentity: flags.hub + no_fields: true + text_header: "\nDeleted revision {{flags.revision}} of template {{ctx.id}}\n" + + - command: list loadtest_template:variables + verb: list + noun: loadtest_template + noun_variant: variables + short: "Show the variables and inputs a template accepts: harness list loadtest_template:variables " + requires_parentid: true + parentid_label: "" + completion_noun: loadtest_template + fields_noun: loadtest_variable + handler_type: endpoint + flags: + - name: revision + description: Revision to read (defaults to the latest) + - name: hub + description: Hub holding the template + endpoint: + path: /gateway/loadTest/manager/api/v1/load-test-templates/{{ctx.parentId}}/variables + items_expr: 'concat(it.inputs ?? [], it.variables ?? [])' + get_id_expr: "-" + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + revision: flags.revision + hubIdentity: flags.hub + paging: + paging_strategy: flat_list + columns: [name, value, type, required, description] + + - command: get loadtest_template:yaml + verb: get + noun: loadtest_template + noun_variant: yaml + short: "Export a template as YAML: harness get loadtest_template:yaml [-o template.yaml]" + id_label: "" + completion_noun: loadtest_template + # Answers application/x-yaml, so it is read and written through as raw bytes. + handler_type: workflow + workflow_id: export_template_yaml + flags: + - name: revision + description: Revision to export (defaults to the latest) + - name: hub + description: Hub holding the template + + # ------------------------------------------------------ composite_loadtest + + - command: list composite_loadtest + verb: list + noun: composite_loadtest + short: "List composite load tests: harness list composite_loadtest" + handler_type: endpoint + flags: + - name: search + description: Filter composite RT load tests by name + endpoint: + path: /gateway/loadTest/manager/api/v1/composite-load-tests + items_expr: it.items + get_id_expr: it.pipelineIdentifier + completion: + id_expr: it.pipelineIdentifier + name_expr: it.name + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + search: flags.search + paging: + paging_strategy: page_index + countable: true + page_index_param: page + page_size_param: limit + page_size_default: 15 + page_size_max: 100 + page_base: 0 + total_expr: it.pagination.totalItems + columns: [pipeline_identifier, name, loadtest_count, probe_count, stage_names, last_updated_at] + + - command: create composite_loadtest + verb: create + noun: composite_loadtest + requires_id: true + short: "Create a composite load test: harness create composite_loadtest --loadtest --probe " + handler_type: endpoint + flags: + - name: name + description: Display name (defaults to the identifier) + - name: description + description: Description of the composite RT load test + - name: objective + description: What this composite RT load test is trying to prove + - name: loadtest + description: Identity of the RT load test to run + required: true + completion_noun: loadtest + - name: probe + description: Identity of the chaos probe to run alongside it + required: true + - name: probe-infra + description: Infrastructure reference for the probe (defaults to a runtime input) + - name: probe-duration + description: How long the probe runs, e.g. 10m + endpoint: + method: POST + path: /gateway/loadTest/manager/api/v1/composite-load-tests + # Keys on "identifier", not "identity", and rejects hyphens with an unclear message, + # so the body is built in Go where the identifier is checked before the call goes out. + body_fn: composite_body + query_params: + organizationIdentifier: 'auth.org != "" ? auth.org : nil' + projectIdentifier: 'auth.project != "" ? auth.project : nil' + item_expr: it + no_fields: true + text_header: "\nCreated composite load test {{it.identifier}} (pipeline {{it.pipelineIdentifier}})\n" + + # ---------------------------------------------------------- loadtest_usage + + - command: list loadtest_usage + verb: list + noun: loadtest_usage + short: "Break account load testing usage down by service: harness list loadtest_usage [--from 30d]" + handler_type: endpoint + flags: + - name: from + description: Start of the window — 30d, 2w, 2026-01-01, or unix millis + - name: to + description: End of the window — same formats as --from + endpoint: + path: /gateway/loadTest/manager/api/v1/load-service-usage/details + items_expr: it.services + get_id_expr: it.serviceId + # Account-wide, so no scope params. The window is built in Go: the spec cannot say "now". + query_params_fn: usage_window + paging: + paging_strategy: flat_list + columns: [service, org, project, run_count, peak_users, total_duration_sec, complexity] + + - command: get loadtest_usage + verb: get + noun: loadtest_usage + no_id: true + short: "Show total account load testing usage: harness get loadtest_usage [--from 30d]" + fields_noun: loadtest_usage_total + handler_type: endpoint + flags: + - name: from + description: Start of the window — 30d, 2w, 2026-01-01, or unix millis + - name: to + description: End of the window — same formats as --from + endpoint: + path: /gateway/loadTest/manager/api/v1/load-service-usage/overall + item_expr: it + query_params_fn: usage_window + + - command: get loadtest_usage:report + verb: get + noun: loadtest_usage + noun_variant: report + no_id: true + short: "Export the account usage report: harness get loadtest_usage:report [--from 30d] [--format csv]" + handler_type: endpoint + flags: + - name: from + description: Start of the window — 30d, 2w, 2026-01-01, or unix millis + - name: to + description: End of the window — same formats as --from + endpoint: + path: /gateway/loadTest/manager/api/v1/load-service-usage/report + query_params_fn: usage_window + # A matrix rather than an object, so columns come from the header row at runtime. + item_expr: it + no_fields: true + list_transform_fn: usage_report_rows