From 7aa34f97469ce2da9df72042f9fcaeb645df049e Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 14 May 2026 11:50:46 +0200 Subject: [PATCH 01/16] feat(plots): add fine-tuning metrics plots with ASCII charts and sparklines --- src/together/lib/cli/__init__.py | 3 + .../lib/cli/api/fine_tuning/__init__.py | 0 .../lib/cli/api/fine_tuning/list_metrics.py | 61 ++ .../lib/cli/api/fine_tuning/retrieve.py | 21 + .../lib/cli/utils/plot_finetune_metrics.py | 151 +++++ src/together/lib/cli/utils/plots/__init__.py | 9 + src/together/lib/cli/utils/plots/_engine.py | 590 ++++++++++++++++++ tests/test_plots_engine.py | 322 ++++++++++ uv.lock | 2 +- 9 files changed, 1158 insertions(+), 1 deletion(-) create mode 100644 src/together/lib/cli/api/fine_tuning/__init__.py create mode 100644 src/together/lib/cli/api/fine_tuning/list_metrics.py create mode 100644 src/together/lib/cli/utils/plot_finetune_metrics.py create mode 100644 src/together/lib/cli/utils/plots/__init__.py create mode 100644 src/together/lib/cli/utils/plots/_engine.py create mode 100644 tests/test_plots_engine.py diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index e4dc2931e..9f2609b99 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -380,6 +380,9 @@ async def run_command() -> None: help_epilogue=FINE_TUNING_DOWNLOAD_HELP_EXAMPLES, ) fine_tuning_app.command((f"{_CLI}.fine_tuning.delete:delete"), alias="-d", help="Delete a fine-tuning job") +fine_tuning_app.command( + (f"{_CLI}.fine_tuning.list_metrics:list_metrics"), help="Retrieve training metrics for a fine-tuning job" +) ## Models API commands models_app = app.command(App(name="models", help="List and upload models", help_epilogue=MODELS_HELP_EXAMPLES)) diff --git a/src/together/lib/cli/api/fine_tuning/__init__.py b/src/together/lib/cli/api/fine_tuning/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py new file mode 100644 index 000000000..730b3c655 --- /dev/null +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Annotated +from datetime import datetime + +from cyclopts import Parameter + +from together._types import Omit, omit +from together._utils._json import openapi_dumps +from together.lib.cli.utils.config import CLIConfigParameter +from together.lib.cli.utils._console import console +from together.lib.cli.components.loader import show_loading_status +from together.lib.cli.utils.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_ascii_charts + + +async def list_metrics( + fine_tune_id: Annotated[str, Parameter(help="The ID of the fine-tuning job")], + *, + config: CLIConfigParameter, + global_step_from: Annotated[int | Omit, Parameter(help="Filter metrics from this global step (inclusive).")] = omit, + global_step_to: Annotated[int | Omit, Parameter(help="Filter metrics to this global step (inclusive).")] = omit, + logged_at_from: Annotated[datetime | Omit, Parameter(help="Filter metrics logged at or after this time.")] = omit, + logged_at_to: Annotated[datetime | Omit, Parameter(help="Filter metrics logged at or before this time.")] = omit, + resolution: Annotated[int | Omit, Parameter(help="Number of data points to return (used for JSON output).")] = omit, +) -> None: + """Retrieve training metrics for a fine-tuning job.""" + + if config.json: + response = await show_loading_status( + "Fetching metrics...", + config.client.fine_tuning.list_metrics( + fine_tune_id, + global_step_from=global_step_from, + global_step_to=global_step_to, + logged_at_from=logged_at_from, + logged_at_to=logged_at_to, + resolution=resolution, + ), + ) + console.print_json(openapi_dumps(response.metrics or []).decode("utf-8")) + return + + # For the ASCII chart always fetch at terminal width resolution for best fidelity. + response = await show_loading_status( + "Fetching metrics...", + config.client.fine_tuning.list_metrics( + fine_tune_id, + global_step_from=global_step_from, + global_step_to=global_step_to, + logged_at_from=logged_at_from, + logged_at_to=logged_at_to, + resolution=console.width - METRICS_WIDTH_PADDING, + ), + ) + metrics = response.metrics or [] + + if not metrics: + console.print(f"[muted]No metrics found for job {fine_tune_id}[/muted]") + return + + console.print(metrics_ascii_charts(metrics, width=console.width - METRICS_WIDTH_PADDING)) diff --git a/src/together/lib/cli/api/fine_tuning/retrieve.py b/src/together/lib/cli/api/fine_tuning/retrieve.py index 64bc68378..421e4bc49 100644 --- a/src/together/lib/cli/api/fine_tuning/retrieve.py +++ b/src/together/lib/cli/api/fine_tuning/retrieve.py @@ -1,7 +1,10 @@ from __future__ import annotations +from typing import Annotated from datetime import datetime +from cyclopts import Parameter + from together._utils._json import openapi_dumps from together.lib.cli.api._utils import generate_progress_bar from together.lib.cli.utils.config import CLIConfigParameter @@ -9,6 +12,7 @@ from together.lib.cli.utils._console import console from together.lib.cli.components.loader import show_loading_status from together.lib.cli.components.model_dump import print_model_dump +from together.lib.cli.utils.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_block_sparklines _NEST_INDENT = 4 @@ -17,6 +21,7 @@ async def retrieve( fine_tune_id: str, *, config: CLIConfigParameter, + plots: Annotated[bool, Parameter(help="Print training metric sparklines.")] = True, ) -> None: """Retrieve fine-tuning job details.""" response = await show_loading_status( @@ -35,6 +40,22 @@ async def retrieve( console.print(progress_text) print_model_dump(response, show_nulls=False) + + if plots: + try: + metrics_response = await show_loading_status( + "Fetching metrics...", + config.client.fine_tuning.list_metrics(fine_tune_id, resolution=console.width - METRICS_WIDTH_PADDING), + ) + metrics = metrics_response.metrics or [] + except Exception: + # Metrics are optional; silently skip if unavailable. + metrics = [] + + if metrics: + console.print("\n[muted]Training metrics:[/muted]") + console.print(metrics_block_sparklines(metrics, width=console.width - METRICS_WIDTH_PADDING)) + if event_count > 0: console.print("\n[dim]FT Events:[/dim]") console.print(f" [dim]Total events:[/dim] {event_count}") diff --git a/src/together/lib/cli/utils/plot_finetune_metrics.py b/src/together/lib/cli/utils/plot_finetune_metrics.py new file mode 100644 index 000000000..2278e4181 --- /dev/null +++ b/src/together/lib/cli/utils/plot_finetune_metrics.py @@ -0,0 +1,151 @@ +"""Fine-tuning metrics plotting utilities. + +Public API +---------- +``metrics_block_sparklines(metrics)`` + One ▁▂▃▄▅▆▇█ sparkline line per metric — used in ``retrieve``. + +``metrics_ascii_charts(metrics, height=6)`` + One full ASCII line chart per metric — used in ``list-metrics``. +""" + +from __future__ import annotations + +import math +from typing import Any + +from rich.text import Text + +from together.lib.cli.utils.plots import should_log, render_line_chart, render_sparklines + +# Columns reserved for the y-axis label area, ┼ connector, leading indent, and +# surrounding margin in the ASCII chart layout. This must be >= label_width + 1 +# (the default label_width used in metrics_ascii_charts is 8, so the minimum is +# 9). Callers subtract this from the terminal width to get the usable plot width. +METRICS_WIDTH_PADDING = 48 + +_SKIP_KEYS: frozenset[str] = frozenset({"timestamp", "step", "global_step", "epoch"}) + + +def _is_skip(k: str) -> bool: + base = k.rsplit("/", 1)[-1] + return base in _SKIP_KEYS or base.endswith("_step") or base.endswith("_epoch") + + +def _get_step(row: dict[str, Any], fallback: int) -> int: + """Extract global step, trying several field names before falling back to index.""" + gs = row.get("global_step", row.get("train/global_step", row.get("step"))) + return int(gs) if gs is not None else fallback + + +def _step_label(x: float) -> str: + return str(int(x)) + + +def _collect_series( + metrics: list[dict[str, Any]], +) -> dict[str, tuple[list[float], list[float]]]: + """Collect plottable numeric series from a list of metric dicts. + + Returns a mapping of name → (xs, ys). Keys are discovered in insertion + order; step/epoch/timestamp fields are skipped. NaN values are converted + to ``-inf`` so the rendering engine plots them at the very bottom of the + chart rather than silently dropping them. + """ + series: dict[str, tuple[list[float], list[float]]] = {} + for i, row in enumerate(metrics): + step = float(_get_step(row, fallback=i)) + for k, v in row.items(): + if _is_skip(k) or isinstance(v, bool) or not isinstance(v, (int, float)): + continue + val = float(v) + # NaN is rendered as a dip to the bottom (-inf sentinel). + if math.isnan(val): + val = float("-inf") + if k not in series: + series[k] = ([], []) + series[k][0].append(step) + series[k][1].append(val) + return series + + +def _no_data() -> Text: + t = Text() + t.append("No plottable metrics found.", style="muted") + return t + + +def metrics_block_sparklines( + metrics: list[dict[str, Any]], + *, + width: int = 60, +) -> Text: + """One block-sparkline line per metric, coloured with the CLI theme. + + Args: + metrics: List of flat metric dicts (one per training step). + width: Sparkline character width (default 60). + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + series = _collect_series(metrics) + if not series: + return _no_data() + label_w = max(len(k) for k in series) + text = Text() + for key, (xs, ys) in series.items(): + text.append_text( + render_sparklines( + key, + xs, + ys, + width=width, + y_log=should_log(ys), + label_width=label_w, + ) + ) + return text + + +def metrics_ascii_charts( + metrics: list[dict[str, Any]], + *, + height: int = 6, + width: int = 60, + label_width: int = 8, +) -> Text: + """One ASCII line chart per metric, with a global-step x-axis. + + Args: + metrics: List of flat metric dicts (one per training step). + height: Chart body height in rows (default 6). + width: Plot character width (default 60). + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + series = _collect_series(metrics) + text = Text() + for key, (xs, ys) in series.items(): + if text: + text.append("\n") + text.append_text( + render_line_chart( + xs, + {key: ys}, + x_label=_step_label, + y_log=should_log(ys), + height=height, + width=width, + label_width=label_width, + ) + ) + return text if text else _no_data() + + +__all__ = [ + "metrics_block_sparklines", + "metrics_ascii_charts", + "METRICS_WIDTH_PADDING", +] diff --git a/src/together/lib/cli/utils/plots/__init__.py b/src/together/lib/cli/utils/plots/__init__.py new file mode 100644 index 000000000..c2cfd857f --- /dev/null +++ b/src/together/lib/cli/utils/plots/__init__.py @@ -0,0 +1,9 @@ +"""Generic CLI plot utilities.""" + +from together.lib.cli.utils.plots._engine import should_log, render_line_chart, render_sparklines + +__all__ = [ + "render_line_chart", + "render_sparklines", + "should_log", +] diff --git a/src/together/lib/cli/utils/plots/_engine.py b/src/together/lib/cli/utils/plots/_engine.py new file mode 100644 index 000000000..21a9c0c69 --- /dev/null +++ b/src/together/lib/cli/utils/plots/_engine.py @@ -0,0 +1,590 @@ +"""ASCII sparkline and chart engine for time-series data. + +Designed for scalar time-series (loss, accuracy, …); not a general-purpose +plotting library. + +Internal pipeline (``_plot``, ``_interpolate``, …) uses a shared x-grid with +named y series: ``xs: list[float]`` + ``ys: dict[str, list[float]]``. + +Public API +---------- +``render_line_chart(xs, ys, ...)`` + One or more named series plotted on a shared ASCII line chart. All series + share the same x-axis and y-scale. + +``render_sparklines(name, xs, ys, ...)`` + A single block-sparkline row (▁▂▃▄▅▆▇█). Call once per series and pass a + shared ``label_width`` across calls for consistent label alignment. Names + are right-justified; those that exceed ``label_width`` are truncated with + ``...``. +""" + +from __future__ import annotations + +import math +import bisect +from typing import Callable + +from rich.text import Text + +_SPARK_BLOCKS = " ▁▂▃▄▅▆▇█" + +# Styles cycled across series in insertion order. +_SERIES_STYLES = ["white", "green", "yellow", "cyan", "magenta"] + +# UI style tokens used throughout the rendering pipeline. +_STYLE_PRIMARY = "primary" # default plot body text +_STYLE_SECONDARY = "secondary" # axis labels and tick text +_STYLE_ACCENT = "accent" # axis border characters (┼ └ ┬ …) +_STYLE_MUTED = "muted" # series name labels and empty-state messages +_STYLE_SPARK = "white" # sparkline bar characters + +# Sentinels used in quantized_ys to signal out-of-range non-finite values. +# Both are outside the valid slot range [0, height-1]. +_NEG_INF_SENTINEL = -1 # -inf: line descends to the x-axis border +_POS_INF_SENTINEL = -2 # +inf: line ascends to the top data row +_NAN_SENTINEL = -3 # NaN: no line at the place + + +def should_log(vals: list[float]) -> bool: + """Return True when values span more than 100×, suggesting log scale.""" + nz = [v for v in vals if v > 0] + return len(nz) > 1 and (max(nz) / min(nz)) > 100 + + +def _uniform_grid(vals: list[float], n: int) -> list[float]: + """Return n evenly-spaced points spanning [min(vals), max(vals)]. + + Non-finite values (e.g. the -inf sentinel used for NaN data points) are + excluded from the range computation so they don't corrupt the grid. + """ + finite = [v for v in vals if math.isfinite(v)] + min_val, max_val = min(finite), max(finite) + if n <= 1: + return [min_val] + return [min_val + (max_val - min_val) * idx / (n - 1) for idx in range(n)] + + +def _interpolate( + xs: list[float], + ys: dict[str, list[float]], + x_grid: list[float], +) -> dict[str, list[float]]: + """Linearly interpolate each named y series onto x_grid; clamp at the edges. + + For each grid point: + - If it falls before the first data point, use the first y value. + - If it falls after the last data point, use the last y value. + - Otherwise, linearly interpolate between the two bracketing data points. + """ + results: dict[str, list[float]] = {} + for name, yvals in ys.items(): + # Sort by x, using insertion order as a tiebreaker so that duplicate + # steps are resolved deterministically (first occurrence wins). + pairs = sorted(enumerate(zip(xs, yvals)), key=lambda t: (t[1][0], t[0])) + xs_s = [x for _, (x, _y) in pairs] + ys_s = [y for _, (_x, y) in pairs] + + interpolated: list[float] = [] + for x_point in x_grid: + pos = bisect.bisect_left(xs_s, x_point) + if pos == 0: + interpolated.append(ys_s[0]) + elif pos == len(xs_s): + interpolated.append(ys_s[-1]) + elif xs_s[pos] == x_point: + interpolated.append(ys_s[pos]) + else: + left_x, left_y = xs_s[pos - 1], ys_s[pos - 1] + right_x, right_y = xs_s[pos], ys_s[pos] + # When either bracket endpoint is the -inf NaN sentinel we + # cannot compute a meaningful slope. Instead, assign this + # grid point to whichever bracket is closer: if that bracket + # is non-finite the dip extends to this column; if it is + # finite we use its value so the dip stays as narrow as the + # grid resolution allows. + if not math.isfinite(left_y) or not math.isfinite(right_y): + closer_y = left_y if (x_point - left_x) <= (right_x - x_point) else right_y + interpolated.append(closer_y) + else: + slope = (right_y - left_y) / (right_x - left_x) + interpolated.append(left_y + slope * (x_point - left_x)) + + results[name] = interpolated + return results + + +def _log_transform( + named_values: dict[str, list[float]], +) -> dict[str, list[float]]: + """Return new traces with ys replaced by their log10 values.""" + result: dict[str, list[float]] = {} + for name, values in named_values.items(): + nz = [value for value in values if value > 0] + eps = min(nz) * 0.01 if nz else 1e-10 + result[name] = [math.log10(max(value, eps)) for value in values] + return result + + +def _quantize_ys( + interpolated_ys: dict[str, list[float]], + y_grid: list[float], +) -> list[list[int]]: + """Snap each interpolated y value to the index of the nearest y_grid slot. + + Non-finite values are mapped to out-of-band sentinels: + + * ``_POS_INF_SENTINEL`` (``-1``) for ``+inf`` — the line spikes to the top + data row. + * ``_NEG_INF_SENTINEL`` (``-2``) for ``-inf`` / ``NaN`` — the line + descends to the x-axis border row. + """ + quantized_ys: list[list[int]] = [] + for ys in interpolated_ys.values(): + row: list[int] = [] + for y in ys: + if math.isfinite(y): + row.append(min(range(len(y_grid)), key=lambda i: abs(y_grid[i] - y))) + elif y > 0: # +inf + row.append(_POS_INF_SENTINEL) + elif math.isinf(y): + row.append(_NEG_INF_SENTINEL) + else: # -inf or NaN (NaN > 0 is False) + row.append(_NAN_SENTINEL) + quantized_ys.append(row) + return quantized_ys + + +def _fit_spark_label(name: str, label_width: int) -> str: + """Right-justify *name* in *label_width* chars, truncating with '...' if needed.""" + if len(name) <= label_width: + return name.rjust(label_width) + return name[: max(0, label_width - 3)] + "..." + + +def _y_labels( + y_grid: list[float], + y_log: bool, + y_label: Callable[[float], str], +) -> list[str]: + """Build y-axis tick label strings from the y grid.""" + labels = [y_label(10**y) if y_log else y_label(y) for y in y_grid[::-1]] + return labels + + +def _x_labels( + x_grid: list[float], + n_xticks: int, + x_label: Callable[[float], str], +) -> list[tuple[int, str]]: + """Return (column_index, label_string) pairs for each x-axis tick.""" + width = len(x_grid) + x_min = x_grid[0] + # Extend by one grid step beyond the last point so the rightmost tick + # label shows the true data maximum. round() suppresses floating-point + # noise that would otherwise accumulate in the tick value calculations. + x_max = round(x_grid[-1] + ((x_grid[-1] - x_grid[0]) / (width - 1) if width > 1 else 0.0), 10) + if n_xticks < 2 or width <= 1: + return [(0, x_label(x_min))] + tick_cols = [round(i * (width - 1) / (n_xticks - 1)) for i in range(n_xticks)] + tick_vals = [x_min + (x_max - x_min) * i / (n_xticks - 1) for i in range(n_xticks)] + return [(col, x_label(val)) for col, val in zip(tick_cols, tick_vals)] + + +def _draw_y_axis( + grid: list[list[str]], + style_grid: list[list[str]], + labels: list[str], + label_w: int, +) -> None: + """Fill y-axis labels and ┼ connectors into the grid.""" + for label, grid_row, style_row in zip(labels, grid, style_grid): + if len(label) > label_w: + label = label[: max(0, label_w - 3)] + "..." + label = label.rjust(label_w) + for ci, ch in enumerate(label): + grid_row[ci] = ch + style_row[ci] = _STYLE_SECONDARY + grid_row[label_w] = "┼" + style_row[label_w] = _STYLE_ACCENT + + +def _draw_lines( + grid: list[list[str]], + style_grid: list[list[str]], + quantized_ys: list[list[int]], + styles: list[str], + label_w: int, +) -> frozenset[int]: + """Draw all series into the shared grid (last writer wins on collision). + + Coordinate system: y_grid index 0 is the *bottom* of the data range, but + grid row 0 is the *top* of the terminal output. The conversion is: + screen_row = len(grid) - y_grid_index - 1 + So a higher y_grid index means a higher data value and a *lower* screen row. + + Out-of-band sentinels (``_NEG_INF_SENTINEL``, ``_POS_INF_SENTINEL``) signal + non-finite source values: + + * ``_NEG_INF_SENTINEL`` (-inf / NaN): line descends to the x-axis border. + The set of affected plot-body column indices is returned so + ``_draw_x_axis`` can mark them with ``┴``. + * ``_POS_INF_SENTINEL`` (+inf): line spikes to the top data row (row 0). + """ + height = len(grid) + border_cols: set[int] = set() + offset = label_w + 1 + width = len(grid[0]) + for style, pv in zip(styles, quantized_ys): + # We look one column ahead (pv[col+1]), so stop one short of the end. + for col_idx in range(width - label_w - 2): + cur = pv[col_idx] + nxt = pv[col_idx + 1] + col = col_idx + offset + + cur_is_neg_inf = cur == _NEG_INF_SENTINEL + nxt_is_neg_inf = nxt == _NEG_INF_SENTINEL + cur_is_pos_inf = cur == _POS_INF_SENTINEL + nxt_is_pos_inf = nxt == _POS_INF_SENTINEL + cur_is_nan = cur == _NAN_SENTINEL + nxt_is_nan = nxt == _NAN_SENTINEL + + # Two consecutive non-finite points of the same kind: nothing to draw. + if ( + (cur_is_neg_inf and nxt_is_neg_inf) + or (cur_is_pos_inf and nxt_is_pos_inf) + or (cur_is_nan and nxt_is_nan) + ): + continue + + screen_row = height - cur - 1 + next_screen_row = height - nxt - 1 + + # Recovering from border: │ up from bottom data row to nxt. + if cur_is_neg_inf: + border_cols.add(col_idx) + grid[next_screen_row][col] = "╭" + style_grid[next_screen_row][col] = style + for mid_row in range(next_screen_row + 1, height): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + continue + + # Descending to border: │ down from cur to bottom data row. + if nxt_is_neg_inf: + border_cols.add(col_idx) + grid[screen_row][col] = "╮" + style_grid[screen_row][col] = style + for mid_row in range(screen_row + 1, height): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + continue + + # Descending from top: │ down from row 0 to nxt. + if cur_is_pos_inf: + grid[0][col] = "│" + style_grid[0][col] = style + for mid_row in range(1, next_screen_row): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + grid[next_screen_row][col] = "╰" + style_grid[next_screen_row][col] = style + continue + + # Ascending to top: │ up from cur to row 0. + if nxt_is_pos_inf: + grid[screen_row][col] = "╯" + style_grid[screen_row][col] = style + for mid_row in range(1, screen_row): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + grid[0][col] = "│" + style_grid[0][col] = style + continue + + # Continue previous line if the next one is NaN + if not cur_is_nan and nxt_is_nan: + grid[screen_row][col] = "─" + continue + + # Start a new line if the current one is nan, but the previous one is not + if cur_is_nan and not nxt_is_nan: + grid[next_screen_row][col] = "─" + continue + + # If everything is finite and good, compare the values and add horizontal line or increasing/decreasing line + if screen_row == next_screen_row: + grid[screen_row][col] = "─" + style_grid[screen_row][col] = style + continue + + going_down = cur > nxt # value decreases → line goes down on screen + grid[screen_row][col] = "╮" if going_down else "╯" + style_grid[screen_row][col] = style + grid[next_screen_row][col] = "╰" if going_down else "╭" + style_grid[next_screen_row][col] = style + for mid_row in range(min(screen_row, next_screen_row) + 1, max(screen_row, next_screen_row)): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + + return frozenset(border_cols) + + +def _draw_x_axis( + grid: list[list[str]], + style_grid: list[list[str]], + label_w: int, + x_labels: list[tuple[int, str]], + nan_cols: frozenset[int] = frozenset(), +) -> None: + """Append the └───┬─── border row and tick label row to the grid. + + ``nan_cols`` is a set of plot-body column indices (0-based within the plot + body, i.e. not including the y-axis label area) where a NaN line descends + to the border. Those positions get ``┴`` instead of ``─``, or ``┼`` when + they coincide with an x-tick ``┬``. + """ + row_len = len(grid[0]) + width = row_len - label_w - 1 + + # Border row: spaces | └ | ─ … ┬ … ─ + tick_cols = {col for col, _ in x_labels} + border_chars = list("─" * width) + for col in tick_cols: + border_chars[col] = "┬" + + # Adding hitting lines to -inf to the border + for col in nan_cols: + if 0 <= col < width: + border_chars[col] = "┼" if col in tick_cols else "┴" + border_row = [" "] * label_w + ["└"] + border_chars + border_styles = [_STYLE_SECONDARY] * label_w + [_STYLE_ACCENT] + [_STYLE_ACCENT] * width + grid.append(border_row) + style_grid.append(border_styles) + + # Label row: tick strings centred under their tick column + label_row = [" "] * row_len + for col, lbl in x_labels: + start = label_w + 1 + col - len(lbl) // 2 + start = max(0, min(start, row_len - len(lbl))) + for i, ch in enumerate(lbl): + label_row[start + i] = ch + grid.append(label_row) + style_grid.append([_STYLE_SECONDARY] * row_len) + + +def _render_data_row( + row: list[str], + style_row: list[str], +) -> Text: + """Colorize one grid row, appending each character with its style.""" + text = Text() + for ch, style in zip(row, style_row): + text.append(ch, style=style) + text.append("\n") + return text + + +def _render_body( + grid: list[list[str]], + style_grid: list[list[str]], +) -> Text: + """Convert the finished grid into a Rich Text object.""" + text = Text() + for row, style_row in zip(grid, style_grid): + text.append_text(_render_data_row(row, style_row)) + return text + + +def _plot( + xs: list[float], + ys: dict[str, list[float]], + *, + width: int = 60, + height: int = 6, + x_label: Callable[[float], str] = str, + y_label: Callable[[float], str] = str, + y_log: bool = False, + n_xticks: int = 3, + label_width: int = 8, +) -> Text: + """Render one or more named y series against a shared x-axis as an ASCII chart. + + Args: + xs: Shared x values for all series. + ys: Mapping of name → y values (must be same length as xs). + width: Number of character columns in the plot body. + height: Number of character rows in the chart body. + x_label: Callable that formats an x value into a tick-label string. + y_label: Callable that formats a y value into a tick-label string. + y_log: When True, values are plotted on a log10 axis. + n_xticks: Number of tick marks and labels on the x-axis (default 3). + label_width: Cap on the y-axis label column width (default 8). + Labels longer than this are truncated with ``...``. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not ys: + t = Text() + t.append("No data.", style=_STYLE_MUTED) + return t + + ordered_styles = [_SERIES_STYLES[i % len(_SERIES_STYLES)] for i in range(len(ys))] + + x_grid = _uniform_grid(xs, width) + interpolated_ys = _interpolate(xs, ys, x_grid) + if y_log: + interpolated_ys = _log_transform(interpolated_ys) + flat_ys = [v for ys_list in interpolated_ys.values() for v in ys_list] + y_grid = _uniform_grid(flat_ys, height) + + quantized_ys = _quantize_ys(interpolated_ys, y_grid) + y_labels = _y_labels(y_grid, y_log, y_label) + x_labels = _x_labels(x_grid, n_xticks, x_label) + + grid: list[list[str]] = [[" "] * (width + label_width + 1) for _ in range(height)] + style_grid: list[list[str]] = [[_STYLE_PRIMARY] * (width + label_width + 1) for _ in range(height)] + + _draw_y_axis(grid, style_grid, y_labels, label_width) + nan_cols = _draw_lines(grid, style_grid, quantized_ys, ordered_styles, label_width) + _draw_x_axis(grid, style_grid, label_width, x_labels, nan_cols) + + text = _render_body(grid, style_grid) + return text + + +def render_sparklines( + name: str, + xs: list[float], + ys: list[float], + *, + width: int = 60, + y_log: bool = False, + label_width: int = 8, +) -> Text: + """Render a single block-sparkline row for one series. + + Call once per series, passing a shared ``label_width`` across all calls to + keep label columns aligned. The name is right-justified within the column; + names longer than ``label_width`` are truncated with ``...``. + + Args: + name: Series name, used as the row label. + xs: X values (e.g. training steps). + ys: Y values. + width: Sparkline character width (default 60). + y_log: When True, plot on a log10 scale (default False). + label_width: Exact label column width (default 8). Pass the same + value to every call in a group to get consistent + alignment. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not xs: + t = Text() + t.append("No plottable data.", style=_STYLE_MUTED) + return t + + x_grid = _uniform_grid(xs, width) + interpolated = _interpolate(xs, {name: ys}, x_grid) + if y_log: + interpolated = _log_transform(interpolated) + + series_vals = interpolated[name] + y_grid = _uniform_grid(series_vals, len(_SPARK_BLOCKS)) + quantized = _quantize_ys({name: series_vals}, y_grid)[0] + + label = _fit_spark_label(name, label_width) + + # The sentinel value (len(y_grid)) indicates a NaN data point; render it + # as a space (the lowest sparkline block) since sparklines have no border row. + # Map out-of-band sentinels to the extreme sparkline blocks: + # _NEG_INF_SENTINEL (-inf) or _NAN_SENTINEL (NaN) → space (lowest block, index 0) + # _POS_INF_SENTINEL (+inf) → █ (highest block, last index) + def _spark_block(idx: int) -> str: + if idx == _NEG_INF_SENTINEL or idx == _NAN_SENTINEL: + return _SPARK_BLOCKS[0] + if idx == _POS_INF_SENTINEL: + return _SPARK_BLOCKS[-1] + return _SPARK_BLOCKS[idx] + + spark = "".join(_spark_block(idx) for idx in quantized).ljust(width) + + text = Text() + text.append(f" {label} ", style=_STYLE_MUTED) + text.append(spark, style=_STYLE_SPARK) + text.append(f" {ys[0]:.4g} → {ys[-1]:.4g}", style=_STYLE_SECONDARY) + text.append("\n") + return text + + +def render_line_chart( + xs: list[float], + ys: dict[str, list[float]], + *, + x_label: Callable[[float], str] = str, + y_log: bool = False, + y_label: Callable[[float], str] | None = None, + width: int = 60, + height: int = 6, + n_xticks: int = 3, + label_width: int = 8, +) -> Text: + """Render one or more named series as a shared ASCII line chart with a legend header. + + All series share the same x-axis (``xs``); each has its own named y values:: + + console.print( + render_line_chart( + steps, + {"train_loss": train_losses, "val_loss": val_losses}, + x_label=lambda s: f"step {s:.0f}", + ) + ) + + Args: + xs: Shared x values for all series. + ys: Mapping of name → y values. + x_label: Callable that formats an x value into a tick-label string. + y_log: When True, plot on a log10 y-axis (default False). + y_label: Callable that formats a y value into a tick-label string. + width: Plot width in terminal characters (default 60). + height: Plot height in terminal rows (default 6). + n_xticks: Number of x-axis tick marks and labels (default 3). + label_width: Cap on the y-axis label column width. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not ys: + t = Text() + t.append("No plottable data.", style=_STYLE_MUTED) + return t + + styles = {key: _SERIES_STYLES[i % len(_SERIES_STYLES)] for i, key in enumerate(ys)} + + text = Text() + x_from = x_label(xs[0]) + x_to = x_label(xs[-1]) + for key in ys: + text.append( + f" {key} ({x_from} – {x_to}) {ys[key][0]:.4g} → {ys[key][-1]:.4g}\n", + style=styles[key], + ) + + text.append_text( + _plot( + xs, + ys, + width=width, + height=height, + x_label=x_label, + y_label=y_label or (lambda v: f"{v:.3g}"), + y_log=y_log, + n_xticks=n_xticks, + label_width=label_width, + ) + ) + return text diff --git a/tests/test_plots_engine.py b/tests/test_plots_engine.py new file mode 100644 index 000000000..aa64ee6d1 --- /dev/null +++ b/tests/test_plots_engine.py @@ -0,0 +1,322 @@ +from __future__ import annotations + +import pytest + +from together.lib.cli.utils.plots._engine import ( + _interpolate, + _uniform_grid, + render_line_chart, + render_sparklines, +) + + +def constant_series(n: int = 5, value: float = 1.0) -> list[tuple[float, float]]: + return [(float(i), value) for i in range(n)] + + +# Shared deterministic series used by golden-output tests +_LOSS = [(float(i), 1.0 - i * 0.1) for i in range(10)] # 1.0 → 0.1 +_ACCURACY = [(float(i), 0.5 + i * 0.05) for i in range(10)] # 0.5 → 0.95 +_WIDE = [(float(i), 10.0**i) for i in range(5)] # 1, 10, 100, 1000, 10000 + +_LOSS_XS = [p[0] for p in _LOSS] +_LOSS_YS = [p[1] for p in _LOSS] +_ACCURACY_XS = [p[0] for p in _ACCURACY] +_ACCURACY_YS = [p[1] for p in _ACCURACY] +_WIDE_XS = [p[0] for p in _WIDE] +_WIDE_YS = [p[1] for p in _WIDE] + + +def _x_label(x: float) -> str: + return str(int(x)) + + +def _interp(xs: list[float], ys: list[float], x_grid: list[float]) -> list[float]: + """Helper: interpolate a single series onto x_grid.""" + return _interpolate(xs, {"s": ys}, x_grid)["s"] + + +class TestInterpolate: + def test_output_length_equals_grid(self) -> None: + xs = [float(i) for i in range(10)] + ys = [float(i) for i in range(10)] + x_grid = _uniform_grid(xs, 5) + result = _interp(xs, ys, x_grid) + assert len(result) == 5 + + def test_linear_data_interpolates_exactly(self) -> None: + xs = [0.0, 9.0] + ys = [0.0, 9.0] + x_grid = _uniform_grid(xs, 10) + result = _interp(xs, ys, x_grid) + # grid points are 0.0, 0.9, 1.8, ..., 8.1 — y=x so values match + assert result == pytest.approx(x_grid, abs=1e-9) # type: ignore[misc] + + def test_constant_series_stays_constant(self) -> None: + xs = [float(i) for i in range(20)] + ys = [7.0] * 20 + x_grid = _uniform_grid(xs, 10) + result = _interp(xs, ys, x_grid) + assert result == pytest.approx([7.0] * 10, abs=1e-9) # type: ignore[misc] + + def test_left_clamp(self) -> None: + xs = [5.0, 9.0] + ys = [99.0, 99.0] + x_grid = _uniform_grid([0.0, 9.0], 10) + result = _interp(xs, ys, x_grid) + assert result == [99.0] * 10 + + def test_right_clamp(self) -> None: + xs = [0.0, 2.0] + ys = [42.0, 42.0] + x_grid = _uniform_grid([0.0, 9.0], 10) + result = _interp(xs, ys, x_grid) + assert result == [42.0] * 10 + + def test_single_point_fills_all(self) -> None: + xs = [5.0] + ys = [3.14] + x_grid = _uniform_grid([0.0, 9.0], 8) + result = _interp(xs, ys, x_grid) + assert result == [3.14] * 8 + + def test_uniform_grid_length(self) -> None: + assert len(_uniform_grid([0.0, 10.0], 5)) == 5 + + def test_uniform_grid_endpoints(self) -> None: + grid = _uniform_grid([0.0, 9.0], 10) + assert grid[0] == pytest.approx(0.0) # type: ignore[misc] + assert grid[-1] == pytest.approx(9.0) # type: ignore[misc] + + +class TestRenderSparklines: + def test_empty_series_returns_no_data_message(self) -> None: + result = render_sparklines("loss", [], [], width=20) + assert result.plain == "No plottable data." + + def test_single_series_golden(self) -> None: + result = render_sparklines("loss", _LOSS_XS, _LOSS_YS, width=20) + assert result.plain == " loss ██▇▇▆▆▅▅▅▄▄▃▃▃▂▂▁▁ 1 → 0.1\n" + + def test_constant_series_golden(self) -> None: + _flat = constant_series(10, 5.0) + result = render_sparklines("flat", [p[0] for p in _flat], [p[1] for p in _flat], width=20) + assert result.plain == " flat 5 → 5\n" + + def test_single_point_golden(self) -> None: + result = render_sparklines("single", [0.0], [1.0], width=20) + assert result.plain == " single 1 → 1\n" + + def test_log_scale_golden(self) -> None: + result = render_sparklines("wide", _WIDE_XS, _WIDE_YS, width=20, y_log=True) + assert result.plain == " wide ▁▁▂▂▂▃▃▄▄▅▅▆▆▆▇▇███ 1 → 1e+04\n" # leading space = first sparkline block + + def test_label_width_truncates_with_ellipsis(self) -> None: + result = render_sparklines("verylongname", _LOSS_XS, _LOSS_YS, width=20, label_width=6) + # "verylongname" (12 chars) truncated to label_width=6: "ver..." + assert result.plain.startswith(" ver... ") + + def test_label_width_truncates_long_name_aligned(self) -> None: + # A name longer than label_width is truncated with ..., staying aligned + r1 = render_sparklines("loss", _LOSS_XS, _LOSS_YS, width=20, label_width=8) + r2 = render_sparklines("averylongmetricname", _LOSS_XS, _LOSS_YS, width=20, label_width=8) + assert r1.plain == " loss ██▇▇▆▆▅▅▅▄▄▃▃▃▂▂▁▁ 1 → 0.1\n" # right-justified + assert r2.plain == " avery... ██▇▇▆▆▅▅▅▄▄▃▃▃▂▂▁▁ 1 → 0.1\n" # truncated to 8 + + def test_aligned_across_calls(self) -> None: + # Pass the same label_width to both calls → sparklines start at the same column + shared_w = 8 + r1 = render_sparklines("loss", _LOSS_XS, _LOSS_YS, width=20, label_width=shared_w) + r2 = render_sparklines("accuracy", _ACCURACY_XS, _ACCURACY_YS, width=20, label_width=shared_w) + assert r1.plain == " loss ██▇▇▆▆▅▅▅▄▄▃▃▃▂▂▁▁ 1 → 0.1\n" # "loss" right-justified in 8 + assert r2.plain == " accuracy ▁▁▂▂▃▃▃▄▄▅▅▅▆▆▇▇██ 0.5 → 0.95\n" # "accuracy" fills 8 exactly + + @pytest.mark.parametrize( + "bad_value, expected", + [ + (float("-inf"), " loss ██▇▇▆▆▅▅▅▄ ▃▃▂▂▁▁ 1 → 0.1\n"), + (float("nan"), " loss ██▇▇▆▆▅▅▅▄ ▃▃▂▂▁▁ 1 → 0.1\n"), + (float("inf"), " loss ██▇▇▆▆▅▅▅▄██▃▃▂▂▁▁ 1 → 0.1\n"), + ], + ids=["neg_inf", "nan", "pos_inf"], + ) + def test_non_finite_rendered_as_extreme_block_golden(self, bad_value: float, expected: str) -> None: + # -inf/NaN → blank (bottom) block; +inf → █ (top) block. + xs = [float(i) for i in range(10)] + ys = [(1.0 - i * 0.1) if i != 5 else bad_value for i in range(10)] + result = render_sparklines("loss", xs, ys, width=20) + assert result.plain == expected + + +class TestRenderLineChart: + def test_empty_series_returns_no_data_message(self) -> None: + result = render_line_chart([], {}) + assert result.plain == "No plottable data." + + def test_single_series_golden(self) -> None: + result = render_line_chart( + _LOSS_XS, + {"loss": _LOSS_YS}, + width=20, + height=4, + n_xticks=3, + x_label=_x_label, + ) + assert result.plain == ( + " loss (0 – 9) 1 → 0.1\n" + " 1┼───╮ \n" + " 0.7┼ ╰─────╮ \n" + " 0.4┼ ╰─────╮ \n" + " 0.1┼ ╰─── \n" + " └┬─────────┬────────┬\n" + " 0 4 9\n" + ) + + def test_multi_series_golden(self) -> None: + # loss and accuracy share the same x-axis (steps 0–9) + result = render_line_chart( + _LOSS_XS, + {"loss": _LOSS_YS, "accuracy": _ACCURACY_YS}, + width=20, + height=4, + n_xticks=3, + x_label=_x_label, + ) + assert result.plain == ( + " loss (0 – 9) 1 → 0.1\n" + " accuracy (0 – 9) 0.5 → 0.95\n" + " 1┼───╮ ╭──── \n" + " 0.7┼ ╭───────────╯ \n" + " 0.4┼──╯ ╰─────╮ \n" + " 0.1┼ ╰─── \n" + " └┬─────────┬────────┬\n" + " 0 4 9\n" + ) + + def test_log_scale_golden(self) -> None: + result = render_line_chart( + _WIDE_XS, + {"metric": _WIDE_YS}, + width=20, + height=4, + n_xticks=3, + x_label=_x_label, + y_log=True, + ) + assert result.plain == ( + " metric (0 – 4) 1 → 1e+04\n" + " 1e+04┼ ╭──── \n" + " 464┼ ╭────╯ \n" + " 21.5┼ ╭───────╯ \n" + " 1┼─╯ \n" + " └┬─────────┬────────┬\n" + " 0 2 4\n" + ) + + def test_constant_series_golden(self) -> None: + _flat = constant_series(10, 42.0) + result = render_line_chart( + [p[0] for p in _flat], + {"flat": [p[1] for p in _flat]}, + width=20, + height=4, + x_label=_x_label, + ) + assert result.plain == ( + " flat (0 – 9) 42 → 42\n" + " 42┼ \n" + " 42┼ \n" + " 42┼ \n" + " 42┼─────────────────── \n" + " └┬─────────┬────────┬\n" + " 0 4 9\n" + ) + + def test_custom_x_label_golden(self) -> None: + result = render_line_chart( + _LOSS_XS, + {"m": _LOSS_YS}, + width=20, + height=4, + n_xticks=3, + x_label=lambda x: f"step{int(x)}", + ) + assert result.plain == ( + " m (step0 – step9) 1 → 0.1\n" + " 1┼───╮ \n" + " 0.7┼ ╰─────╮ \n" + " 0.4┼ ╰─────╮ \n" + " 0.1┼ ╰─── \n" + " └┬─────────┬────────┬\n" + " step0 step4 step9\n" + ) + + @pytest.mark.parametrize( + "bad_value, expected", + [ + ( + float("-inf"), + ( + " loss (0 – 9) 1 → 0.1\n" + " 1┼───╮ \n" + " 0.7┼ ╰─────╮ \n" + " 0.4┼ │ ╭───╮ \n" + " 0.1┼ │ │ ╰─── \n" + " └┬────────┴┬┴───────┬\n" + " 0 4 9\n" + ), + ), + ( + float("nan"), + ( + " loss (0 – 9) 1 → 0.1\n" + " 1┼───╮ \n" + " 0.7┼ ╰────── \n" + " 0.4┼ ────╮ \n" + " 0.1┼ ╰─── \n" + " └┬─────────┬────────┬\n" + " 0 4 9\n" + ), + ), + ( + float("inf"), + ( + " loss (0 – 9) 1 → 0.1\n" + " 1┼───╮ │ │ \n" + " 0.7┼ ╰─────╯ │ \n" + " 0.4┼ ╰───╮ \n" + " 0.1┼ ╰─── \n" + " └┬─────────┬────────┬\n" + " 0 4 9\n" + ), + ), + ], + ids=["neg_inf", "nan", "pos_inf"], + ) + def test_non_finite_rendered_as_extreme_golden(self, bad_value: float, expected: str) -> None: + # -inf/NaN → dip to x-axis border; +inf → spike to top data row. + xs = [float(i) for i in range(10)] + ys = [(1.0 - i * 0.1) if i != 5 else bad_value for i in range(10)] + result = render_line_chart(xs, {"loss": ys}, width=20, height=4, n_xticks=3, x_label=_x_label) + assert result.plain == expected + + def test_label_width_caps_y_axis(self) -> None: + # "1e+04" is exactly 5 chars; label_width=5 fits it without truncation + result = render_line_chart( + _WIDE_XS, + {"metric": _WIDE_YS}, + width=20, + height=4, + x_label=_x_label, + y_log=True, + label_width=5, + ) + assert result.plain == ( + " metric (0 – 4) 1 → 1e+04\n" + "1e+04┼ ╭──── \n" + " 464┼ ╭────╯ \n" + " 21.5┼ ╭───────╯ \n" + " 1┼─╯ \n" + " └┬─────────┬────────┬\n" + " 0 2 4\n" + ) diff --git a/uv.lock b/uv.lock index a10f2d912..abe93dbc1 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "together" -version = "2.12.0" +version = "2.14.0" source = { editable = "." } dependencies = [ { name = "anyio" }, From 950baf710e878e75c74e22cdbe5863667010b716 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 14 May 2026 14:07:06 +0200 Subject: [PATCH 02/16] fix comment --- src/together/lib/cli/utils/plots/_engine.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/together/lib/cli/utils/plots/_engine.py b/src/together/lib/cli/utils/plots/_engine.py index 21a9c0c69..c3f9f1ab4 100644 --- a/src/together/lib/cli/utils/plots/_engine.py +++ b/src/together/lib/cli/utils/plots/_engine.py @@ -97,12 +97,12 @@ def _interpolate( else: left_x, left_y = xs_s[pos - 1], ys_s[pos - 1] right_x, right_y = xs_s[pos], ys_s[pos] - # When either bracket endpoint is the -inf NaN sentinel we - # cannot compute a meaningful slope. Instead, assign this - # grid point to whichever bracket is closer: if that bracket - # is non-finite the dip extends to this column; if it is - # finite we use its value so the dip stays as narrow as the - # grid resolution allows. + # When either bracket endpoint is a non-finite sentinel + # (-inf/NaN or +inf) we cannot compute a meaningful slope. + # Instead, assign this grid point to whichever bracket is + # closer: if that bracket is non-finite the spike/dip extends + # to this column; if it is finite we use its value so the + # spike/dip stays as narrow as the grid resolution allows. if not math.isfinite(left_y) or not math.isfinite(right_y): closer_y = left_y if (x_point - left_x) <= (right_x - x_point) else right_y interpolated.append(closer_y) From 37259dbf62462dbc3c030c67d6b93025f4710ef8 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Thu, 14 May 2026 14:09:44 +0200 Subject: [PATCH 03/16] revert --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index abe93dbc1..a10f2d912 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "together" -version = "2.14.0" +version = "2.12.0" source = { editable = "." } dependencies = [ { name = "anyio" }, From d753f70904c10a5f7a5cebad6abc2abdf713588f Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 09:53:54 +0200 Subject: [PATCH 04/16] first pack of fixes --- src/together/lib/cli/__init__.py | 5 +- .../lib/cli/api/fine_tuning/__init__.py | 0 .../lib/cli/api/fine_tuning/list_metrics.py | 37 +- .../lib/cli/api/fine_tuning/retrieve.py | 20 +- src/together/lib/cli/utils/_help_examples.py | 14 + .../lib/cli/utils/plot_finetune_metrics.py | 151 ----- src/together/lib/cli/utils/plots/__init__.py | 9 - src/together/lib/cli/utils/plots/_engine.py | 590 ------------------ tests/test_plots_engine.py | 19 +- uv.lock | 2 +- 10 files changed, 48 insertions(+), 799 deletions(-) delete mode 100644 src/together/lib/cli/api/fine_tuning/__init__.py delete mode 100644 src/together/lib/cli/utils/plot_finetune_metrics.py delete mode 100644 src/together/lib/cli/utils/plots/__init__.py delete mode 100644 src/together/lib/cli/utils/plots/_engine.py diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index 9f2609b99..467cde712 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -53,6 +53,7 @@ JIG_SECRETS_UNSET_HELP_EXAMPLES, ENDPOINTS_HARDWARE_HELP_EXAMPLES, FINE_TUNING_CREATE_HELP_EXAMPLES, + FINE_TUNING_LIST_METRICS_HELP_EXAMPLES, JIG_SECRETS_DELETE_HELP_EXAMPLES, JIG_VOLUMES_CREATE_HELP_EXAMPLES, JIG_VOLUMES_UPDATE_HELP_EXAMPLES, @@ -381,7 +382,9 @@ async def run_command() -> None: ) fine_tuning_app.command((f"{_CLI}.fine_tuning.delete:delete"), alias="-d", help="Delete a fine-tuning job") fine_tuning_app.command( - (f"{_CLI}.fine_tuning.list_metrics:list_metrics"), help="Retrieve training metrics for a fine-tuning job" + (f"{_CLI}.fine_tuning.list_metrics:list_metrics"), + help="Retrieve training metrics for a fine-tuning job", + help_epilogue=FINE_TUNING_LIST_METRICS_HELP_EXAMPLES, ) ## Models API commands diff --git a/src/together/lib/cli/api/fine_tuning/__init__.py b/src/together/lib/cli/api/fine_tuning/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 730b3c655..be37118c1 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -5,42 +5,26 @@ from cyclopts import Parameter -from together._types import Omit, omit from together._utils._json import openapi_dumps from together.lib.cli.utils.config import CLIConfigParameter from together.lib.cli.utils._console import console from together.lib.cli.components.loader import show_loading_status -from together.lib.cli.utils.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_ascii_charts +from together.lib.cli.components.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_ascii_charts async def list_metrics( fine_tune_id: Annotated[str, Parameter(help="The ID of the fine-tuning job")], *, config: CLIConfigParameter, - global_step_from: Annotated[int | Omit, Parameter(help="Filter metrics from this global step (inclusive).")] = omit, - global_step_to: Annotated[int | Omit, Parameter(help="Filter metrics to this global step (inclusive).")] = omit, - logged_at_from: Annotated[datetime | Omit, Parameter(help="Filter metrics logged at or after this time.")] = omit, - logged_at_to: Annotated[datetime | Omit, Parameter(help="Filter metrics logged at or before this time.")] = omit, - resolution: Annotated[int | Omit, Parameter(help="Number of data points to return (used for JSON output).")] = omit, + global_step_from: Annotated[int | None, Parameter(help="Filter metrics from this global step (inclusive).")] = None, + global_step_to: Annotated[int | None, Parameter(help="Filter metrics to this global step (inclusive).")] = None, + logged_at_from: Annotated[datetime | None, Parameter(help="Filter metrics logged at or after this time.")] = None, + logged_at_to: Annotated[datetime | None, Parameter(help="Filter metrics logged at or before this time.")] = None, + resolution: Annotated[int | None, Parameter(help="Number of training metric points to return. Does not limit the number of eval metric points.")] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" - if config.json: - response = await show_loading_status( - "Fetching metrics...", - config.client.fine_tuning.list_metrics( - fine_tune_id, - global_step_from=global_step_from, - global_step_to=global_step_to, - logged_at_from=logged_at_from, - logged_at_to=logged_at_to, - resolution=resolution, - ), - ) - console.print_json(openapi_dumps(response.metrics or []).decode("utf-8")) - return - - # For the ASCII chart always fetch at terminal width resolution for best fidelity. + resolution_value = console.width - METRICS_WIDTH_PADDING if not config.json else resolution response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( @@ -49,9 +33,14 @@ async def list_metrics( global_step_to=global_step_to, logged_at_from=logged_at_from, logged_at_to=logged_at_to, - resolution=console.width - METRICS_WIDTH_PADDING, + resolution=resolution_value, ), ) + + if config.json: + console.print_json(openapi_dumps(response.metrics or []).decode("utf-8")) + return + metrics = response.metrics or [] if not metrics: diff --git a/src/together/lib/cli/api/fine_tuning/retrieve.py b/src/together/lib/cli/api/fine_tuning/retrieve.py index 421e4bc49..dcfe80192 100644 --- a/src/together/lib/cli/api/fine_tuning/retrieve.py +++ b/src/together/lib/cli/api/fine_tuning/retrieve.py @@ -12,7 +12,7 @@ from together.lib.cli.utils._console import console from together.lib.cli.components.loader import show_loading_status from together.lib.cli.components.model_dump import print_model_dump -from together.lib.cli.utils.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_block_sparklines +from together.lib.cli.components.plot_finetune_metrics import METRICS_WIDTH_PADDING, metrics_block_sparklines _NEST_INDENT = 4 @@ -21,7 +21,7 @@ async def retrieve( fine_tune_id: str, *, config: CLIConfigParameter, - plots: Annotated[bool, Parameter(help="Print training metric sparklines.")] = True, + no_plots: Annotated[bool, Parameter(help="Print training metric sparklines.")] = False, ) -> None: """Retrieve fine-tuning job details.""" response = await show_loading_status( @@ -41,16 +41,12 @@ async def retrieve( print_model_dump(response, show_nulls=False) - if plots: - try: - metrics_response = await show_loading_status( - "Fetching metrics...", - config.client.fine_tuning.list_metrics(fine_tune_id, resolution=console.width - METRICS_WIDTH_PADDING), - ) - metrics = metrics_response.metrics or [] - except Exception: - # Metrics are optional; silently skip if unavailable. - metrics = [] + if not no_plots: + metrics_response = await show_loading_status( + "Fetching metrics...", + config.client.fine_tuning.list_metrics(fine_tune_id, resolution=console.width - METRICS_WIDTH_PADDING), + ) + metrics = metrics_response.metrics or [] if metrics: console.print("\n[muted]Training metrics:[/muted]") diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index 224eee8a4..619957328 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -100,6 +100,20 @@ [primary]tg ft create --n-checkpoints 3 -M Qwen/Qwen2-1.5B --training-file ./my-dataset.jsonl[/primary] """ +FINE_TUNING_LIST_METRICS_HELP_EXAMPLES = """[dim]Examples:[/dim] +[dim]-[/dim] Retrieve metrics for a fine-tuning job: + [primary]tg ft list-metrics [/primary] + +[dim]-[/dim] Retrieve metrics from a specific global step range: + [primary]tg ft list-metrics --global-step-from 100 --global-step-to 500[/primary] + +[dim]-[/dim] Retrieve metrics logged within a time range: + [primary]tg ft list-metrics --logged-at-from 2024-01-01T00:00:00 --logged-at-to 2024-01-02T00:00:00[/primary] + +[dim]-[/dim] Retrieve a fixed number of data points as JSON: + [primary]tg ft list-metrics --resolution 50 --json[/primary] +""" + FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] [dim]-[/dim] Download a fine-tuned model's weights: [primary]tg ft download --output-dir ./my-model[/primary] diff --git a/src/together/lib/cli/utils/plot_finetune_metrics.py b/src/together/lib/cli/utils/plot_finetune_metrics.py deleted file mode 100644 index 2278e4181..000000000 --- a/src/together/lib/cli/utils/plot_finetune_metrics.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Fine-tuning metrics plotting utilities. - -Public API ----------- -``metrics_block_sparklines(metrics)`` - One ▁▂▃▄▅▆▇█ sparkline line per metric — used in ``retrieve``. - -``metrics_ascii_charts(metrics, height=6)`` - One full ASCII line chart per metric — used in ``list-metrics``. -""" - -from __future__ import annotations - -import math -from typing import Any - -from rich.text import Text - -from together.lib.cli.utils.plots import should_log, render_line_chart, render_sparklines - -# Columns reserved for the y-axis label area, ┼ connector, leading indent, and -# surrounding margin in the ASCII chart layout. This must be >= label_width + 1 -# (the default label_width used in metrics_ascii_charts is 8, so the minimum is -# 9). Callers subtract this from the terminal width to get the usable plot width. -METRICS_WIDTH_PADDING = 48 - -_SKIP_KEYS: frozenset[str] = frozenset({"timestamp", "step", "global_step", "epoch"}) - - -def _is_skip(k: str) -> bool: - base = k.rsplit("/", 1)[-1] - return base in _SKIP_KEYS or base.endswith("_step") or base.endswith("_epoch") - - -def _get_step(row: dict[str, Any], fallback: int) -> int: - """Extract global step, trying several field names before falling back to index.""" - gs = row.get("global_step", row.get("train/global_step", row.get("step"))) - return int(gs) if gs is not None else fallback - - -def _step_label(x: float) -> str: - return str(int(x)) - - -def _collect_series( - metrics: list[dict[str, Any]], -) -> dict[str, tuple[list[float], list[float]]]: - """Collect plottable numeric series from a list of metric dicts. - - Returns a mapping of name → (xs, ys). Keys are discovered in insertion - order; step/epoch/timestamp fields are skipped. NaN values are converted - to ``-inf`` so the rendering engine plots them at the very bottom of the - chart rather than silently dropping them. - """ - series: dict[str, tuple[list[float], list[float]]] = {} - for i, row in enumerate(metrics): - step = float(_get_step(row, fallback=i)) - for k, v in row.items(): - if _is_skip(k) or isinstance(v, bool) or not isinstance(v, (int, float)): - continue - val = float(v) - # NaN is rendered as a dip to the bottom (-inf sentinel). - if math.isnan(val): - val = float("-inf") - if k not in series: - series[k] = ([], []) - series[k][0].append(step) - series[k][1].append(val) - return series - - -def _no_data() -> Text: - t = Text() - t.append("No plottable metrics found.", style="muted") - return t - - -def metrics_block_sparklines( - metrics: list[dict[str, Any]], - *, - width: int = 60, -) -> Text: - """One block-sparkline line per metric, coloured with the CLI theme. - - Args: - metrics: List of flat metric dicts (one per training step). - width: Sparkline character width (default 60). - - Returns: - A ``rich.text.Text`` ready for ``console.print()``. - """ - series = _collect_series(metrics) - if not series: - return _no_data() - label_w = max(len(k) for k in series) - text = Text() - for key, (xs, ys) in series.items(): - text.append_text( - render_sparklines( - key, - xs, - ys, - width=width, - y_log=should_log(ys), - label_width=label_w, - ) - ) - return text - - -def metrics_ascii_charts( - metrics: list[dict[str, Any]], - *, - height: int = 6, - width: int = 60, - label_width: int = 8, -) -> Text: - """One ASCII line chart per metric, with a global-step x-axis. - - Args: - metrics: List of flat metric dicts (one per training step). - height: Chart body height in rows (default 6). - width: Plot character width (default 60). - - Returns: - A ``rich.text.Text`` ready for ``console.print()``. - """ - series = _collect_series(metrics) - text = Text() - for key, (xs, ys) in series.items(): - if text: - text.append("\n") - text.append_text( - render_line_chart( - xs, - {key: ys}, - x_label=_step_label, - y_log=should_log(ys), - height=height, - width=width, - label_width=label_width, - ) - ) - return text if text else _no_data() - - -__all__ = [ - "metrics_block_sparklines", - "metrics_ascii_charts", - "METRICS_WIDTH_PADDING", -] diff --git a/src/together/lib/cli/utils/plots/__init__.py b/src/together/lib/cli/utils/plots/__init__.py deleted file mode 100644 index c2cfd857f..000000000 --- a/src/together/lib/cli/utils/plots/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Generic CLI plot utilities.""" - -from together.lib.cli.utils.plots._engine import should_log, render_line_chart, render_sparklines - -__all__ = [ - "render_line_chart", - "render_sparklines", - "should_log", -] diff --git a/src/together/lib/cli/utils/plots/_engine.py b/src/together/lib/cli/utils/plots/_engine.py deleted file mode 100644 index c3f9f1ab4..000000000 --- a/src/together/lib/cli/utils/plots/_engine.py +++ /dev/null @@ -1,590 +0,0 @@ -"""ASCII sparkline and chart engine for time-series data. - -Designed for scalar time-series (loss, accuracy, …); not a general-purpose -plotting library. - -Internal pipeline (``_plot``, ``_interpolate``, …) uses a shared x-grid with -named y series: ``xs: list[float]`` + ``ys: dict[str, list[float]]``. - -Public API ----------- -``render_line_chart(xs, ys, ...)`` - One or more named series plotted on a shared ASCII line chart. All series - share the same x-axis and y-scale. - -``render_sparklines(name, xs, ys, ...)`` - A single block-sparkline row (▁▂▃▄▅▆▇█). Call once per series and pass a - shared ``label_width`` across calls for consistent label alignment. Names - are right-justified; those that exceed ``label_width`` are truncated with - ``...``. -""" - -from __future__ import annotations - -import math -import bisect -from typing import Callable - -from rich.text import Text - -_SPARK_BLOCKS = " ▁▂▃▄▅▆▇█" - -# Styles cycled across series in insertion order. -_SERIES_STYLES = ["white", "green", "yellow", "cyan", "magenta"] - -# UI style tokens used throughout the rendering pipeline. -_STYLE_PRIMARY = "primary" # default plot body text -_STYLE_SECONDARY = "secondary" # axis labels and tick text -_STYLE_ACCENT = "accent" # axis border characters (┼ └ ┬ …) -_STYLE_MUTED = "muted" # series name labels and empty-state messages -_STYLE_SPARK = "white" # sparkline bar characters - -# Sentinels used in quantized_ys to signal out-of-range non-finite values. -# Both are outside the valid slot range [0, height-1]. -_NEG_INF_SENTINEL = -1 # -inf: line descends to the x-axis border -_POS_INF_SENTINEL = -2 # +inf: line ascends to the top data row -_NAN_SENTINEL = -3 # NaN: no line at the place - - -def should_log(vals: list[float]) -> bool: - """Return True when values span more than 100×, suggesting log scale.""" - nz = [v for v in vals if v > 0] - return len(nz) > 1 and (max(nz) / min(nz)) > 100 - - -def _uniform_grid(vals: list[float], n: int) -> list[float]: - """Return n evenly-spaced points spanning [min(vals), max(vals)]. - - Non-finite values (e.g. the -inf sentinel used for NaN data points) are - excluded from the range computation so they don't corrupt the grid. - """ - finite = [v for v in vals if math.isfinite(v)] - min_val, max_val = min(finite), max(finite) - if n <= 1: - return [min_val] - return [min_val + (max_val - min_val) * idx / (n - 1) for idx in range(n)] - - -def _interpolate( - xs: list[float], - ys: dict[str, list[float]], - x_grid: list[float], -) -> dict[str, list[float]]: - """Linearly interpolate each named y series onto x_grid; clamp at the edges. - - For each grid point: - - If it falls before the first data point, use the first y value. - - If it falls after the last data point, use the last y value. - - Otherwise, linearly interpolate between the two bracketing data points. - """ - results: dict[str, list[float]] = {} - for name, yvals in ys.items(): - # Sort by x, using insertion order as a tiebreaker so that duplicate - # steps are resolved deterministically (first occurrence wins). - pairs = sorted(enumerate(zip(xs, yvals)), key=lambda t: (t[1][0], t[0])) - xs_s = [x for _, (x, _y) in pairs] - ys_s = [y for _, (_x, y) in pairs] - - interpolated: list[float] = [] - for x_point in x_grid: - pos = bisect.bisect_left(xs_s, x_point) - if pos == 0: - interpolated.append(ys_s[0]) - elif pos == len(xs_s): - interpolated.append(ys_s[-1]) - elif xs_s[pos] == x_point: - interpolated.append(ys_s[pos]) - else: - left_x, left_y = xs_s[pos - 1], ys_s[pos - 1] - right_x, right_y = xs_s[pos], ys_s[pos] - # When either bracket endpoint is a non-finite sentinel - # (-inf/NaN or +inf) we cannot compute a meaningful slope. - # Instead, assign this grid point to whichever bracket is - # closer: if that bracket is non-finite the spike/dip extends - # to this column; if it is finite we use its value so the - # spike/dip stays as narrow as the grid resolution allows. - if not math.isfinite(left_y) or not math.isfinite(right_y): - closer_y = left_y if (x_point - left_x) <= (right_x - x_point) else right_y - interpolated.append(closer_y) - else: - slope = (right_y - left_y) / (right_x - left_x) - interpolated.append(left_y + slope * (x_point - left_x)) - - results[name] = interpolated - return results - - -def _log_transform( - named_values: dict[str, list[float]], -) -> dict[str, list[float]]: - """Return new traces with ys replaced by their log10 values.""" - result: dict[str, list[float]] = {} - for name, values in named_values.items(): - nz = [value for value in values if value > 0] - eps = min(nz) * 0.01 if nz else 1e-10 - result[name] = [math.log10(max(value, eps)) for value in values] - return result - - -def _quantize_ys( - interpolated_ys: dict[str, list[float]], - y_grid: list[float], -) -> list[list[int]]: - """Snap each interpolated y value to the index of the nearest y_grid slot. - - Non-finite values are mapped to out-of-band sentinels: - - * ``_POS_INF_SENTINEL`` (``-1``) for ``+inf`` — the line spikes to the top - data row. - * ``_NEG_INF_SENTINEL`` (``-2``) for ``-inf`` / ``NaN`` — the line - descends to the x-axis border row. - """ - quantized_ys: list[list[int]] = [] - for ys in interpolated_ys.values(): - row: list[int] = [] - for y in ys: - if math.isfinite(y): - row.append(min(range(len(y_grid)), key=lambda i: abs(y_grid[i] - y))) - elif y > 0: # +inf - row.append(_POS_INF_SENTINEL) - elif math.isinf(y): - row.append(_NEG_INF_SENTINEL) - else: # -inf or NaN (NaN > 0 is False) - row.append(_NAN_SENTINEL) - quantized_ys.append(row) - return quantized_ys - - -def _fit_spark_label(name: str, label_width: int) -> str: - """Right-justify *name* in *label_width* chars, truncating with '...' if needed.""" - if len(name) <= label_width: - return name.rjust(label_width) - return name[: max(0, label_width - 3)] + "..." - - -def _y_labels( - y_grid: list[float], - y_log: bool, - y_label: Callable[[float], str], -) -> list[str]: - """Build y-axis tick label strings from the y grid.""" - labels = [y_label(10**y) if y_log else y_label(y) for y in y_grid[::-1]] - return labels - - -def _x_labels( - x_grid: list[float], - n_xticks: int, - x_label: Callable[[float], str], -) -> list[tuple[int, str]]: - """Return (column_index, label_string) pairs for each x-axis tick.""" - width = len(x_grid) - x_min = x_grid[0] - # Extend by one grid step beyond the last point so the rightmost tick - # label shows the true data maximum. round() suppresses floating-point - # noise that would otherwise accumulate in the tick value calculations. - x_max = round(x_grid[-1] + ((x_grid[-1] - x_grid[0]) / (width - 1) if width > 1 else 0.0), 10) - if n_xticks < 2 or width <= 1: - return [(0, x_label(x_min))] - tick_cols = [round(i * (width - 1) / (n_xticks - 1)) for i in range(n_xticks)] - tick_vals = [x_min + (x_max - x_min) * i / (n_xticks - 1) for i in range(n_xticks)] - return [(col, x_label(val)) for col, val in zip(tick_cols, tick_vals)] - - -def _draw_y_axis( - grid: list[list[str]], - style_grid: list[list[str]], - labels: list[str], - label_w: int, -) -> None: - """Fill y-axis labels and ┼ connectors into the grid.""" - for label, grid_row, style_row in zip(labels, grid, style_grid): - if len(label) > label_w: - label = label[: max(0, label_w - 3)] + "..." - label = label.rjust(label_w) - for ci, ch in enumerate(label): - grid_row[ci] = ch - style_row[ci] = _STYLE_SECONDARY - grid_row[label_w] = "┼" - style_row[label_w] = _STYLE_ACCENT - - -def _draw_lines( - grid: list[list[str]], - style_grid: list[list[str]], - quantized_ys: list[list[int]], - styles: list[str], - label_w: int, -) -> frozenset[int]: - """Draw all series into the shared grid (last writer wins on collision). - - Coordinate system: y_grid index 0 is the *bottom* of the data range, but - grid row 0 is the *top* of the terminal output. The conversion is: - screen_row = len(grid) - y_grid_index - 1 - So a higher y_grid index means a higher data value and a *lower* screen row. - - Out-of-band sentinels (``_NEG_INF_SENTINEL``, ``_POS_INF_SENTINEL``) signal - non-finite source values: - - * ``_NEG_INF_SENTINEL`` (-inf / NaN): line descends to the x-axis border. - The set of affected plot-body column indices is returned so - ``_draw_x_axis`` can mark them with ``┴``. - * ``_POS_INF_SENTINEL`` (+inf): line spikes to the top data row (row 0). - """ - height = len(grid) - border_cols: set[int] = set() - offset = label_w + 1 - width = len(grid[0]) - for style, pv in zip(styles, quantized_ys): - # We look one column ahead (pv[col+1]), so stop one short of the end. - for col_idx in range(width - label_w - 2): - cur = pv[col_idx] - nxt = pv[col_idx + 1] - col = col_idx + offset - - cur_is_neg_inf = cur == _NEG_INF_SENTINEL - nxt_is_neg_inf = nxt == _NEG_INF_SENTINEL - cur_is_pos_inf = cur == _POS_INF_SENTINEL - nxt_is_pos_inf = nxt == _POS_INF_SENTINEL - cur_is_nan = cur == _NAN_SENTINEL - nxt_is_nan = nxt == _NAN_SENTINEL - - # Two consecutive non-finite points of the same kind: nothing to draw. - if ( - (cur_is_neg_inf and nxt_is_neg_inf) - or (cur_is_pos_inf and nxt_is_pos_inf) - or (cur_is_nan and nxt_is_nan) - ): - continue - - screen_row = height - cur - 1 - next_screen_row = height - nxt - 1 - - # Recovering from border: │ up from bottom data row to nxt. - if cur_is_neg_inf: - border_cols.add(col_idx) - grid[next_screen_row][col] = "╭" - style_grid[next_screen_row][col] = style - for mid_row in range(next_screen_row + 1, height): - grid[mid_row][col] = "│" - style_grid[mid_row][col] = style - continue - - # Descending to border: │ down from cur to bottom data row. - if nxt_is_neg_inf: - border_cols.add(col_idx) - grid[screen_row][col] = "╮" - style_grid[screen_row][col] = style - for mid_row in range(screen_row + 1, height): - grid[mid_row][col] = "│" - style_grid[mid_row][col] = style - continue - - # Descending from top: │ down from row 0 to nxt. - if cur_is_pos_inf: - grid[0][col] = "│" - style_grid[0][col] = style - for mid_row in range(1, next_screen_row): - grid[mid_row][col] = "│" - style_grid[mid_row][col] = style - grid[next_screen_row][col] = "╰" - style_grid[next_screen_row][col] = style - continue - - # Ascending to top: │ up from cur to row 0. - if nxt_is_pos_inf: - grid[screen_row][col] = "╯" - style_grid[screen_row][col] = style - for mid_row in range(1, screen_row): - grid[mid_row][col] = "│" - style_grid[mid_row][col] = style - grid[0][col] = "│" - style_grid[0][col] = style - continue - - # Continue previous line if the next one is NaN - if not cur_is_nan and nxt_is_nan: - grid[screen_row][col] = "─" - continue - - # Start a new line if the current one is nan, but the previous one is not - if cur_is_nan and not nxt_is_nan: - grid[next_screen_row][col] = "─" - continue - - # If everything is finite and good, compare the values and add horizontal line or increasing/decreasing line - if screen_row == next_screen_row: - grid[screen_row][col] = "─" - style_grid[screen_row][col] = style - continue - - going_down = cur > nxt # value decreases → line goes down on screen - grid[screen_row][col] = "╮" if going_down else "╯" - style_grid[screen_row][col] = style - grid[next_screen_row][col] = "╰" if going_down else "╭" - style_grid[next_screen_row][col] = style - for mid_row in range(min(screen_row, next_screen_row) + 1, max(screen_row, next_screen_row)): - grid[mid_row][col] = "│" - style_grid[mid_row][col] = style - - return frozenset(border_cols) - - -def _draw_x_axis( - grid: list[list[str]], - style_grid: list[list[str]], - label_w: int, - x_labels: list[tuple[int, str]], - nan_cols: frozenset[int] = frozenset(), -) -> None: - """Append the └───┬─── border row and tick label row to the grid. - - ``nan_cols`` is a set of plot-body column indices (0-based within the plot - body, i.e. not including the y-axis label area) where a NaN line descends - to the border. Those positions get ``┴`` instead of ``─``, or ``┼`` when - they coincide with an x-tick ``┬``. - """ - row_len = len(grid[0]) - width = row_len - label_w - 1 - - # Border row: spaces | └ | ─ … ┬ … ─ - tick_cols = {col for col, _ in x_labels} - border_chars = list("─" * width) - for col in tick_cols: - border_chars[col] = "┬" - - # Adding hitting lines to -inf to the border - for col in nan_cols: - if 0 <= col < width: - border_chars[col] = "┼" if col in tick_cols else "┴" - border_row = [" "] * label_w + ["└"] + border_chars - border_styles = [_STYLE_SECONDARY] * label_w + [_STYLE_ACCENT] + [_STYLE_ACCENT] * width - grid.append(border_row) - style_grid.append(border_styles) - - # Label row: tick strings centred under their tick column - label_row = [" "] * row_len - for col, lbl in x_labels: - start = label_w + 1 + col - len(lbl) // 2 - start = max(0, min(start, row_len - len(lbl))) - for i, ch in enumerate(lbl): - label_row[start + i] = ch - grid.append(label_row) - style_grid.append([_STYLE_SECONDARY] * row_len) - - -def _render_data_row( - row: list[str], - style_row: list[str], -) -> Text: - """Colorize one grid row, appending each character with its style.""" - text = Text() - for ch, style in zip(row, style_row): - text.append(ch, style=style) - text.append("\n") - return text - - -def _render_body( - grid: list[list[str]], - style_grid: list[list[str]], -) -> Text: - """Convert the finished grid into a Rich Text object.""" - text = Text() - for row, style_row in zip(grid, style_grid): - text.append_text(_render_data_row(row, style_row)) - return text - - -def _plot( - xs: list[float], - ys: dict[str, list[float]], - *, - width: int = 60, - height: int = 6, - x_label: Callable[[float], str] = str, - y_label: Callable[[float], str] = str, - y_log: bool = False, - n_xticks: int = 3, - label_width: int = 8, -) -> Text: - """Render one or more named y series against a shared x-axis as an ASCII chart. - - Args: - xs: Shared x values for all series. - ys: Mapping of name → y values (must be same length as xs). - width: Number of character columns in the plot body. - height: Number of character rows in the chart body. - x_label: Callable that formats an x value into a tick-label string. - y_label: Callable that formats a y value into a tick-label string. - y_log: When True, values are plotted on a log10 axis. - n_xticks: Number of tick marks and labels on the x-axis (default 3). - label_width: Cap on the y-axis label column width (default 8). - Labels longer than this are truncated with ``...``. - - Returns: - A ``rich.text.Text`` ready for ``console.print()``. - """ - if not ys: - t = Text() - t.append("No data.", style=_STYLE_MUTED) - return t - - ordered_styles = [_SERIES_STYLES[i % len(_SERIES_STYLES)] for i in range(len(ys))] - - x_grid = _uniform_grid(xs, width) - interpolated_ys = _interpolate(xs, ys, x_grid) - if y_log: - interpolated_ys = _log_transform(interpolated_ys) - flat_ys = [v for ys_list in interpolated_ys.values() for v in ys_list] - y_grid = _uniform_grid(flat_ys, height) - - quantized_ys = _quantize_ys(interpolated_ys, y_grid) - y_labels = _y_labels(y_grid, y_log, y_label) - x_labels = _x_labels(x_grid, n_xticks, x_label) - - grid: list[list[str]] = [[" "] * (width + label_width + 1) for _ in range(height)] - style_grid: list[list[str]] = [[_STYLE_PRIMARY] * (width + label_width + 1) for _ in range(height)] - - _draw_y_axis(grid, style_grid, y_labels, label_width) - nan_cols = _draw_lines(grid, style_grid, quantized_ys, ordered_styles, label_width) - _draw_x_axis(grid, style_grid, label_width, x_labels, nan_cols) - - text = _render_body(grid, style_grid) - return text - - -def render_sparklines( - name: str, - xs: list[float], - ys: list[float], - *, - width: int = 60, - y_log: bool = False, - label_width: int = 8, -) -> Text: - """Render a single block-sparkline row for one series. - - Call once per series, passing a shared ``label_width`` across all calls to - keep label columns aligned. The name is right-justified within the column; - names longer than ``label_width`` are truncated with ``...``. - - Args: - name: Series name, used as the row label. - xs: X values (e.g. training steps). - ys: Y values. - width: Sparkline character width (default 60). - y_log: When True, plot on a log10 scale (default False). - label_width: Exact label column width (default 8). Pass the same - value to every call in a group to get consistent - alignment. - - Returns: - A ``rich.text.Text`` ready for ``console.print()``. - """ - if not xs: - t = Text() - t.append("No plottable data.", style=_STYLE_MUTED) - return t - - x_grid = _uniform_grid(xs, width) - interpolated = _interpolate(xs, {name: ys}, x_grid) - if y_log: - interpolated = _log_transform(interpolated) - - series_vals = interpolated[name] - y_grid = _uniform_grid(series_vals, len(_SPARK_BLOCKS)) - quantized = _quantize_ys({name: series_vals}, y_grid)[0] - - label = _fit_spark_label(name, label_width) - - # The sentinel value (len(y_grid)) indicates a NaN data point; render it - # as a space (the lowest sparkline block) since sparklines have no border row. - # Map out-of-band sentinels to the extreme sparkline blocks: - # _NEG_INF_SENTINEL (-inf) or _NAN_SENTINEL (NaN) → space (lowest block, index 0) - # _POS_INF_SENTINEL (+inf) → █ (highest block, last index) - def _spark_block(idx: int) -> str: - if idx == _NEG_INF_SENTINEL or idx == _NAN_SENTINEL: - return _SPARK_BLOCKS[0] - if idx == _POS_INF_SENTINEL: - return _SPARK_BLOCKS[-1] - return _SPARK_BLOCKS[idx] - - spark = "".join(_spark_block(idx) for idx in quantized).ljust(width) - - text = Text() - text.append(f" {label} ", style=_STYLE_MUTED) - text.append(spark, style=_STYLE_SPARK) - text.append(f" {ys[0]:.4g} → {ys[-1]:.4g}", style=_STYLE_SECONDARY) - text.append("\n") - return text - - -def render_line_chart( - xs: list[float], - ys: dict[str, list[float]], - *, - x_label: Callable[[float], str] = str, - y_log: bool = False, - y_label: Callable[[float], str] | None = None, - width: int = 60, - height: int = 6, - n_xticks: int = 3, - label_width: int = 8, -) -> Text: - """Render one or more named series as a shared ASCII line chart with a legend header. - - All series share the same x-axis (``xs``); each has its own named y values:: - - console.print( - render_line_chart( - steps, - {"train_loss": train_losses, "val_loss": val_losses}, - x_label=lambda s: f"step {s:.0f}", - ) - ) - - Args: - xs: Shared x values for all series. - ys: Mapping of name → y values. - x_label: Callable that formats an x value into a tick-label string. - y_log: When True, plot on a log10 y-axis (default False). - y_label: Callable that formats a y value into a tick-label string. - width: Plot width in terminal characters (default 60). - height: Plot height in terminal rows (default 6). - n_xticks: Number of x-axis tick marks and labels (default 3). - label_width: Cap on the y-axis label column width. - - Returns: - A ``rich.text.Text`` ready for ``console.print()``. - """ - if not ys: - t = Text() - t.append("No plottable data.", style=_STYLE_MUTED) - return t - - styles = {key: _SERIES_STYLES[i % len(_SERIES_STYLES)] for i, key in enumerate(ys)} - - text = Text() - x_from = x_label(xs[0]) - x_to = x_label(xs[-1]) - for key in ys: - text.append( - f" {key} ({x_from} – {x_to}) {ys[key][0]:.4g} → {ys[key][-1]:.4g}\n", - style=styles[key], - ) - - text.append_text( - _plot( - xs, - ys, - width=width, - height=height, - x_label=x_label, - y_label=y_label or (lambda v: f"{v:.3g}"), - y_log=y_log, - n_xticks=n_xticks, - label_width=label_width, - ) - ) - return text diff --git a/tests/test_plots_engine.py b/tests/test_plots_engine.py index aa64ee6d1..f612cffc2 100644 --- a/tests/test_plots_engine.py +++ b/tests/test_plots_engine.py @@ -2,12 +2,13 @@ import pytest -from together.lib.cli.utils.plots._engine import ( +from together.lib.cli.components.plots._engine import ( _interpolate, _uniform_grid, render_line_chart, render_sparklines, ) +from together.lib.cli.components.plot_finetune_metrics import _step_label def constant_series(n: int = 5, value: float = 1.0) -> list[tuple[float, float]]: @@ -27,10 +28,6 @@ def constant_series(n: int = 5, value: float = 1.0) -> list[tuple[float, float]] _WIDE_YS = [p[1] for p in _WIDE] -def _x_label(x: float) -> str: - return str(int(x)) - - def _interp(xs: list[float], ys: list[float], x_grid: list[float]) -> list[float]: """Helper: interpolate a single series onto x_grid.""" return _interpolate(xs, {"s": ys}, x_grid)["s"] @@ -160,7 +157,7 @@ def test_single_series_golden(self) -> None: width=20, height=4, n_xticks=3, - x_label=_x_label, + x_label=_step_label, ) assert result.plain == ( " loss (0 – 9) 1 → 0.1\n" @@ -180,7 +177,7 @@ def test_multi_series_golden(self) -> None: width=20, height=4, n_xticks=3, - x_label=_x_label, + x_label=_step_label, ) assert result.plain == ( " loss (0 – 9) 1 → 0.1\n" @@ -200,7 +197,7 @@ def test_log_scale_golden(self) -> None: width=20, height=4, n_xticks=3, - x_label=_x_label, + x_label=_step_label, y_log=True, ) assert result.plain == ( @@ -220,7 +217,7 @@ def test_constant_series_golden(self) -> None: {"flat": [p[1] for p in _flat]}, width=20, height=4, - x_label=_x_label, + x_label=_step_label, ) assert result.plain == ( " flat (0 – 9) 42 → 42\n" @@ -297,7 +294,7 @@ def test_non_finite_rendered_as_extreme_golden(self, bad_value: float, expected: # -inf/NaN → dip to x-axis border; +inf → spike to top data row. xs = [float(i) for i in range(10)] ys = [(1.0 - i * 0.1) if i != 5 else bad_value for i in range(10)] - result = render_line_chart(xs, {"loss": ys}, width=20, height=4, n_xticks=3, x_label=_x_label) + result = render_line_chart(xs, {"loss": ys}, width=20, height=4, n_xticks=3, x_label=_step_label) assert result.plain == expected def test_label_width_caps_y_axis(self) -> None: @@ -307,7 +304,7 @@ def test_label_width_caps_y_axis(self) -> None: {"metric": _WIDE_YS}, width=20, height=4, - x_label=_x_label, + x_label=_step_label, y_log=True, label_width=5, ) diff --git a/uv.lock b/uv.lock index a10f2d912..abe93dbc1 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "together" -version = "2.12.0" +version = "2.14.0" source = { editable = "." } dependencies = [ { name = "anyio" }, From ecd5c7b8a6e7d1f4fba7ed057efe1e45389aa9f5 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 09:59:08 +0200 Subject: [PATCH 05/16] fixes --- metrics.json | 1 + src/together/lib/cli/__init__.py | 2 +- .../lib/cli/api/fine_tuning/list_metrics.py | 46 +- .../cli/components/plot_finetune_metrics.py | 144 +++++ .../lib/cli/components/plots/__init__.py | 9 + .../lib/cli/components/plots/_engine.py | 590 ++++++++++++++++++ 6 files changed, 776 insertions(+), 16 deletions(-) create mode 100644 metrics.json create mode 100644 src/together/lib/cli/components/plot_finetune_metrics.py create mode 100644 src/together/lib/cli/components/plots/__init__.py create mode 100644 src/together/lib/cli/components/plots/_engine.py diff --git a/metrics.json b/metrics.json new file mode 100644 index 000000000..fe692c818 --- /dev/null +++ b/metrics.json @@ -0,0 +1 @@ +[{"timestamp":1.7770314739931374e+18,"train/loss":0.5725,"train/epoch":0.047619047619047616,"train/grad_norm":1.5748692750930786,"train/global_step":1.0,"train/learning_rate":1e-05},{"timestamp":1.7770315379636355e+18,"train/loss":0.5267,"train/epoch":0.14285714285714285,"train/grad_norm":1.4396071434020996,"train/global_step":3.0,"train/learning_rate":9.991050648838676e-06},{"timestamp":1.7770315693690883e+18,"train/loss":0.5325,"train/epoch":0.19047619047619047,"train/grad_norm":1.0482614040374756,"train/global_step":4.0,"train/learning_rate":9.979871469976197e-06},{"timestamp":1.777031600296171e+18,"train/loss":0.4901,"train/epoch":0.23809523809523808,"train/grad_norm":1.0096107721328735,"train/global_step":5.0,"train/learning_rate":9.964234631709188e-06},{"timestamp":1.7770316263251743e+18,"train/loss":0.4648,"train/epoch":0.2857142857142857,"train/grad_norm":1.0371627807617188,"train/global_step":6.0,"train/learning_rate":9.944154131125643e-06},{"timestamp":1.7770316556595707e+18,"train/loss":0.4851,"train/epoch":0.3333333333333333,"train/grad_norm":0.7985115647315979,"train/global_step":7.0,"train/learning_rate":9.91964794299315e-06},{"timestamp":1.777031656171934e+18,"train/loss":0.417,"train/epoch":0.38095238095238093,"train/grad_norm":0.6295253038406372,"train/global_step":8.0,"train/learning_rate":9.890738003669029e-06},{"timestamp":1.777031682139413e+18,"train/loss":0.4299,"train/epoch":0.42857142857142855,"train/grad_norm":0.5814775228500366,"train/global_step":9.0,"train/learning_rate":9.857450191464337e-06},{"timestamp":1.7770317086595318e+18,"train/loss":0.4688,"train/epoch":0.47619047619047616,"train/grad_norm":0.5624698400497437,"train/global_step":10.0,"train/learning_rate":9.819814303479268e-06},{"timestamp":1.7770317090552543e+18,"train/loss":0.442,"train/epoch":0.5238095238095238,"train/grad_norm":0.4839235544204712,"train/global_step":11.0,"train/learning_rate":9.777864028930705e-06},{"timestamp":1.77703173877343e+18,"train/loss":0.4521,"train/epoch":0.5714285714285714,"train/grad_norm":0.41900405287742615,"train/global_step":12.0,"train/learning_rate":9.731636918995821e-06},{"timestamp":1.7770317671818757e+18,"train/loss":0.4293,"train/epoch":0.6190476190476191,"train/grad_norm":0.35965147614479065,"train/global_step":13.0,"train/learning_rate":9.681174353198687e-06},{"timestamp":1.7770317675498414e+18,"train/loss":0.4072,"train/epoch":0.6666666666666666,"train/grad_norm":0.34983327984809875,"train/global_step":14.0,"train/learning_rate":9.626521502369984e-06},{"timestamp":1.7770317960450604e+18,"train/loss":0.4075,"train/epoch":0.7619047619047619,"train/grad_norm":0.3408649265766144,"train/global_step":16.0,"train/learning_rate":9.504844339512096e-06},{"timestamp":1.7770318221638467e+18,"train/loss":0.3632,"train/epoch":0.8095238095238095,"train/grad_norm":0.29929089546203613,"train/global_step":17.0,"train/learning_rate":9.437928945022772e-06},{"timestamp":1.7770318491147359e+18,"train/loss":0.444,"train/epoch":0.8571428571428571,"train/grad_norm":0.30117282271385193,"train/global_step":18.0,"train/learning_rate":9.36704100308565e-06},{"timestamp":1.777031849517807e+18,"train/loss":0.4315,"train/epoch":0.9047619047619048,"train/grad_norm":0.3524363040924072,"train/global_step":19.0,"train/learning_rate":9.292243968009332e-06},{"timestamp":1.777031876918584e+18,"train/loss":0.4081,"train/epoch":0.9523809523809523,"train/grad_norm":0.27605122327804565,"train/global_step":20.0,"train/learning_rate":9.213604793270196e-06},{"timestamp":1.7770319047202714e+18,"train/loss":0.4143,"train/epoch":1.0,"train/grad_norm":0.30752140283584595,"train/global_step":21.0,"train/learning_rate":9.131193871579975e-06},{"eval/loss":5.939453125,"timestamp":1.7770319182683062e+18,"train/epoch":1.0,"train/global_step":21.0},{"timestamp":1.7770319186845947e+18,"train/loss":0.3546,"train/epoch":1.0476190476190477,"train/grad_norm":0.3130515217781067,"train/global_step":22.0,"train/learning_rate":9.045084971874738e-06},{"timestamp":1.7770319191048975e+18,"train/loss":0.3728,"train/epoch":1.0952380952380953,"train/grad_norm":0.29933640360832214,"train/global_step":23.0,"train/learning_rate":8.955355173281709e-06},{"timestamp":1.7770319195131405e+18,"train/loss":0.3843,"train/epoch":1.1428571428571428,"train/grad_norm":0.26397213339805603,"train/global_step":24.0,"train/learning_rate":8.862084796122998e-06},{"timestamp":1.777031920588432e+18,"train/loss":0.4,"train/epoch":1.1904761904761905,"train/grad_norm":0.33784887194633484,"train/global_step":25.0,"train/learning_rate":8.765357330018056e-06},{"timestamp":1.7770319209921388e+18,"train/loss":0.3812,"train/epoch":1.2380952380952381,"train/grad_norm":0.32685449719429016,"train/global_step":26.0,"train/learning_rate":8.665259359149132e-06},{"timestamp":1.7770319213974446e+18,"train/loss":0.3792,"train/epoch":1.2857142857142856,"train/grad_norm":0.28974828124046326,"train/global_step":27.0,"train/learning_rate":8.561880484756726e-06},{"timestamp":1.777031922236854e+18,"train/loss":0.3536,"train/epoch":1.380952380952381,"train/grad_norm":0.26806962490081787,"train/global_step":29.0,"train/learning_rate":8.345653031794292e-06},{"timestamp":1.7770319226348698e+18,"train/loss":0.3622,"train/epoch":1.4285714285714286,"train/grad_norm":0.3048115670681,"train/global_step":30.0,"train/learning_rate":8.232998006078998e-06},{"timestamp":1.7770319229918449e+18,"train/loss":0.4069,"train/epoch":1.4761904761904763,"train/grad_norm":0.31159284710884094,"train/global_step":31.0,"train/learning_rate":8.117449009293668e-06},{"timestamp":1.7770319233036634e+18,"train/loss":0.3862,"train/epoch":1.5238095238095237,"train/grad_norm":0.2849102318286896,"train/global_step":32.0,"train/learning_rate":7.99910947343957e-06},{"timestamp":1.777031923574597e+18,"train/loss":0.4019,"train/epoch":1.5714285714285714,"train/grad_norm":0.3013054430484772,"train/global_step":33.0,"train/learning_rate":7.87808532842837e-06},{"timestamp":1.7770319238637245e+18,"train/loss":0.3843,"train/epoch":1.619047619047619,"train/grad_norm":0.2755144238471985,"train/global_step":34.0,"train/learning_rate":7.754484907260513e-06},{"timestamp":1.7770319244743252e+18,"train/loss":0.3647,"train/epoch":1.6666666666666665,"train/grad_norm":0.31459057331085205,"train/global_step":35.0,"train/learning_rate":7.628418849052523e-06},{"timestamp":1.7770319250751076e+18,"train/loss":0.3413,"train/epoch":1.7142857142857144,"train/grad_norm":0.2461976408958435,"train/global_step":36.0,"train/learning_rate":7.500000000000001e-06},{"timestamp":1.7770319253530952e+18,"train/loss":0.3684,"train/epoch":1.7619047619047619,"train/grad_norm":0.3002825677394867,"train/global_step":37.0,"train/learning_rate":7.369343312364994e-06},{"timestamp":1.7770319256612006e+18,"train/loss":0.334,"train/epoch":1.8095238095238095,"train/grad_norm":0.25984328985214233,"train/global_step":38.0,"train/learning_rate":7.236565741578163e-06},{"timestamp":1.7770319259561108e+18,"train/loss":0.4156,"train/epoch":1.8571428571428572,"train/grad_norm":0.2751716077327728,"train/global_step":39.0,"train/learning_rate":7.101786141547829e-06},{"timestamp":1.7770319262290227e+18,"train/loss":0.3953,"train/epoch":1.9047619047619047,"train/grad_norm":0.2947269380092621,"train/global_step":40.0,"train/learning_rate":6.965125158269619e-06},{"timestamp":1.7770319268452413e+18,"train/loss":0.3856,"train/epoch":2.0,"train/grad_norm":0.269094854593277,"train/global_step":42.0,"train/learning_rate":6.686649936914151e-06},{"eval/loss":5.833984375,"timestamp":1.777031927850314e+18,"train/epoch":2.0,"train/global_step":42.0},{"timestamp":1.7770319281532844e+18,"train/loss":0.3284,"train/epoch":2.0476190476190474,"train/grad_norm":0.27204498648643494,"train/global_step":43.0,"train/learning_rate":6.545084971874738e-06},{"timestamp":1.7770319284494723e+18,"train/loss":0.3508,"train/epoch":2.0952380952380953,"train/grad_norm":0.2703777849674225,"train/global_step":44.0,"train/learning_rate":6.402136946530014e-06},{"timestamp":1.7770319287519683e+18,"train/loss":0.3624,"train/epoch":2.142857142857143,"train/grad_norm":0.25161707401275635,"train/global_step":45.0,"train/learning_rate":6.257933818722544e-06},{"timestamp":1.7770319290746417e+18,"train/loss":0.3719,"train/epoch":2.1904761904761907,"train/grad_norm":0.30092841386795044,"train/global_step":46.0,"train/learning_rate":6.112604669781572e-06},{"timestamp":1.7770319293780872e+18,"train/loss":0.3538,"train/epoch":2.238095238095238,"train/grad_norm":0.30388522148132324,"train/global_step":47.0,"train/learning_rate":5.9662795889777666e-06},{"timestamp":1.7770319296753797e+18,"train/loss":0.3568,"train/epoch":2.2857142857142856,"train/grad_norm":0.2648054361343384,"train/global_step":48.0,"train/learning_rate":5.819089557075689e-06},{"timestamp":1.7770319303294776e+18,"train/loss":0.4008,"train/epoch":2.3333333333333335,"train/grad_norm":0.25187763571739197,"train/global_step":49.0,"train/learning_rate":5.671166329088278e-06},{"timestamp":1.777031930678494e+18,"train/loss":0.3353,"train/epoch":2.380952380952381,"train/grad_norm":0.26126545667648315,"train/global_step":50.0,"train/learning_rate":5.522642316338268e-06},{"timestamp":1.777031931009004e+18,"train/loss":0.3403,"train/epoch":2.4285714285714284,"train/grad_norm":0.29477638006210327,"train/global_step":51.0,"train/learning_rate":5.373650467932122e-06},{"timestamp":1.7770319313563054e+18,"train/loss":0.3843,"train/epoch":2.4761904761904763,"train/grad_norm":0.3020482659339905,"train/global_step":52.0,"train/learning_rate":5.224324151752575e-06},{"timestamp":1.7770319316935217e+18,"train/loss":0.3658,"train/epoch":2.5238095238095237,"train/grad_norm":0.2735633850097656,"train/global_step":53.0,"train/learning_rate":5.074797035076319e-06},{"timestamp":1.77703193236958e+18,"train/loss":0.3667,"train/epoch":2.619047619047619,"train/grad_norm":0.2675466239452362,"train/global_step":55.0,"train/learning_rate":4.775675848247427e-06},{"timestamp":1.777031932715636e+18,"train/loss":0.3463,"train/epoch":2.6666666666666665,"train/grad_norm":0.30449649691581726,"train/global_step":56.0,"train/learning_rate":4.626349532067879e-06},{"timestamp":1.777031933071996e+18,"train/loss":0.3292,"train/epoch":2.7142857142857144,"train/grad_norm":0.24315503239631653,"train/global_step":57.0,"train/learning_rate":4.477357683661734e-06},{"timestamp":1.7770319337252216e+18,"train/loss":0.3512,"train/epoch":2.761904761904762,"train/grad_norm":0.2935601472854614,"train/global_step":58.0,"train/learning_rate":4.3288336709117246e-06},{"timestamp":1.7770319340828797e+18,"train/loss":0.3204,"train/epoch":2.8095238095238093,"train/grad_norm":0.2543131113052368,"train/global_step":59.0,"train/learning_rate":4.180910442924312e-06},{"timestamp":1.7770319344318339e+18,"train/loss":0.4008,"train/epoch":2.857142857142857,"train/grad_norm":0.2751201093196869,"train/global_step":60.0,"train/learning_rate":4.033720411022235e-06},{"timestamp":1.7770319350886285e+18,"train/loss":0.3774,"train/epoch":2.9047619047619047,"train/grad_norm":0.30817028880119324,"train/global_step":61.0,"train/learning_rate":3.887395330218429e-06},{"timestamp":1.7770319354579146e+18,"train/loss":0.3708,"train/epoch":2.9523809523809526,"train/grad_norm":0.2573518455028534,"train/global_step":62.0,"train/learning_rate":3.7420661812774577e-06},{"timestamp":1.777031936142077e+18,"train/loss":0.3719,"train/epoch":3.0,"train/grad_norm":0.2749069929122925,"train/global_step":63.0,"train/learning_rate":3.5978630534699873e-06},{"eval/loss":5.82421875,"timestamp":1.777031937157716e+18,"train/epoch":3.0,"train/global_step":63.0},{"timestamp":1.7770319374588646e+18,"train/loss":0.3154,"train/epoch":3.0476190476190474,"train/grad_norm":0.2740519940853119,"train/global_step":64.0,"train/learning_rate":3.4549150281252635e-06},{"timestamp":1.7770319377837844e+18,"train/loss":0.3397,"train/epoch":3.0952380952380953,"train/grad_norm":0.2745459973812103,"train/global_step":65.0,"train/learning_rate":3.3133500630858507e-06},{"timestamp":1.7770319380782636e+18,"train/loss":0.353,"train/epoch":3.142857142857143,"train/grad_norm":0.24551734328269958,"train/global_step":66.0,"train/learning_rate":3.173294878168025e-06},{"timestamp":1.7770319386512013e+18,"train/loss":0.3402,"train/epoch":3.238095238095238,"train/grad_norm":0.301933616399765,"train/global_step":68.0,"train/learning_rate":2.8982138584521734e-06},{"timestamp":1.777031938949057e+18,"train/loss":0.3459,"train/epoch":3.2857142857142856,"train/grad_norm":0.2628892660140991,"train/global_step":69.0,"train/learning_rate":2.7634342584218364e-06},{"timestamp":1.7770319392524636e+18,"train/loss":0.3926,"train/epoch":3.3333333333333335,"train/grad_norm":0.25296470522880554,"train/global_step":70.0,"train/learning_rate":2.6306566876350072e-06},{"timestamp":1.7770319395400535e+18,"train/loss":0.3273,"train/epoch":3.380952380952381,"train/grad_norm":0.2628619372844696,"train/global_step":71.0,"train/learning_rate":2.5000000000000015e-06},{"timestamp":1.777031939819372e+18,"train/loss":0.3311,"train/epoch":3.4285714285714284,"train/grad_norm":0.2902317941188812,"train/global_step":72.0,"train/learning_rate":2.371581150947476e-06},{"timestamp":1.7770319403707064e+18,"train/loss":0.3748,"train/epoch":3.4761904761904763,"train/grad_norm":0.30575087666511536,"train/global_step":73.0,"train/learning_rate":2.245515092739488e-06},{"timestamp":1.777031940655665e+18,"train/loss":0.3573,"train/epoch":3.5238095238095237,"train/grad_norm":0.27393031120300293,"train/global_step":74.0,"train/learning_rate":2.1219146715716332e-06},{"timestamp":1.777031940933279e+18,"train/loss":0.374,"train/epoch":3.571428571428571,"train/grad_norm":0.29484593868255615,"train/global_step":75.0,"train/learning_rate":2.0008905265604316e-06},{"timestamp":1.7770319412286344e+18,"train/loss":0.3587,"train/epoch":3.619047619047619,"train/grad_norm":0.27231061458587646,"train/global_step":76.0,"train/learning_rate":1.8825509907063328e-06},{"timestamp":1.7770319415158525e+18,"train/loss":0.3385,"train/epoch":3.6666666666666665,"train/grad_norm":0.3046872913837433,"train/global_step":77.0,"train/learning_rate":1.7670019939210025e-06},{"timestamp":1.777031941878367e+18,"train/loss":0.3229,"train/epoch":3.7142857142857144,"train/grad_norm":0.24565838277339935,"train/global_step":78.0,"train/learning_rate":1.6543469682057105e-06},{"timestamp":1.777031942242403e+18,"train/loss":0.344,"train/epoch":3.761904761904762,"train/grad_norm":0.29259249567985535,"train/global_step":79.0,"train/learning_rate":1.544686755065677e-06},{"timestamp":1.7770319429027528e+18,"train/loss":0.3933,"train/epoch":3.857142857142857,"train/grad_norm":0.2701041102409363,"train/global_step":81.0,"train/learning_rate":1.3347406408508695e-06},{"timestamp":1.777031943485962e+18,"train/loss":0.3705,"train/epoch":3.9047619047619047,"train/grad_norm":0.3030908405780792,"train/global_step":82.0,"train/learning_rate":1.234642669981946e-06},{"timestamp":1.7770319437911675e+18,"train/loss":0.3651,"train/epoch":3.9523809523809526,"train/grad_norm":0.25513312220573425,"train/global_step":83.0,"train/learning_rate":1.137915203877003e-06},{"timestamp":1.7770319441107953e+18,"train/loss":0.367,"train/epoch":4.0,"train/grad_norm":0.27516013383865356,"train/global_step":84.0,"train/learning_rate":1.044644826718295e-06},{"eval/loss":5.82421875,"timestamp":1.777031945116394e+18,"train/epoch":4.0,"train/global_step":84.0},{"timestamp":1.777031945404118e+18,"train/loss":0.3101,"train/epoch":4.0476190476190474,"train/grad_norm":0.2693224251270294,"train/global_step":85.0,"train/learning_rate":9.549150281252633e-07},{"timestamp":1.7770319456981092e+18,"train/loss":0.3345,"train/epoch":4.095238095238095,"train/grad_norm":0.2737109363079071,"train/global_step":86.0,"train/learning_rate":8.688061284200266e-07},{"timestamp":1.7770319459895841e+18,"train/loss":0.3476,"train/epoch":4.142857142857143,"train/grad_norm":0.24587756395339966,"train/global_step":87.0,"train/learning_rate":7.863952067298042e-07},{"timestamp":1.77703194626355e+18,"train/loss":0.3522,"train/epoch":4.190476190476191,"train/grad_norm":0.3057315945625305,"train/global_step":88.0,"train/learning_rate":7.077560319906696e-07},{"timestamp":1.7770319468008266e+18,"train/loss":0.3336,"train/epoch":4.238095238095238,"train/grad_norm":0.3030700385570526,"train/global_step":89.0,"train/learning_rate":6.329589969143518e-07},{"timestamp":1.777031947416965e+18,"train/loss":0.342,"train/epoch":4.285714285714286,"train/grad_norm":0.2629696726799011,"train/global_step":90.0,"train/learning_rate":5.620710549772295e-07},{"timestamp":1.777031947766062e+18,"train/loss":0.3896,"train/epoch":4.333333333333333,"train/grad_norm":0.253886878490448,"train/global_step":91.0,"train/learning_rate":4.951556604879049e-07},{"timestamp":1.777031948056129e+18,"train/loss":0.3239,"train/epoch":4.380952380952381,"train/grad_norm":0.2629833519458771,"train/global_step":92.0,"train/learning_rate":4.322727117869951e-07},{"timestamp":1.7770319486236925e+18,"train/loss":0.3718,"train/epoch":4.476190476190476,"train/grad_norm":0.30251115560531616,"train/global_step":94.0,"train/learning_rate":3.18825646801314e-07},{"timestamp":1.7770319489037783e+18,"train/loss":0.3542,"train/epoch":4.523809523809524,"train/grad_norm":0.27201929688453674,"train/global_step":95.0,"train/learning_rate":2.6836308100417874e-07},{"timestamp":1.7770319491791521e+18,"train/loss":0.3717,"train/epoch":4.571428571428571,"train/grad_norm":0.2946451008319855,"train/global_step":96.0,"train/learning_rate":2.2213597106929608e-07},{"timestamp":1.77703194947258e+18,"train/loss":0.3558,"train/epoch":4.619047619047619,"train/grad_norm":0.2709577679634094,"train/global_step":97.0,"train/learning_rate":1.801856965207338e-07},{"timestamp":1.7770319497592973e+18,"train/loss":0.3358,"train/epoch":4.666666666666667,"train/grad_norm":0.3053297698497772,"train/global_step":98.0,"train/learning_rate":1.4254980853566248e-07},{"timestamp":1.7770319500497021e+18,"train/loss":0.3209,"train/epoch":4.714285714285714,"train/grad_norm":0.24520252645015717,"train/global_step":99.0,"train/learning_rate":1.0926199633097156e-07},{"timestamp":1.7770319503302083e+18,"train/loss":0.3419,"train/epoch":4.761904761904762,"train/grad_norm":0.2922167479991913,"train/global_step":100.0,"train/learning_rate":8.035205700685167e-08},{"timestamp":1.7770319506339753e+18,"train/loss":0.3129,"train/epoch":4.809523809523809,"train/grad_norm":0.25622737407684326,"train/global_step":101.0,"train/learning_rate":5.584586887435739e-08},{"timestamp":1.7770319509281615e+18,"train/loss":0.3922,"train/epoch":4.857142857142857,"train/grad_norm":0.2694510519504547,"train/global_step":102.0,"train/learning_rate":3.576536829081323e-08},{"timestamp":1.777031951209152e+18,"train/loss":0.3677,"train/epoch":4.904761904761905,"train/grad_norm":0.29987600445747375,"train/global_step":103.0,"train/learning_rate":2.012853002380466e-08},{"timestamp":1.7770319517874483e+18,"train/loss":0.3643,"train/epoch":4.9523809523809526,"train/grad_norm":0.2551063597202301,"train/global_step":104.0,"train/learning_rate":8.949351161324227e-09},{"timestamp":1.7770319521104128e+18,"train/loss":0.3647,"train/epoch":5.0,"train/grad_norm":0.2752157151699066,"train/global_step":105.0,"train/learning_rate":2.237838582483387e-09},{"eval/loss":5.82421875,"timestamp":1.777031953116419e+18,"train/epoch":5.0,"train/global_step":105.0}] \ No newline at end of file diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index 467cde712..2854cdf93 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -53,7 +53,6 @@ JIG_SECRETS_UNSET_HELP_EXAMPLES, ENDPOINTS_HARDWARE_HELP_EXAMPLES, FINE_TUNING_CREATE_HELP_EXAMPLES, - FINE_TUNING_LIST_METRICS_HELP_EXAMPLES, JIG_SECRETS_DELETE_HELP_EXAMPLES, JIG_VOLUMES_CREATE_HELP_EXAMPLES, JIG_VOLUMES_UPDATE_HELP_EXAMPLES, @@ -62,6 +61,7 @@ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES, BETA_CLUSTERS_STORAGE_HELP_EXAMPLES, FILES_RETRIEVE_CONTENT_HELP_EXAMPLES, + FINE_TUNING_LIST_METRICS_HELP_EXAMPLES, BETA_CLUSTERS_STORAGE_CREATE_HELP_EXAMPLES, BETA_CLUSTERS_STORAGE_UPDATE_HELP_EXAMPLES, BETA_CLUSTERS_GET_CREDENTIALS_HELP_EXAMPLES, diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index be37118c1..a45c2c8fb 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -1,10 +1,12 @@ from __future__ import annotations -from typing import Annotated +from typing import Optional, Annotated +from pathlib import Path from datetime import datetime from cyclopts import Parameter +from together import omit from together._utils._json import openapi_dumps from together.lib.cli.utils.config import CLIConfigParameter from together.lib.cli.utils._console import console @@ -16,32 +18,46 @@ async def list_metrics( fine_tune_id: Annotated[str, Parameter(help="The ID of the fine-tuning job")], *, config: CLIConfigParameter, - global_step_from: Annotated[int | None, Parameter(help="Filter metrics from this global step (inclusive).")] = None, - global_step_to: Annotated[int | None, Parameter(help="Filter metrics to this global step (inclusive).")] = None, - logged_at_from: Annotated[datetime | None, Parameter(help="Filter metrics logged at or after this time.")] = None, - logged_at_to: Annotated[datetime | None, Parameter(help="Filter metrics logged at or before this time.")] = None, - resolution: Annotated[int | None, Parameter(help="Number of training metric points to return. Does not limit the number of eval metric points.")] = None, + global_step_from: Annotated[ + Optional[int], Parameter(help="Filter metrics from this global step (inclusive).") + ] = None, + global_step_to: Annotated[Optional[int], Parameter(help="Filter metrics to this global step (inclusive).")] = None, + logged_at_from: Annotated[ + Optional[datetime], Parameter(help="Filter metrics logged at or after this time.") + ] = None, + logged_at_to: Annotated[Optional[datetime], Parameter(help="Filter metrics logged at or before this time.")] = None, + resolution: Annotated[ + Optional[int], + Parameter(help="Number of training metric points to return. Does not limit the number of eval metric points."), + ] = None, + save: Annotated[Optional[Path], Parameter("--save", help="Save metrics to a file as JSON.")] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" - resolution_value = console.width - METRICS_WIDTH_PADDING if not config.json else resolution + resolution_value = resolution if config.json else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( fine_tune_id, - global_step_from=global_step_from, - global_step_to=global_step_to, - logged_at_from=logged_at_from, - logged_at_to=logged_at_to, - resolution=resolution_value, + global_step_from=global_step_from or omit, + global_step_to=global_step_to or omit, + logged_at_from=logged_at_from or omit, + logged_at_to=logged_at_to or omit, + resolution=resolution_value or omit, ), ) - if config.json: - console.print_json(openapi_dumps(response.metrics or []).decode("utf-8")) + metrics = response.metrics or [] + json_bytes = openapi_dumps(metrics) + + if save is not None: + save.write_bytes(json_bytes) + console.print(f"[success]Metrics saved to {save}[/success]") return - metrics = response.metrics or [] + if config.json: + console.print_json(json_bytes.decode("utf-8")) + return if not metrics: console.print(f"[muted]No metrics found for job {fine_tune_id}[/muted]") diff --git a/src/together/lib/cli/components/plot_finetune_metrics.py b/src/together/lib/cli/components/plot_finetune_metrics.py new file mode 100644 index 000000000..8a926bc84 --- /dev/null +++ b/src/together/lib/cli/components/plot_finetune_metrics.py @@ -0,0 +1,144 @@ +"""Fine-tuning metrics plotting utilities. + +Public API +---------- +``metrics_block_sparklines(metrics)`` + One ▁▂▃▄▅▆▇█ sparkline line per metric — used in ``retrieve``. + +``metrics_ascii_charts(metrics, height=6)`` + One full ASCII line chart per metric — used in ``list-metrics``. +""" + +from __future__ import annotations + +import math +from typing import Any +from collections import defaultdict + +from rich.text import Text + +from together.lib.cli.components.plots import should_log, render_line_chart, render_sparklines + +# Columns reserved for the y-axis label area, ┼ connector, leading indent, and +# surrounding margin in the ASCII chart layout. This must be >= label_width + 1 +# (the default label_width used in metrics_ascii_charts is 8, so the minimum is +# 9). Callers subtract this from the terminal width to get the usable plot width. +METRICS_WIDTH_PADDING = 48 + +_SKIP_KEYS: frozenset[str] = frozenset({"timestamp", "step", "global_step", "epoch"}) + + +def _is_skip(k: str) -> bool: + base = k.rsplit("/", 1)[-1] + return base in _SKIP_KEYS or base.endswith("_step") or base.endswith("_epoch") + + +def _step_label(x: float) -> str: + return str(int(x)) + + +def _collect_series( + metrics: list[dict[str, Any]], +) -> dict[str, tuple[list[float], list[float]]]: + """Collect plottable numeric series from a list of metric dicts. + + Returns a mapping of name → (xs, ys). Keys are discovered in insertion + order; step/epoch/timestamp fields are skipped. NaN values are converted + to ``-inf`` so the rendering engine plots them at the very bottom of the + chart rather than silently dropping them. + """ + series: dict[str, tuple[list[float], list[float]]] = defaultdict(lambda: ([], [])) + for row in metrics: + step = float(row["train/global_step"]) + for k, v in row.items(): + if _is_skip(k) or isinstance(v, bool) or not isinstance(v, (int, float)): + continue + val = float(v) + # NaN is rendered as a dip to the bottom (-inf sentinel). + if math.isnan(val): + val = float("-inf") + series[k][0].append(step) + series[k][1].append(val) + return series + + +def _no_data() -> Text: + t = Text() + t.append("No plottable metrics found.", style="muted") + return t + + +def metrics_block_sparklines( + metrics: list[dict[str, Any]], + *, + width: int = 60, +) -> Text: + """One block-sparkline line per metric, coloured with the CLI theme. + + Args: + metrics: List of flat metric dicts (one per training step). + width: Sparkline character width (default 60). + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + series = _collect_series(metrics) + if not series: + return _no_data() + label_w = max(len(k) for k in series) + text = Text() + for key, (xs, ys) in series.items(): + text.append_text( + render_sparklines( + key, + xs, + ys, + width=width, + y_log=should_log(ys), + label_width=label_w, + ) + ) + return text + + +def metrics_ascii_charts( + metrics: list[dict[str, Any]], + *, + height: int = 6, + width: int = 60, + label_width: int = 8, +) -> Text: + """One ASCII line chart per metric, with a global-step x-axis. + + Args: + metrics: List of flat metric dicts (one per training step). + height: Chart body height in rows (default 6). + width: Plot character width (default 60). + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + series = _collect_series(metrics) + text = Text() + for key, (xs, ys) in series.items(): + if text: + text.append("\n") + text.append_text( + render_line_chart( + xs, + {key: ys}, + x_label=_step_label, + y_log=should_log(ys), + height=height, + width=width, + label_width=label_width, + ) + ) + return text if text else _no_data() + + +__all__ = [ + "metrics_block_sparklines", + "metrics_ascii_charts", + "METRICS_WIDTH_PADDING", +] diff --git a/src/together/lib/cli/components/plots/__init__.py b/src/together/lib/cli/components/plots/__init__.py new file mode 100644 index 000000000..c2d08bf68 --- /dev/null +++ b/src/together/lib/cli/components/plots/__init__.py @@ -0,0 +1,9 @@ +"""Generic CLI plot utilities.""" + +from together.lib.cli.components.plots._engine import should_log, render_line_chart, render_sparklines + +__all__ = [ + "render_line_chart", + "render_sparklines", + "should_log", +] diff --git a/src/together/lib/cli/components/plots/_engine.py b/src/together/lib/cli/components/plots/_engine.py new file mode 100644 index 000000000..c3f9f1ab4 --- /dev/null +++ b/src/together/lib/cli/components/plots/_engine.py @@ -0,0 +1,590 @@ +"""ASCII sparkline and chart engine for time-series data. + +Designed for scalar time-series (loss, accuracy, …); not a general-purpose +plotting library. + +Internal pipeline (``_plot``, ``_interpolate``, …) uses a shared x-grid with +named y series: ``xs: list[float]`` + ``ys: dict[str, list[float]]``. + +Public API +---------- +``render_line_chart(xs, ys, ...)`` + One or more named series plotted on a shared ASCII line chart. All series + share the same x-axis and y-scale. + +``render_sparklines(name, xs, ys, ...)`` + A single block-sparkline row (▁▂▃▄▅▆▇█). Call once per series and pass a + shared ``label_width`` across calls for consistent label alignment. Names + are right-justified; those that exceed ``label_width`` are truncated with + ``...``. +""" + +from __future__ import annotations + +import math +import bisect +from typing import Callable + +from rich.text import Text + +_SPARK_BLOCKS = " ▁▂▃▄▅▆▇█" + +# Styles cycled across series in insertion order. +_SERIES_STYLES = ["white", "green", "yellow", "cyan", "magenta"] + +# UI style tokens used throughout the rendering pipeline. +_STYLE_PRIMARY = "primary" # default plot body text +_STYLE_SECONDARY = "secondary" # axis labels and tick text +_STYLE_ACCENT = "accent" # axis border characters (┼ └ ┬ …) +_STYLE_MUTED = "muted" # series name labels and empty-state messages +_STYLE_SPARK = "white" # sparkline bar characters + +# Sentinels used in quantized_ys to signal out-of-range non-finite values. +# Both are outside the valid slot range [0, height-1]. +_NEG_INF_SENTINEL = -1 # -inf: line descends to the x-axis border +_POS_INF_SENTINEL = -2 # +inf: line ascends to the top data row +_NAN_SENTINEL = -3 # NaN: no line at the place + + +def should_log(vals: list[float]) -> bool: + """Return True when values span more than 100×, suggesting log scale.""" + nz = [v for v in vals if v > 0] + return len(nz) > 1 and (max(nz) / min(nz)) > 100 + + +def _uniform_grid(vals: list[float], n: int) -> list[float]: + """Return n evenly-spaced points spanning [min(vals), max(vals)]. + + Non-finite values (e.g. the -inf sentinel used for NaN data points) are + excluded from the range computation so they don't corrupt the grid. + """ + finite = [v for v in vals if math.isfinite(v)] + min_val, max_val = min(finite), max(finite) + if n <= 1: + return [min_val] + return [min_val + (max_val - min_val) * idx / (n - 1) for idx in range(n)] + + +def _interpolate( + xs: list[float], + ys: dict[str, list[float]], + x_grid: list[float], +) -> dict[str, list[float]]: + """Linearly interpolate each named y series onto x_grid; clamp at the edges. + + For each grid point: + - If it falls before the first data point, use the first y value. + - If it falls after the last data point, use the last y value. + - Otherwise, linearly interpolate between the two bracketing data points. + """ + results: dict[str, list[float]] = {} + for name, yvals in ys.items(): + # Sort by x, using insertion order as a tiebreaker so that duplicate + # steps are resolved deterministically (first occurrence wins). + pairs = sorted(enumerate(zip(xs, yvals)), key=lambda t: (t[1][0], t[0])) + xs_s = [x for _, (x, _y) in pairs] + ys_s = [y for _, (_x, y) in pairs] + + interpolated: list[float] = [] + for x_point in x_grid: + pos = bisect.bisect_left(xs_s, x_point) + if pos == 0: + interpolated.append(ys_s[0]) + elif pos == len(xs_s): + interpolated.append(ys_s[-1]) + elif xs_s[pos] == x_point: + interpolated.append(ys_s[pos]) + else: + left_x, left_y = xs_s[pos - 1], ys_s[pos - 1] + right_x, right_y = xs_s[pos], ys_s[pos] + # When either bracket endpoint is a non-finite sentinel + # (-inf/NaN or +inf) we cannot compute a meaningful slope. + # Instead, assign this grid point to whichever bracket is + # closer: if that bracket is non-finite the spike/dip extends + # to this column; if it is finite we use its value so the + # spike/dip stays as narrow as the grid resolution allows. + if not math.isfinite(left_y) or not math.isfinite(right_y): + closer_y = left_y if (x_point - left_x) <= (right_x - x_point) else right_y + interpolated.append(closer_y) + else: + slope = (right_y - left_y) / (right_x - left_x) + interpolated.append(left_y + slope * (x_point - left_x)) + + results[name] = interpolated + return results + + +def _log_transform( + named_values: dict[str, list[float]], +) -> dict[str, list[float]]: + """Return new traces with ys replaced by their log10 values.""" + result: dict[str, list[float]] = {} + for name, values in named_values.items(): + nz = [value for value in values if value > 0] + eps = min(nz) * 0.01 if nz else 1e-10 + result[name] = [math.log10(max(value, eps)) for value in values] + return result + + +def _quantize_ys( + interpolated_ys: dict[str, list[float]], + y_grid: list[float], +) -> list[list[int]]: + """Snap each interpolated y value to the index of the nearest y_grid slot. + + Non-finite values are mapped to out-of-band sentinels: + + * ``_POS_INF_SENTINEL`` (``-1``) for ``+inf`` — the line spikes to the top + data row. + * ``_NEG_INF_SENTINEL`` (``-2``) for ``-inf`` / ``NaN`` — the line + descends to the x-axis border row. + """ + quantized_ys: list[list[int]] = [] + for ys in interpolated_ys.values(): + row: list[int] = [] + for y in ys: + if math.isfinite(y): + row.append(min(range(len(y_grid)), key=lambda i: abs(y_grid[i] - y))) + elif y > 0: # +inf + row.append(_POS_INF_SENTINEL) + elif math.isinf(y): + row.append(_NEG_INF_SENTINEL) + else: # -inf or NaN (NaN > 0 is False) + row.append(_NAN_SENTINEL) + quantized_ys.append(row) + return quantized_ys + + +def _fit_spark_label(name: str, label_width: int) -> str: + """Right-justify *name* in *label_width* chars, truncating with '...' if needed.""" + if len(name) <= label_width: + return name.rjust(label_width) + return name[: max(0, label_width - 3)] + "..." + + +def _y_labels( + y_grid: list[float], + y_log: bool, + y_label: Callable[[float], str], +) -> list[str]: + """Build y-axis tick label strings from the y grid.""" + labels = [y_label(10**y) if y_log else y_label(y) for y in y_grid[::-1]] + return labels + + +def _x_labels( + x_grid: list[float], + n_xticks: int, + x_label: Callable[[float], str], +) -> list[tuple[int, str]]: + """Return (column_index, label_string) pairs for each x-axis tick.""" + width = len(x_grid) + x_min = x_grid[0] + # Extend by one grid step beyond the last point so the rightmost tick + # label shows the true data maximum. round() suppresses floating-point + # noise that would otherwise accumulate in the tick value calculations. + x_max = round(x_grid[-1] + ((x_grid[-1] - x_grid[0]) / (width - 1) if width > 1 else 0.0), 10) + if n_xticks < 2 or width <= 1: + return [(0, x_label(x_min))] + tick_cols = [round(i * (width - 1) / (n_xticks - 1)) for i in range(n_xticks)] + tick_vals = [x_min + (x_max - x_min) * i / (n_xticks - 1) for i in range(n_xticks)] + return [(col, x_label(val)) for col, val in zip(tick_cols, tick_vals)] + + +def _draw_y_axis( + grid: list[list[str]], + style_grid: list[list[str]], + labels: list[str], + label_w: int, +) -> None: + """Fill y-axis labels and ┼ connectors into the grid.""" + for label, grid_row, style_row in zip(labels, grid, style_grid): + if len(label) > label_w: + label = label[: max(0, label_w - 3)] + "..." + label = label.rjust(label_w) + for ci, ch in enumerate(label): + grid_row[ci] = ch + style_row[ci] = _STYLE_SECONDARY + grid_row[label_w] = "┼" + style_row[label_w] = _STYLE_ACCENT + + +def _draw_lines( + grid: list[list[str]], + style_grid: list[list[str]], + quantized_ys: list[list[int]], + styles: list[str], + label_w: int, +) -> frozenset[int]: + """Draw all series into the shared grid (last writer wins on collision). + + Coordinate system: y_grid index 0 is the *bottom* of the data range, but + grid row 0 is the *top* of the terminal output. The conversion is: + screen_row = len(grid) - y_grid_index - 1 + So a higher y_grid index means a higher data value and a *lower* screen row. + + Out-of-band sentinels (``_NEG_INF_SENTINEL``, ``_POS_INF_SENTINEL``) signal + non-finite source values: + + * ``_NEG_INF_SENTINEL`` (-inf / NaN): line descends to the x-axis border. + The set of affected plot-body column indices is returned so + ``_draw_x_axis`` can mark them with ``┴``. + * ``_POS_INF_SENTINEL`` (+inf): line spikes to the top data row (row 0). + """ + height = len(grid) + border_cols: set[int] = set() + offset = label_w + 1 + width = len(grid[0]) + for style, pv in zip(styles, quantized_ys): + # We look one column ahead (pv[col+1]), so stop one short of the end. + for col_idx in range(width - label_w - 2): + cur = pv[col_idx] + nxt = pv[col_idx + 1] + col = col_idx + offset + + cur_is_neg_inf = cur == _NEG_INF_SENTINEL + nxt_is_neg_inf = nxt == _NEG_INF_SENTINEL + cur_is_pos_inf = cur == _POS_INF_SENTINEL + nxt_is_pos_inf = nxt == _POS_INF_SENTINEL + cur_is_nan = cur == _NAN_SENTINEL + nxt_is_nan = nxt == _NAN_SENTINEL + + # Two consecutive non-finite points of the same kind: nothing to draw. + if ( + (cur_is_neg_inf and nxt_is_neg_inf) + or (cur_is_pos_inf and nxt_is_pos_inf) + or (cur_is_nan and nxt_is_nan) + ): + continue + + screen_row = height - cur - 1 + next_screen_row = height - nxt - 1 + + # Recovering from border: │ up from bottom data row to nxt. + if cur_is_neg_inf: + border_cols.add(col_idx) + grid[next_screen_row][col] = "╭" + style_grid[next_screen_row][col] = style + for mid_row in range(next_screen_row + 1, height): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + continue + + # Descending to border: │ down from cur to bottom data row. + if nxt_is_neg_inf: + border_cols.add(col_idx) + grid[screen_row][col] = "╮" + style_grid[screen_row][col] = style + for mid_row in range(screen_row + 1, height): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + continue + + # Descending from top: │ down from row 0 to nxt. + if cur_is_pos_inf: + grid[0][col] = "│" + style_grid[0][col] = style + for mid_row in range(1, next_screen_row): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + grid[next_screen_row][col] = "╰" + style_grid[next_screen_row][col] = style + continue + + # Ascending to top: │ up from cur to row 0. + if nxt_is_pos_inf: + grid[screen_row][col] = "╯" + style_grid[screen_row][col] = style + for mid_row in range(1, screen_row): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + grid[0][col] = "│" + style_grid[0][col] = style + continue + + # Continue previous line if the next one is NaN + if not cur_is_nan and nxt_is_nan: + grid[screen_row][col] = "─" + continue + + # Start a new line if the current one is nan, but the previous one is not + if cur_is_nan and not nxt_is_nan: + grid[next_screen_row][col] = "─" + continue + + # If everything is finite and good, compare the values and add horizontal line or increasing/decreasing line + if screen_row == next_screen_row: + grid[screen_row][col] = "─" + style_grid[screen_row][col] = style + continue + + going_down = cur > nxt # value decreases → line goes down on screen + grid[screen_row][col] = "╮" if going_down else "╯" + style_grid[screen_row][col] = style + grid[next_screen_row][col] = "╰" if going_down else "╭" + style_grid[next_screen_row][col] = style + for mid_row in range(min(screen_row, next_screen_row) + 1, max(screen_row, next_screen_row)): + grid[mid_row][col] = "│" + style_grid[mid_row][col] = style + + return frozenset(border_cols) + + +def _draw_x_axis( + grid: list[list[str]], + style_grid: list[list[str]], + label_w: int, + x_labels: list[tuple[int, str]], + nan_cols: frozenset[int] = frozenset(), +) -> None: + """Append the └───┬─── border row and tick label row to the grid. + + ``nan_cols`` is a set of plot-body column indices (0-based within the plot + body, i.e. not including the y-axis label area) where a NaN line descends + to the border. Those positions get ``┴`` instead of ``─``, or ``┼`` when + they coincide with an x-tick ``┬``. + """ + row_len = len(grid[0]) + width = row_len - label_w - 1 + + # Border row: spaces | └ | ─ … ┬ … ─ + tick_cols = {col for col, _ in x_labels} + border_chars = list("─" * width) + for col in tick_cols: + border_chars[col] = "┬" + + # Adding hitting lines to -inf to the border + for col in nan_cols: + if 0 <= col < width: + border_chars[col] = "┼" if col in tick_cols else "┴" + border_row = [" "] * label_w + ["└"] + border_chars + border_styles = [_STYLE_SECONDARY] * label_w + [_STYLE_ACCENT] + [_STYLE_ACCENT] * width + grid.append(border_row) + style_grid.append(border_styles) + + # Label row: tick strings centred under their tick column + label_row = [" "] * row_len + for col, lbl in x_labels: + start = label_w + 1 + col - len(lbl) // 2 + start = max(0, min(start, row_len - len(lbl))) + for i, ch in enumerate(lbl): + label_row[start + i] = ch + grid.append(label_row) + style_grid.append([_STYLE_SECONDARY] * row_len) + + +def _render_data_row( + row: list[str], + style_row: list[str], +) -> Text: + """Colorize one grid row, appending each character with its style.""" + text = Text() + for ch, style in zip(row, style_row): + text.append(ch, style=style) + text.append("\n") + return text + + +def _render_body( + grid: list[list[str]], + style_grid: list[list[str]], +) -> Text: + """Convert the finished grid into a Rich Text object.""" + text = Text() + for row, style_row in zip(grid, style_grid): + text.append_text(_render_data_row(row, style_row)) + return text + + +def _plot( + xs: list[float], + ys: dict[str, list[float]], + *, + width: int = 60, + height: int = 6, + x_label: Callable[[float], str] = str, + y_label: Callable[[float], str] = str, + y_log: bool = False, + n_xticks: int = 3, + label_width: int = 8, +) -> Text: + """Render one or more named y series against a shared x-axis as an ASCII chart. + + Args: + xs: Shared x values for all series. + ys: Mapping of name → y values (must be same length as xs). + width: Number of character columns in the plot body. + height: Number of character rows in the chart body. + x_label: Callable that formats an x value into a tick-label string. + y_label: Callable that formats a y value into a tick-label string. + y_log: When True, values are plotted on a log10 axis. + n_xticks: Number of tick marks and labels on the x-axis (default 3). + label_width: Cap on the y-axis label column width (default 8). + Labels longer than this are truncated with ``...``. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not ys: + t = Text() + t.append("No data.", style=_STYLE_MUTED) + return t + + ordered_styles = [_SERIES_STYLES[i % len(_SERIES_STYLES)] for i in range(len(ys))] + + x_grid = _uniform_grid(xs, width) + interpolated_ys = _interpolate(xs, ys, x_grid) + if y_log: + interpolated_ys = _log_transform(interpolated_ys) + flat_ys = [v for ys_list in interpolated_ys.values() for v in ys_list] + y_grid = _uniform_grid(flat_ys, height) + + quantized_ys = _quantize_ys(interpolated_ys, y_grid) + y_labels = _y_labels(y_grid, y_log, y_label) + x_labels = _x_labels(x_grid, n_xticks, x_label) + + grid: list[list[str]] = [[" "] * (width + label_width + 1) for _ in range(height)] + style_grid: list[list[str]] = [[_STYLE_PRIMARY] * (width + label_width + 1) for _ in range(height)] + + _draw_y_axis(grid, style_grid, y_labels, label_width) + nan_cols = _draw_lines(grid, style_grid, quantized_ys, ordered_styles, label_width) + _draw_x_axis(grid, style_grid, label_width, x_labels, nan_cols) + + text = _render_body(grid, style_grid) + return text + + +def render_sparklines( + name: str, + xs: list[float], + ys: list[float], + *, + width: int = 60, + y_log: bool = False, + label_width: int = 8, +) -> Text: + """Render a single block-sparkline row for one series. + + Call once per series, passing a shared ``label_width`` across all calls to + keep label columns aligned. The name is right-justified within the column; + names longer than ``label_width`` are truncated with ``...``. + + Args: + name: Series name, used as the row label. + xs: X values (e.g. training steps). + ys: Y values. + width: Sparkline character width (default 60). + y_log: When True, plot on a log10 scale (default False). + label_width: Exact label column width (default 8). Pass the same + value to every call in a group to get consistent + alignment. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not xs: + t = Text() + t.append("No plottable data.", style=_STYLE_MUTED) + return t + + x_grid = _uniform_grid(xs, width) + interpolated = _interpolate(xs, {name: ys}, x_grid) + if y_log: + interpolated = _log_transform(interpolated) + + series_vals = interpolated[name] + y_grid = _uniform_grid(series_vals, len(_SPARK_BLOCKS)) + quantized = _quantize_ys({name: series_vals}, y_grid)[0] + + label = _fit_spark_label(name, label_width) + + # The sentinel value (len(y_grid)) indicates a NaN data point; render it + # as a space (the lowest sparkline block) since sparklines have no border row. + # Map out-of-band sentinels to the extreme sparkline blocks: + # _NEG_INF_SENTINEL (-inf) or _NAN_SENTINEL (NaN) → space (lowest block, index 0) + # _POS_INF_SENTINEL (+inf) → █ (highest block, last index) + def _spark_block(idx: int) -> str: + if idx == _NEG_INF_SENTINEL or idx == _NAN_SENTINEL: + return _SPARK_BLOCKS[0] + if idx == _POS_INF_SENTINEL: + return _SPARK_BLOCKS[-1] + return _SPARK_BLOCKS[idx] + + spark = "".join(_spark_block(idx) for idx in quantized).ljust(width) + + text = Text() + text.append(f" {label} ", style=_STYLE_MUTED) + text.append(spark, style=_STYLE_SPARK) + text.append(f" {ys[0]:.4g} → {ys[-1]:.4g}", style=_STYLE_SECONDARY) + text.append("\n") + return text + + +def render_line_chart( + xs: list[float], + ys: dict[str, list[float]], + *, + x_label: Callable[[float], str] = str, + y_log: bool = False, + y_label: Callable[[float], str] | None = None, + width: int = 60, + height: int = 6, + n_xticks: int = 3, + label_width: int = 8, +) -> Text: + """Render one or more named series as a shared ASCII line chart with a legend header. + + All series share the same x-axis (``xs``); each has its own named y values:: + + console.print( + render_line_chart( + steps, + {"train_loss": train_losses, "val_loss": val_losses}, + x_label=lambda s: f"step {s:.0f}", + ) + ) + + Args: + xs: Shared x values for all series. + ys: Mapping of name → y values. + x_label: Callable that formats an x value into a tick-label string. + y_log: When True, plot on a log10 y-axis (default False). + y_label: Callable that formats a y value into a tick-label string. + width: Plot width in terminal characters (default 60). + height: Plot height in terminal rows (default 6). + n_xticks: Number of x-axis tick marks and labels (default 3). + label_width: Cap on the y-axis label column width. + + Returns: + A ``rich.text.Text`` ready for ``console.print()``. + """ + if not ys: + t = Text() + t.append("No plottable data.", style=_STYLE_MUTED) + return t + + styles = {key: _SERIES_STYLES[i % len(_SERIES_STYLES)] for i, key in enumerate(ys)} + + text = Text() + x_from = x_label(xs[0]) + x_to = x_label(xs[-1]) + for key in ys: + text.append( + f" {key} ({x_from} – {x_to}) {ys[key][0]:.4g} → {ys[key][-1]:.4g}\n", + style=styles[key], + ) + + text.append_text( + _plot( + xs, + ys, + width=width, + height=height, + x_label=x_label, + y_label=y_label or (lambda v: f"{v:.3g}"), + y_log=y_log, + n_xticks=n_xticks, + label_width=label_width, + ) + ) + return text From ad7f85c44ab04bce42a69c37f8dae9a578ca82a4 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 10:00:28 +0200 Subject: [PATCH 06/16] revert --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index abe93dbc1..a10f2d912 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "together" -version = "2.14.0" +version = "2.12.0" source = { editable = "." } dependencies = [ { name = "anyio" }, From 6af55964fbd049514c5d48f25ed2db56c2a6fe9d Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 10:09:06 +0200 Subject: [PATCH 07/16] final fixes --- metrics.json | 1 - src/together/lib/cli/api/fine_tuning/retrieve.py | 2 +- src/together/lib/cli/components/plots/_engine.py | 7 ++++--- src/together/lib/cli/utils/_help_examples.py | 3 +++ 4 files changed, 8 insertions(+), 5 deletions(-) delete mode 100644 metrics.json diff --git a/metrics.json b/metrics.json deleted file mode 100644 index fe692c818..000000000 --- a/metrics.json +++ /dev/null @@ -1 +0,0 @@ -[{"timestamp":1.7770314739931374e+18,"train/loss":0.5725,"train/epoch":0.047619047619047616,"train/grad_norm":1.5748692750930786,"train/global_step":1.0,"train/learning_rate":1e-05},{"timestamp":1.7770315379636355e+18,"train/loss":0.5267,"train/epoch":0.14285714285714285,"train/grad_norm":1.4396071434020996,"train/global_step":3.0,"train/learning_rate":9.991050648838676e-06},{"timestamp":1.7770315693690883e+18,"train/loss":0.5325,"train/epoch":0.19047619047619047,"train/grad_norm":1.0482614040374756,"train/global_step":4.0,"train/learning_rate":9.979871469976197e-06},{"timestamp":1.777031600296171e+18,"train/loss":0.4901,"train/epoch":0.23809523809523808,"train/grad_norm":1.0096107721328735,"train/global_step":5.0,"train/learning_rate":9.964234631709188e-06},{"timestamp":1.7770316263251743e+18,"train/loss":0.4648,"train/epoch":0.2857142857142857,"train/grad_norm":1.0371627807617188,"train/global_step":6.0,"train/learning_rate":9.944154131125643e-06},{"timestamp":1.7770316556595707e+18,"train/loss":0.4851,"train/epoch":0.3333333333333333,"train/grad_norm":0.7985115647315979,"train/global_step":7.0,"train/learning_rate":9.91964794299315e-06},{"timestamp":1.777031656171934e+18,"train/loss":0.417,"train/epoch":0.38095238095238093,"train/grad_norm":0.6295253038406372,"train/global_step":8.0,"train/learning_rate":9.890738003669029e-06},{"timestamp":1.777031682139413e+18,"train/loss":0.4299,"train/epoch":0.42857142857142855,"train/grad_norm":0.5814775228500366,"train/global_step":9.0,"train/learning_rate":9.857450191464337e-06},{"timestamp":1.7770317086595318e+18,"train/loss":0.4688,"train/epoch":0.47619047619047616,"train/grad_norm":0.5624698400497437,"train/global_step":10.0,"train/learning_rate":9.819814303479268e-06},{"timestamp":1.7770317090552543e+18,"train/loss":0.442,"train/epoch":0.5238095238095238,"train/grad_norm":0.4839235544204712,"train/global_step":11.0,"train/learning_rate":9.777864028930705e-06},{"timestamp":1.77703173877343e+18,"train/loss":0.4521,"train/epoch":0.5714285714285714,"train/grad_norm":0.41900405287742615,"train/global_step":12.0,"train/learning_rate":9.731636918995821e-06},{"timestamp":1.7770317671818757e+18,"train/loss":0.4293,"train/epoch":0.6190476190476191,"train/grad_norm":0.35965147614479065,"train/global_step":13.0,"train/learning_rate":9.681174353198687e-06},{"timestamp":1.7770317675498414e+18,"train/loss":0.4072,"train/epoch":0.6666666666666666,"train/grad_norm":0.34983327984809875,"train/global_step":14.0,"train/learning_rate":9.626521502369984e-06},{"timestamp":1.7770317960450604e+18,"train/loss":0.4075,"train/epoch":0.7619047619047619,"train/grad_norm":0.3408649265766144,"train/global_step":16.0,"train/learning_rate":9.504844339512096e-06},{"timestamp":1.7770318221638467e+18,"train/loss":0.3632,"train/epoch":0.8095238095238095,"train/grad_norm":0.29929089546203613,"train/global_step":17.0,"train/learning_rate":9.437928945022772e-06},{"timestamp":1.7770318491147359e+18,"train/loss":0.444,"train/epoch":0.8571428571428571,"train/grad_norm":0.30117282271385193,"train/global_step":18.0,"train/learning_rate":9.36704100308565e-06},{"timestamp":1.777031849517807e+18,"train/loss":0.4315,"train/epoch":0.9047619047619048,"train/grad_norm":0.3524363040924072,"train/global_step":19.0,"train/learning_rate":9.292243968009332e-06},{"timestamp":1.777031876918584e+18,"train/loss":0.4081,"train/epoch":0.9523809523809523,"train/grad_norm":0.27605122327804565,"train/global_step":20.0,"train/learning_rate":9.213604793270196e-06},{"timestamp":1.7770319047202714e+18,"train/loss":0.4143,"train/epoch":1.0,"train/grad_norm":0.30752140283584595,"train/global_step":21.0,"train/learning_rate":9.131193871579975e-06},{"eval/loss":5.939453125,"timestamp":1.7770319182683062e+18,"train/epoch":1.0,"train/global_step":21.0},{"timestamp":1.7770319186845947e+18,"train/loss":0.3546,"train/epoch":1.0476190476190477,"train/grad_norm":0.3130515217781067,"train/global_step":22.0,"train/learning_rate":9.045084971874738e-06},{"timestamp":1.7770319191048975e+18,"train/loss":0.3728,"train/epoch":1.0952380952380953,"train/grad_norm":0.29933640360832214,"train/global_step":23.0,"train/learning_rate":8.955355173281709e-06},{"timestamp":1.7770319195131405e+18,"train/loss":0.3843,"train/epoch":1.1428571428571428,"train/grad_norm":0.26397213339805603,"train/global_step":24.0,"train/learning_rate":8.862084796122998e-06},{"timestamp":1.777031920588432e+18,"train/loss":0.4,"train/epoch":1.1904761904761905,"train/grad_norm":0.33784887194633484,"train/global_step":25.0,"train/learning_rate":8.765357330018056e-06},{"timestamp":1.7770319209921388e+18,"train/loss":0.3812,"train/epoch":1.2380952380952381,"train/grad_norm":0.32685449719429016,"train/global_step":26.0,"train/learning_rate":8.665259359149132e-06},{"timestamp":1.7770319213974446e+18,"train/loss":0.3792,"train/epoch":1.2857142857142856,"train/grad_norm":0.28974828124046326,"train/global_step":27.0,"train/learning_rate":8.561880484756726e-06},{"timestamp":1.777031922236854e+18,"train/loss":0.3536,"train/epoch":1.380952380952381,"train/grad_norm":0.26806962490081787,"train/global_step":29.0,"train/learning_rate":8.345653031794292e-06},{"timestamp":1.7770319226348698e+18,"train/loss":0.3622,"train/epoch":1.4285714285714286,"train/grad_norm":0.3048115670681,"train/global_step":30.0,"train/learning_rate":8.232998006078998e-06},{"timestamp":1.7770319229918449e+18,"train/loss":0.4069,"train/epoch":1.4761904761904763,"train/grad_norm":0.31159284710884094,"train/global_step":31.0,"train/learning_rate":8.117449009293668e-06},{"timestamp":1.7770319233036634e+18,"train/loss":0.3862,"train/epoch":1.5238095238095237,"train/grad_norm":0.2849102318286896,"train/global_step":32.0,"train/learning_rate":7.99910947343957e-06},{"timestamp":1.777031923574597e+18,"train/loss":0.4019,"train/epoch":1.5714285714285714,"train/grad_norm":0.3013054430484772,"train/global_step":33.0,"train/learning_rate":7.87808532842837e-06},{"timestamp":1.7770319238637245e+18,"train/loss":0.3843,"train/epoch":1.619047619047619,"train/grad_norm":0.2755144238471985,"train/global_step":34.0,"train/learning_rate":7.754484907260513e-06},{"timestamp":1.7770319244743252e+18,"train/loss":0.3647,"train/epoch":1.6666666666666665,"train/grad_norm":0.31459057331085205,"train/global_step":35.0,"train/learning_rate":7.628418849052523e-06},{"timestamp":1.7770319250751076e+18,"train/loss":0.3413,"train/epoch":1.7142857142857144,"train/grad_norm":0.2461976408958435,"train/global_step":36.0,"train/learning_rate":7.500000000000001e-06},{"timestamp":1.7770319253530952e+18,"train/loss":0.3684,"train/epoch":1.7619047619047619,"train/grad_norm":0.3002825677394867,"train/global_step":37.0,"train/learning_rate":7.369343312364994e-06},{"timestamp":1.7770319256612006e+18,"train/loss":0.334,"train/epoch":1.8095238095238095,"train/grad_norm":0.25984328985214233,"train/global_step":38.0,"train/learning_rate":7.236565741578163e-06},{"timestamp":1.7770319259561108e+18,"train/loss":0.4156,"train/epoch":1.8571428571428572,"train/grad_norm":0.2751716077327728,"train/global_step":39.0,"train/learning_rate":7.101786141547829e-06},{"timestamp":1.7770319262290227e+18,"train/loss":0.3953,"train/epoch":1.9047619047619047,"train/grad_norm":0.2947269380092621,"train/global_step":40.0,"train/learning_rate":6.965125158269619e-06},{"timestamp":1.7770319268452413e+18,"train/loss":0.3856,"train/epoch":2.0,"train/grad_norm":0.269094854593277,"train/global_step":42.0,"train/learning_rate":6.686649936914151e-06},{"eval/loss":5.833984375,"timestamp":1.777031927850314e+18,"train/epoch":2.0,"train/global_step":42.0},{"timestamp":1.7770319281532844e+18,"train/loss":0.3284,"train/epoch":2.0476190476190474,"train/grad_norm":0.27204498648643494,"train/global_step":43.0,"train/learning_rate":6.545084971874738e-06},{"timestamp":1.7770319284494723e+18,"train/loss":0.3508,"train/epoch":2.0952380952380953,"train/grad_norm":0.2703777849674225,"train/global_step":44.0,"train/learning_rate":6.402136946530014e-06},{"timestamp":1.7770319287519683e+18,"train/loss":0.3624,"train/epoch":2.142857142857143,"train/grad_norm":0.25161707401275635,"train/global_step":45.0,"train/learning_rate":6.257933818722544e-06},{"timestamp":1.7770319290746417e+18,"train/loss":0.3719,"train/epoch":2.1904761904761907,"train/grad_norm":0.30092841386795044,"train/global_step":46.0,"train/learning_rate":6.112604669781572e-06},{"timestamp":1.7770319293780872e+18,"train/loss":0.3538,"train/epoch":2.238095238095238,"train/grad_norm":0.30388522148132324,"train/global_step":47.0,"train/learning_rate":5.9662795889777666e-06},{"timestamp":1.7770319296753797e+18,"train/loss":0.3568,"train/epoch":2.2857142857142856,"train/grad_norm":0.2648054361343384,"train/global_step":48.0,"train/learning_rate":5.819089557075689e-06},{"timestamp":1.7770319303294776e+18,"train/loss":0.4008,"train/epoch":2.3333333333333335,"train/grad_norm":0.25187763571739197,"train/global_step":49.0,"train/learning_rate":5.671166329088278e-06},{"timestamp":1.777031930678494e+18,"train/loss":0.3353,"train/epoch":2.380952380952381,"train/grad_norm":0.26126545667648315,"train/global_step":50.0,"train/learning_rate":5.522642316338268e-06},{"timestamp":1.777031931009004e+18,"train/loss":0.3403,"train/epoch":2.4285714285714284,"train/grad_norm":0.29477638006210327,"train/global_step":51.0,"train/learning_rate":5.373650467932122e-06},{"timestamp":1.7770319313563054e+18,"train/loss":0.3843,"train/epoch":2.4761904761904763,"train/grad_norm":0.3020482659339905,"train/global_step":52.0,"train/learning_rate":5.224324151752575e-06},{"timestamp":1.7770319316935217e+18,"train/loss":0.3658,"train/epoch":2.5238095238095237,"train/grad_norm":0.2735633850097656,"train/global_step":53.0,"train/learning_rate":5.074797035076319e-06},{"timestamp":1.77703193236958e+18,"train/loss":0.3667,"train/epoch":2.619047619047619,"train/grad_norm":0.2675466239452362,"train/global_step":55.0,"train/learning_rate":4.775675848247427e-06},{"timestamp":1.777031932715636e+18,"train/loss":0.3463,"train/epoch":2.6666666666666665,"train/grad_norm":0.30449649691581726,"train/global_step":56.0,"train/learning_rate":4.626349532067879e-06},{"timestamp":1.777031933071996e+18,"train/loss":0.3292,"train/epoch":2.7142857142857144,"train/grad_norm":0.24315503239631653,"train/global_step":57.0,"train/learning_rate":4.477357683661734e-06},{"timestamp":1.7770319337252216e+18,"train/loss":0.3512,"train/epoch":2.761904761904762,"train/grad_norm":0.2935601472854614,"train/global_step":58.0,"train/learning_rate":4.3288336709117246e-06},{"timestamp":1.7770319340828797e+18,"train/loss":0.3204,"train/epoch":2.8095238095238093,"train/grad_norm":0.2543131113052368,"train/global_step":59.0,"train/learning_rate":4.180910442924312e-06},{"timestamp":1.7770319344318339e+18,"train/loss":0.4008,"train/epoch":2.857142857142857,"train/grad_norm":0.2751201093196869,"train/global_step":60.0,"train/learning_rate":4.033720411022235e-06},{"timestamp":1.7770319350886285e+18,"train/loss":0.3774,"train/epoch":2.9047619047619047,"train/grad_norm":0.30817028880119324,"train/global_step":61.0,"train/learning_rate":3.887395330218429e-06},{"timestamp":1.7770319354579146e+18,"train/loss":0.3708,"train/epoch":2.9523809523809526,"train/grad_norm":0.2573518455028534,"train/global_step":62.0,"train/learning_rate":3.7420661812774577e-06},{"timestamp":1.777031936142077e+18,"train/loss":0.3719,"train/epoch":3.0,"train/grad_norm":0.2749069929122925,"train/global_step":63.0,"train/learning_rate":3.5978630534699873e-06},{"eval/loss":5.82421875,"timestamp":1.777031937157716e+18,"train/epoch":3.0,"train/global_step":63.0},{"timestamp":1.7770319374588646e+18,"train/loss":0.3154,"train/epoch":3.0476190476190474,"train/grad_norm":0.2740519940853119,"train/global_step":64.0,"train/learning_rate":3.4549150281252635e-06},{"timestamp":1.7770319377837844e+18,"train/loss":0.3397,"train/epoch":3.0952380952380953,"train/grad_norm":0.2745459973812103,"train/global_step":65.0,"train/learning_rate":3.3133500630858507e-06},{"timestamp":1.7770319380782636e+18,"train/loss":0.353,"train/epoch":3.142857142857143,"train/grad_norm":0.24551734328269958,"train/global_step":66.0,"train/learning_rate":3.173294878168025e-06},{"timestamp":1.7770319386512013e+18,"train/loss":0.3402,"train/epoch":3.238095238095238,"train/grad_norm":0.301933616399765,"train/global_step":68.0,"train/learning_rate":2.8982138584521734e-06},{"timestamp":1.777031938949057e+18,"train/loss":0.3459,"train/epoch":3.2857142857142856,"train/grad_norm":0.2628892660140991,"train/global_step":69.0,"train/learning_rate":2.7634342584218364e-06},{"timestamp":1.7770319392524636e+18,"train/loss":0.3926,"train/epoch":3.3333333333333335,"train/grad_norm":0.25296470522880554,"train/global_step":70.0,"train/learning_rate":2.6306566876350072e-06},{"timestamp":1.7770319395400535e+18,"train/loss":0.3273,"train/epoch":3.380952380952381,"train/grad_norm":0.2628619372844696,"train/global_step":71.0,"train/learning_rate":2.5000000000000015e-06},{"timestamp":1.777031939819372e+18,"train/loss":0.3311,"train/epoch":3.4285714285714284,"train/grad_norm":0.2902317941188812,"train/global_step":72.0,"train/learning_rate":2.371581150947476e-06},{"timestamp":1.7770319403707064e+18,"train/loss":0.3748,"train/epoch":3.4761904761904763,"train/grad_norm":0.30575087666511536,"train/global_step":73.0,"train/learning_rate":2.245515092739488e-06},{"timestamp":1.777031940655665e+18,"train/loss":0.3573,"train/epoch":3.5238095238095237,"train/grad_norm":0.27393031120300293,"train/global_step":74.0,"train/learning_rate":2.1219146715716332e-06},{"timestamp":1.777031940933279e+18,"train/loss":0.374,"train/epoch":3.571428571428571,"train/grad_norm":0.29484593868255615,"train/global_step":75.0,"train/learning_rate":2.0008905265604316e-06},{"timestamp":1.7770319412286344e+18,"train/loss":0.3587,"train/epoch":3.619047619047619,"train/grad_norm":0.27231061458587646,"train/global_step":76.0,"train/learning_rate":1.8825509907063328e-06},{"timestamp":1.7770319415158525e+18,"train/loss":0.3385,"train/epoch":3.6666666666666665,"train/grad_norm":0.3046872913837433,"train/global_step":77.0,"train/learning_rate":1.7670019939210025e-06},{"timestamp":1.777031941878367e+18,"train/loss":0.3229,"train/epoch":3.7142857142857144,"train/grad_norm":0.24565838277339935,"train/global_step":78.0,"train/learning_rate":1.6543469682057105e-06},{"timestamp":1.777031942242403e+18,"train/loss":0.344,"train/epoch":3.761904761904762,"train/grad_norm":0.29259249567985535,"train/global_step":79.0,"train/learning_rate":1.544686755065677e-06},{"timestamp":1.7770319429027528e+18,"train/loss":0.3933,"train/epoch":3.857142857142857,"train/grad_norm":0.2701041102409363,"train/global_step":81.0,"train/learning_rate":1.3347406408508695e-06},{"timestamp":1.777031943485962e+18,"train/loss":0.3705,"train/epoch":3.9047619047619047,"train/grad_norm":0.3030908405780792,"train/global_step":82.0,"train/learning_rate":1.234642669981946e-06},{"timestamp":1.7770319437911675e+18,"train/loss":0.3651,"train/epoch":3.9523809523809526,"train/grad_norm":0.25513312220573425,"train/global_step":83.0,"train/learning_rate":1.137915203877003e-06},{"timestamp":1.7770319441107953e+18,"train/loss":0.367,"train/epoch":4.0,"train/grad_norm":0.27516013383865356,"train/global_step":84.0,"train/learning_rate":1.044644826718295e-06},{"eval/loss":5.82421875,"timestamp":1.777031945116394e+18,"train/epoch":4.0,"train/global_step":84.0},{"timestamp":1.777031945404118e+18,"train/loss":0.3101,"train/epoch":4.0476190476190474,"train/grad_norm":0.2693224251270294,"train/global_step":85.0,"train/learning_rate":9.549150281252633e-07},{"timestamp":1.7770319456981092e+18,"train/loss":0.3345,"train/epoch":4.095238095238095,"train/grad_norm":0.2737109363079071,"train/global_step":86.0,"train/learning_rate":8.688061284200266e-07},{"timestamp":1.7770319459895841e+18,"train/loss":0.3476,"train/epoch":4.142857142857143,"train/grad_norm":0.24587756395339966,"train/global_step":87.0,"train/learning_rate":7.863952067298042e-07},{"timestamp":1.77703194626355e+18,"train/loss":0.3522,"train/epoch":4.190476190476191,"train/grad_norm":0.3057315945625305,"train/global_step":88.0,"train/learning_rate":7.077560319906696e-07},{"timestamp":1.7770319468008266e+18,"train/loss":0.3336,"train/epoch":4.238095238095238,"train/grad_norm":0.3030700385570526,"train/global_step":89.0,"train/learning_rate":6.329589969143518e-07},{"timestamp":1.777031947416965e+18,"train/loss":0.342,"train/epoch":4.285714285714286,"train/grad_norm":0.2629696726799011,"train/global_step":90.0,"train/learning_rate":5.620710549772295e-07},{"timestamp":1.777031947766062e+18,"train/loss":0.3896,"train/epoch":4.333333333333333,"train/grad_norm":0.253886878490448,"train/global_step":91.0,"train/learning_rate":4.951556604879049e-07},{"timestamp":1.777031948056129e+18,"train/loss":0.3239,"train/epoch":4.380952380952381,"train/grad_norm":0.2629833519458771,"train/global_step":92.0,"train/learning_rate":4.322727117869951e-07},{"timestamp":1.7770319486236925e+18,"train/loss":0.3718,"train/epoch":4.476190476190476,"train/grad_norm":0.30251115560531616,"train/global_step":94.0,"train/learning_rate":3.18825646801314e-07},{"timestamp":1.7770319489037783e+18,"train/loss":0.3542,"train/epoch":4.523809523809524,"train/grad_norm":0.27201929688453674,"train/global_step":95.0,"train/learning_rate":2.6836308100417874e-07},{"timestamp":1.7770319491791521e+18,"train/loss":0.3717,"train/epoch":4.571428571428571,"train/grad_norm":0.2946451008319855,"train/global_step":96.0,"train/learning_rate":2.2213597106929608e-07},{"timestamp":1.77703194947258e+18,"train/loss":0.3558,"train/epoch":4.619047619047619,"train/grad_norm":0.2709577679634094,"train/global_step":97.0,"train/learning_rate":1.801856965207338e-07},{"timestamp":1.7770319497592973e+18,"train/loss":0.3358,"train/epoch":4.666666666666667,"train/grad_norm":0.3053297698497772,"train/global_step":98.0,"train/learning_rate":1.4254980853566248e-07},{"timestamp":1.7770319500497021e+18,"train/loss":0.3209,"train/epoch":4.714285714285714,"train/grad_norm":0.24520252645015717,"train/global_step":99.0,"train/learning_rate":1.0926199633097156e-07},{"timestamp":1.7770319503302083e+18,"train/loss":0.3419,"train/epoch":4.761904761904762,"train/grad_norm":0.2922167479991913,"train/global_step":100.0,"train/learning_rate":8.035205700685167e-08},{"timestamp":1.7770319506339753e+18,"train/loss":0.3129,"train/epoch":4.809523809523809,"train/grad_norm":0.25622737407684326,"train/global_step":101.0,"train/learning_rate":5.584586887435739e-08},{"timestamp":1.7770319509281615e+18,"train/loss":0.3922,"train/epoch":4.857142857142857,"train/grad_norm":0.2694510519504547,"train/global_step":102.0,"train/learning_rate":3.576536829081323e-08},{"timestamp":1.777031951209152e+18,"train/loss":0.3677,"train/epoch":4.904761904761905,"train/grad_norm":0.29987600445747375,"train/global_step":103.0,"train/learning_rate":2.012853002380466e-08},{"timestamp":1.7770319517874483e+18,"train/loss":0.3643,"train/epoch":4.9523809523809526,"train/grad_norm":0.2551063597202301,"train/global_step":104.0,"train/learning_rate":8.949351161324227e-09},{"timestamp":1.7770319521104128e+18,"train/loss":0.3647,"train/epoch":5.0,"train/grad_norm":0.2752157151699066,"train/global_step":105.0,"train/learning_rate":2.237838582483387e-09},{"eval/loss":5.82421875,"timestamp":1.777031953116419e+18,"train/epoch":5.0,"train/global_step":105.0}] \ No newline at end of file diff --git a/src/together/lib/cli/api/fine_tuning/retrieve.py b/src/together/lib/cli/api/fine_tuning/retrieve.py index dcfe80192..0b77997e0 100644 --- a/src/together/lib/cli/api/fine_tuning/retrieve.py +++ b/src/together/lib/cli/api/fine_tuning/retrieve.py @@ -21,7 +21,7 @@ async def retrieve( fine_tune_id: str, *, config: CLIConfigParameter, - no_plots: Annotated[bool, Parameter(help="Print training metric sparklines.")] = False, + no_plots: Annotated[bool, Parameter(help="Print training metric sparklines.", negative=())] = False, ) -> None: """Retrieve fine-tuning job details.""" response = await show_loading_status( diff --git a/src/together/lib/cli/components/plots/_engine.py b/src/together/lib/cli/components/plots/_engine.py index c3f9f1ab4..474d16cc7 100644 --- a/src/together/lib/cli/components/plots/_engine.py +++ b/src/together/lib/cli/components/plots/_engine.py @@ -134,10 +134,11 @@ def _quantize_ys( Non-finite values are mapped to out-of-band sentinels: - * ``_POS_INF_SENTINEL`` (``-1``) for ``+inf`` — the line spikes to the top + * ``_NEG_INF_SENTINEL`` (``-1``) for ``-inf`` — the line descends to the + x-axis border row. + * ``_POS_INF_SENTINEL`` (``-2``) for ``+inf`` — the line spikes to the top data row. - * ``_NEG_INF_SENTINEL`` (``-2``) for ``-inf`` / ``NaN`` — the line - descends to the x-axis border row. + * ``_NAN_SENTINEL`` (``-3``) for ``NaN`` — no line is drawn at that point. """ quantized_ys: list[list[int]] = [] for ys in interpolated_ys.values(): diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index 619957328..fed4c5605 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -112,6 +112,9 @@ [dim]-[/dim] Retrieve a fixed number of data points as JSON: [primary]tg ft list-metrics --resolution 50 --json[/primary] + +[dim]-[/dim] Save metrics to a file: + [primary]tg ft list-metrics --save ./metrics.json[/primary] """ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] From 54583a7bf49a7311babe4a5bf36153ecd90c5b99 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 15:46:32 +0200 Subject: [PATCH 08/16] remove save --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 8 +------- src/together/lib/cli/utils/_help_examples.py | 3 --- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index a45c2c8fb..0c807fe82 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -30,7 +30,6 @@ async def list_metrics( Optional[int], Parameter(help="Number of training metric points to return. Does not limit the number of eval metric points."), ] = None, - save: Annotated[Optional[Path], Parameter("--save", help="Save metrics to a file as JSON.")] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" @@ -48,14 +47,9 @@ async def list_metrics( ) metrics = response.metrics or [] - json_bytes = openapi_dumps(metrics) - - if save is not None: - save.write_bytes(json_bytes) - console.print(f"[success]Metrics saved to {save}[/success]") - return if config.json: + json_bytes = openapi_dumps(metrics) console.print_json(json_bytes.decode("utf-8")) return diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index fed4c5605..619957328 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -112,9 +112,6 @@ [dim]-[/dim] Retrieve a fixed number of data points as JSON: [primary]tg ft list-metrics --resolution 50 --json[/primary] - -[dim]-[/dim] Save metrics to a file: - [primary]tg ft list-metrics --save ./metrics.json[/primary] """ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] From 0cee80a569808e7c0f257a1adeb4eb316a2a98fd Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Fri, 15 May 2026 15:48:05 +0200 Subject: [PATCH 09/16] feedback fixes --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 2 +- src/together/lib/cli/components/plots/_engine.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 0c807fe82..c817c8b8d 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -28,7 +28,7 @@ async def list_metrics( logged_at_to: Annotated[Optional[datetime], Parameter(help="Filter metrics logged at or before this time.")] = None, resolution: Annotated[ Optional[int], - Parameter(help="Number of training metric points to return. Does not limit the number of eval metric points."), + Parameter(help="Number of uniformly sampled training metric points to return. Does not limit the number of eval metric points."), ] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" diff --git a/src/together/lib/cli/components/plots/_engine.py b/src/together/lib/cli/components/plots/_engine.py index 474d16cc7..62c2f6738 100644 --- a/src/together/lib/cli/components/plots/_engine.py +++ b/src/together/lib/cli/components/plots/_engine.py @@ -48,8 +48,8 @@ def should_log(vals: list[float]) -> bool: """Return True when values span more than 100×, suggesting log scale.""" - nz = [v for v in vals if v > 0] - return len(nz) > 1 and (max(nz) / min(nz)) > 100 + positive_val = [v for v in vals if v > 0] + return len(positive_val) > 1 and (max(positive_val) / min(positive_val)) > 100 def _uniform_grid(vals: list[float], n: int) -> list[float]: @@ -58,8 +58,8 @@ def _uniform_grid(vals: list[float], n: int) -> list[float]: Non-finite values (e.g. the -inf sentinel used for NaN data points) are excluded from the range computation so they don't corrupt the grid. """ - finite = [v for v in vals if math.isfinite(v)] - min_val, max_val = min(finite), max(finite) + finite_val = [v for v in vals if math.isfinite(v)] + min_val, max_val = min(finite_val), max(finite_val) if n <= 1: return [min_val] return [min_val + (max_val - min_val) * idx / (n - 1) for idx in range(n)] From 528956b03576a1b168b590b7fb1061666c1f09e2 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Tue, 19 May 2026 13:54:51 +0200 Subject: [PATCH 10/16] add tty detector --- .../lib/cli/api/fine_tuning/list_metrics.py | 28 +++++++++++++++---- src/together/lib/cli/utils/_help_examples.py | 6 ++++ uv.lock | 2 +- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index c817c8b8d..f32945ab9 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -1,7 +1,7 @@ from __future__ import annotations +import sys from typing import Optional, Annotated -from pathlib import Path from datetime import datetime from cyclopts import Parameter @@ -28,12 +28,26 @@ async def list_metrics( logged_at_to: Annotated[Optional[datetime], Parameter(help="Filter metrics logged at or before this time.")] = None, resolution: Annotated[ Optional[int], - Parameter(help="Number of uniformly sampled training metric points to return. Does not limit the number of eval metric points."), + Parameter( + help="Number of uniformly sampled training metric points to return. Does not limit the number of eval metric points." + ), ] = None, + force_plots: Annotated[ + bool, + Parameter( + "--force-plots", + help="Force rendering ASCII plots even when stdout is not a terminal (e.g. when redirecting output to a file).", + ), + ] = False, ) -> None: """Retrieve training metrics for a fine-tuning job.""" - resolution_value = resolution if config.json else console.width - METRICS_WIDTH_PADDING + is_tty = sys.stdout.isatty() + # Show plots only when writing to a real terminal (or --force-plots is set) and --json wasn't requested. + # When stdout is redirected (e.g. > file.txt or | jq), is_tty is False and we fall back to raw JSON. + show_plots = (is_tty or force_plots) and not config.json + + resolution_value = resolution if not show_plots else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( @@ -48,9 +62,13 @@ async def list_metrics( metrics = response.metrics or [] - if config.json: + if not show_plots: json_bytes = openapi_dumps(metrics) - console.print_json(json_bytes.decode("utf-8")) + if config.json: + console.print_json(json_bytes.decode("utf-8")) + else: + # stdout is redirected — print raw JSON so it pipes cleanly + sys.stdout.write(json_bytes.decode("utf-8") + "\n") return if not metrics: diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index 619957328..01eb4ae8a 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -112,6 +112,12 @@ [dim]-[/dim] Retrieve a fixed number of data points as JSON: [primary]tg ft list-metrics --resolution 50 --json[/primary] + +[dim]-[/dim] Save raw metrics to a file (plots are disabled automatically when redirecting): + [primary]tg ft list-metrics > metrics.json[/primary] + +[dim]-[/dim] Save ASCII plots to a file: + [primary]tg ft list-metrics --force-plots > plots.txt[/primary] """ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] diff --git a/uv.lock b/uv.lock index a10f2d912..abe93dbc1 100644 --- a/uv.lock +++ b/uv.lock @@ -1559,7 +1559,7 @@ wheels = [ [[package]] name = "together" -version = "2.12.0" +version = "2.14.0" source = { editable = "." } dependencies = [ { name = "anyio" }, From 2fb9a362ba4004a3e6273747a72b96f1521cca0d Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 16:54:20 +0200 Subject: [PATCH 11/16] make resolution affect plots --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index f32945ab9..755913e4a 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -47,7 +47,7 @@ async def list_metrics( # When stdout is redirected (e.g. > file.txt or | jq), is_tty is False and we fall back to raw JSON. show_plots = (is_tty or force_plots) and not config.json - resolution_value = resolution if not show_plots else console.width - METRICS_WIDTH_PADDING + resolution_value = resolution if resolution else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( From 5dbf05cf8d3655f8b40d4a5baccb660da41afc1e Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 17:11:46 +0200 Subject: [PATCH 12/16] use --output flag --- .../lib/cli/api/fine_tuning/list_metrics.py | 24 +++++++++---------- src/together/lib/cli/utils/_help_examples.py | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 755913e4a..2cdd0611d 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -1,7 +1,7 @@ from __future__ import annotations import sys -from typing import Optional, Annotated +from typing import Literal, Optional, Annotated from datetime import datetime from cyclopts import Parameter @@ -32,22 +32,22 @@ async def list_metrics( help="Number of uniformly sampled training metric points to return. Does not limit the number of eval metric points." ), ] = None, - force_plots: Annotated[ - bool, + output: Annotated[ + Optional[Literal["json", "graph"]], Parameter( - "--force-plots", - help="Force rendering ASCII plots even when stdout is not a terminal (e.g. when redirecting output to a file).", + "--output", + help="Override the output format. 'json' prints raw JSON, 'graph' renders ASCII plots. By default the format is chosen automatically based on whether stdout is a terminal.", ), - ] = False, + ] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" is_tty = sys.stdout.isatty() - # Show plots only when writing to a real terminal (or --force-plots is set) and --json wasn't requested. + # Determine output format: explicit --output flag takes priority, then auto-detect via isatty. # When stdout is redirected (e.g. > file.txt or | jq), is_tty is False and we fall back to raw JSON. - show_plots = (is_tty or force_plots) and not config.json + show_plots = (output == "graph") if output else (is_tty and output != "json") - resolution_value = resolution if resolution else console.width - METRICS_WIDTH_PADDING + resolution_value = resolution if not show_plots else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( @@ -64,11 +64,11 @@ async def list_metrics( if not show_plots: json_bytes = openapi_dumps(metrics) - if config.json: - console.print_json(json_bytes.decode("utf-8")) - else: + if not is_tty: # stdout is redirected — print raw JSON so it pipes cleanly sys.stdout.write(json_bytes.decode("utf-8") + "\n") + else: + console.print_json(json_bytes.decode("utf-8")) return if not metrics: diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index 117d2074a..9695fc6df 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -111,13 +111,13 @@ [primary]tg ft list-metrics --logged-at-from 2024-01-01T00:00:00 --logged-at-to 2024-01-02T00:00:00[/primary] [dim]-[/dim] Retrieve a fixed number of data points as JSON: - [primary]tg ft list-metrics --resolution 50 --json[/primary] + [primary]tg ft list-metrics --resolution 50 --output json[/primary] -[dim]-[/dim] Save raw metrics to a file (plots are disabled automatically when redirecting): +[dim]-[/dim] Save raw metrics to a file (JSON is chosen automatically when redirecting): [primary]tg ft list-metrics > metrics.json[/primary] [dim]-[/dim] Save ASCII plots to a file: - [primary]tg ft list-metrics --force-plots > plots.txt[/primary] + [primary]tg ft list-metrics --output graph > plots.txt[/primary] """ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] From 8b95caa819417c3546b895250d96583023573451 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 17:20:40 +0200 Subject: [PATCH 13/16] return --json flag --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 2cdd0611d..2de3306a0 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -42,10 +42,16 @@ async def list_metrics( ) -> None: """Retrieve training metrics for a fine-tuning job.""" + if output != "json" and config.json: + raise ValueError( + f"--output {output!r} conflicts with --json. Either remove --json or set --output json." + ) + output_json = output == "json" or config.json + is_tty = sys.stdout.isatty() - # Determine output format: explicit --output flag takes priority, then auto-detect via isatty. + # Determine output format: explicit --output or --json flag takes priority, then auto-detect via isatty. # When stdout is redirected (e.g. > file.txt or | jq), is_tty is False and we fall back to raw JSON. - show_plots = (output == "graph") if output else (is_tty and output != "json") + show_plots = (output == "graph") if output else (is_tty and not output_json) resolution_value = resolution if not show_plots else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( From a418fa0d943d24641a3cc730a047833c12661144 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 17:26:46 +0200 Subject: [PATCH 14/16] simplify --- .../lib/cli/api/fine_tuning/list_metrics.py | 30 ++----------------- src/together/lib/cli/utils/_help_examples.py | 8 ++--- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 2de3306a0..9597d3d7d 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -32,28 +32,8 @@ async def list_metrics( help="Number of uniformly sampled training metric points to return. Does not limit the number of eval metric points." ), ] = None, - output: Annotated[ - Optional[Literal["json", "graph"]], - Parameter( - "--output", - help="Override the output format. 'json' prints raw JSON, 'graph' renders ASCII plots. By default the format is chosen automatically based on whether stdout is a terminal.", - ), - ] = None, ) -> None: """Retrieve training metrics for a fine-tuning job.""" - - if output != "json" and config.json: - raise ValueError( - f"--output {output!r} conflicts with --json. Either remove --json or set --output json." - ) - output_json = output == "json" or config.json - - is_tty = sys.stdout.isatty() - # Determine output format: explicit --output or --json flag takes priority, then auto-detect via isatty. - # When stdout is redirected (e.g. > file.txt or | jq), is_tty is False and we fall back to raw JSON. - show_plots = (output == "graph") if output else (is_tty and not output_json) - - resolution_value = resolution if not show_plots else console.width - METRICS_WIDTH_PADDING response = await show_loading_status( "Fetching metrics...", config.client.fine_tuning.list_metrics( @@ -62,19 +42,15 @@ async def list_metrics( global_step_to=global_step_to or omit, logged_at_from=logged_at_from or omit, logged_at_to=logged_at_to or omit, - resolution=resolution_value or omit, + resolution=resolution or omit, ), ) metrics = response.metrics or [] - if not show_plots: + if not config.json: json_bytes = openapi_dumps(metrics) - if not is_tty: - # stdout is redirected — print raw JSON so it pipes cleanly - sys.stdout.write(json_bytes.decode("utf-8") + "\n") - else: - console.print_json(json_bytes.decode("utf-8")) + console.print_json(json_bytes.decode("utf-8")) return if not metrics: diff --git a/src/together/lib/cli/utils/_help_examples.py b/src/together/lib/cli/utils/_help_examples.py index 9695fc6df..70285209e 100644 --- a/src/together/lib/cli/utils/_help_examples.py +++ b/src/together/lib/cli/utils/_help_examples.py @@ -111,13 +111,13 @@ [primary]tg ft list-metrics --logged-at-from 2024-01-01T00:00:00 --logged-at-to 2024-01-02T00:00:00[/primary] [dim]-[/dim] Retrieve a fixed number of data points as JSON: - [primary]tg ft list-metrics --resolution 50 --output json[/primary] + [primary]tg ft list-metrics --resolution 50 --json[/primary] -[dim]-[/dim] Save raw metrics to a file (JSON is chosen automatically when redirecting): - [primary]tg ft list-metrics > metrics.json[/primary] +[dim]-[/dim] Save raw metrics to a file: + [primary]tg ft list-metrics --json > metrics.json[/primary] [dim]-[/dim] Save ASCII plots to a file: - [primary]tg ft list-metrics --output graph > plots.txt[/primary] + [primary]tg ft list-metrics > plots.txt[/primary] """ FINE_TUNING_DOWNLOAD_HELP_EXAMPLES = """[dim]Examples:[/dim] From 6817537b978365af8d600d8a5eedaf4e41c34bb4 Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 17:44:08 +0200 Subject: [PATCH 15/16] fix --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index 9597d3d7d..f0fe5577f 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -48,7 +48,7 @@ async def list_metrics( metrics = response.metrics or [] - if not config.json: + if config.json: json_bytes = openapi_dumps(metrics) console.print_json(json_bytes.decode("utf-8")) return From d6e28c432c8ab081582a28ea02e2f9e8ffe1f4fd Mon Sep 17 00:00:00 2001 From: Artem Chumachenko Date: Wed, 20 May 2026 18:40:55 +0200 Subject: [PATCH 16/16] Apply suggestion from @blainekasten Co-authored-by: Blaine Kasten --- src/together/lib/cli/api/fine_tuning/list_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/together/lib/cli/api/fine_tuning/list_metrics.py b/src/together/lib/cli/api/fine_tuning/list_metrics.py index f0fe5577f..5e15a753b 100644 --- a/src/together/lib/cli/api/fine_tuning/list_metrics.py +++ b/src/together/lib/cli/api/fine_tuning/list_metrics.py @@ -53,7 +53,7 @@ async def list_metrics( console.print_json(json_bytes.decode("utf-8")) return - if not metrics: + if len(metrics) == 0: console.print(f"[muted]No metrics found for job {fine_tune_id}[/muted]") return