Change profile.shared.stackTable.prefix to prefixOffset (delta from current index)

docs-developer/CHANGELOG-formats.md

@@ -6,6 +6,27 @@ Note that this is not an exhaustive list. Processed profile format upgraders can
 
 ## Processed profile format
 
+### Version 67
+
+The `prefix` column of `profile.shared.stackTable` was replaced with a
+`prefixOffset` column. For each stack `i`, `prefixOffset[i] === 0` means
+`i` is a root, and `prefixOffset[i] === k` (with `k > 0`) means `i`'s parent
+is at index `i - k`. All values are non-negative because the stack table is
+stored in topological order. In-memory the column is an `Int32Array`. The
+motivation is to shrink the encoded prefix values for large profiles, so that
+they compress better in the on-wire JSON.
+
+### Version 66
+
+The `prefix` column of `profile.shared.stackTable` now uses `-1` instead of `null`
+to indicate "this stack node is a root". In-memory it is now an `Int32Array`;
+on the JSON wire, the column is just an array of numbers (where `-1` marks a
+root).
+
+### Version 65
+
+The stack table's `frame` column (stored at `profile.shared.stackTable.frame`) can now optionally be stored as an `Int32Array`, for profiles loaded from [JsonSlabs](https://github.com/mstange/json-slabs/) files (.jslb, .jslb.gz). Regular JS / JSON arrays are still accepted.
+
 ### Version 64
 
 A new `SourceLocationTable` has been added to `profile.shared.sourceLocationTable`. It holds the original (pre-compilation) source positions produced by source map symbolication, paired with the generated `line`/`column` already on `FrameTable`.

src/app-logic/constants.ts

@@ -12,7 +12,7 @@ export const GECKO_PROFILE_VERSION = 34;
 // The current version of the "processed" profile format.
 // Please don't forget to update the processed profile format changelog in
 // `docs-developer/CHANGELOG-formats.md`.
-export const PROCESSED_PROFILE_VERSION = 64;
+export const PROCESSED_PROFILE_VERSION = 67;
 
 // The following are the margin sizes for the left and right of the timeline. Independent
 // components need to share these values.

src/app-logic/url-handling.ts

@@ -1649,19 +1649,20 @@ function getStackIndexFromVersion3JSCallNodePath(
   oldCallNodePath: CallNodePath
 ): IndexIntoStackTable | null {
   const { stackTable, funcTable, frameTable } = shared;
-  const stackIndexDepth: Map<IndexIntoStackTable | null, number> = new Map();
-  stackIndexDepth.set(null, -1);
+  const stackIndexDepth: Map<IndexIntoStackTable, number> = new Map();
+  // Map key -1 represents the virtual root above all stack indexes.
+  stackIndexDepth.set(-1, -1);
 
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
-    const prefix = stackTable.prefix[stackIndex];
+    const offset = stackTable.prefixOffset[stackIndex];
+    const prefix = offset === 0 ? -1 : stackIndex - offset;
     const frameIndex = stackTable.frame[stackIndex];
     const funcIndex = frameTable.func[frameIndex];
     const isJS = funcTable.isJS[funcIndex];
     // We know that at this point stack table is sorted and the following
     // condition holds:
-    // assert(prefixStack === null || prefixStack < stackIndex);
-    const doesPrefixMatchCallNodePath =
-      prefix === null || stackIndexDepth.has(prefix);
+    // assert(prefixStack === -1 || prefixStack < stackIndex);
+    const doesPrefixMatchCallNodePath = stackIndexDepth.has(prefix);
 
     if (!doesPrefixMatchCallNodePath) {
       continue;
@@ -1696,14 +1697,15 @@ function getVersion4JSCallNodePathFromStackIndex(
 ): CallNodePath {
   const { funcTable, stackTable, frameTable } = shared;
   const callNodePath = [];
-  let nextStackIndex: IndexIntoStackTable | null = stackIndex;
-  while (nextStackIndex !== null) {
+  let nextStackIndex: IndexIntoStackTable = stackIndex;
+  while (nextStackIndex !== -1) {
     const frameIndex: IndexIntoFrameTable = stackTable.frame[nextStackIndex];
     const funcIndex = frameTable.func[frameIndex];
     if (funcTable.isJS[funcIndex] || funcTable.relevantForJS[funcIndex]) {
       callNodePath.unshift(funcIndex);
     }
-    nextStackIndex = stackTable.prefix[nextStackIndex];
+    const offset = stackTable.prefixOffset[nextStackIndex];
+    nextStackIndex = offset === 0 ? -1 : nextStackIndex - offset;
   }
   return callNodePath;
 }

src/profile-logic/address-timings.ts

@@ -127,7 +127,7 @@ export function getStackAddressInfo(
   for (let stackIndex = 0; stackIndex < stackTable.length; stackIndex++) {
     const prefixStack = stackTable.prefix[stackIndex];
     const prefixAddressSet: IndexIntoAddressSetTable | -1 =
-      prefixStack !== null ? stackIndexToAddressSetIndex[prefixStack] : -1;
+      prefixStack !== -1 ? stackIndexToAddressSetIndex[prefixStack] : -1;
 
     const frame = stackTable.frame[stackIndex];
     const nativeSymbolOfThisStack = frameTable.nativeSymbol[frame];

src/profile-logic/data-structures.ts

@@ -27,6 +27,8 @@ import type {
   CallNodeTable,
   SourceTable,
   SourceLocationTable,
+  IndexIntoFrameTable,
+  IndexIntoStackTable,
 } from 'firefox-profiler/types';
 
 /**
@@ -47,7 +49,13 @@ export function getEmptySamplesTable(): RawSamplesTable {
   };
 }
 
-export function getEmptyRawStackTable(): RawStackTable {
+export type RawStackTableBuilder = {
+  frame: IndexIntoFrameTable[];
+  prefix: Array<IndexIntoStackTable | null>;
+  length: number;
+};
+
+export function getRawStackTableBuilder(): RawStackTableBuilder {
   return {
     // Important!
     // If modifying this structure, please update all callers of this function to ensure
@@ -59,6 +67,37 @@ export function getEmptyRawStackTable(): RawStackTable {
   };
 }
 
+export function getRawStackTableBuilderWithExistingContents(
+  existing: RawStackTable
+): RawStackTableBuilder {
+  const prefix = new Array<IndexIntoStackTable | null>(existing.length);
+  for (let i = 0; i < existing.length; i++) {
+    const offset = existing.prefixOffset[i];
+    prefix[i] = offset === 0 ? null : i - offset;
+  }
+  return {
+    frame: [...existing.frame],
+    prefix,
+    length: existing.length,
+  };
+}
+
+export function finishRawStackTableBuilder(
+  builder: RawStackTableBuilder
+): RawStackTable {
+  const { frame, prefix, length } = builder;
+  const prefixOffset = new Int32Array(length);
+  for (let i = 0; i < length; i++) {
+    const p = prefix[i];
+    prefixOffset[i] = p === null ? 0 : i - p;
+  }
+  return {
+    frame: new Int32Array(frame),
+    prefixOffset,
+    length,
+  };
+}
+
 /**
  * Returns an empty samples table with eventDelay field instead of responsiveness.
  * eventDelay is a new field and it replaced responsiveness. We should still
@@ -393,7 +432,7 @@ export function getEmptyThread(overrides?: Partial<RawThread>): RawThread {
 
 export function getEmptySharedData(): RawProfileSharedData {
   return {
-    stackTable: getEmptyRawStackTable(),
+    stackTable: finishRawStackTableBuilder(getRawStackTableBuilder()),
     frameTable: getEmptyFrameTable(),
     funcTable: getEmptyFuncTable(),
     resourceTable: getEmptyResourceTable(),

src/profile-logic/global-data-collector.ts

@@ -4,13 +4,14 @@
 
 import { StringTable } from '../utils/string-table';
 import {
+  finishRawStackTableBuilder,
   getEmptyFrameTable,
   getEmptyFuncTable,
   getEmptyNativeSymbolTable,
-  getEmptyRawStackTable,
   getEmptyResourceTable,
   getEmptySourceTable,
   getEmptySourceLocationTable,
+  getRawStackTableBuilder,
 } from './data-structures';
 
 import type {
@@ -22,7 +23,6 @@ import type {
   RawProfileSharedData,
   SourceTable,
   FrameTable,
-  RawStackTable,
   FuncTable,
   ResourceTable,
   NativeSymbolTable,
@@ -34,6 +34,7 @@ import type {
   Bytes,
 } from 'firefox-profiler/types';
 import { ResourceType } from 'firefox-profiler/types';
+import type { RawStackTableBuilder } from './data-structures';
 
 /**
  * GlobalDataCollector collects data which is global in the processed profile
@@ -50,7 +51,7 @@ export class GlobalDataCollector {
   _stringTable: StringTable = StringTable.withBackingArray(this._stringArray);
   _sources: SourceTable = getEmptySourceTable();
   _frameTable: FrameTable = getEmptyFrameTable();
-  _stackTable: RawStackTable = getEmptyRawStackTable();
+  _stackTableBuilder: RawStackTableBuilder = getRawStackTableBuilder();
   _funcTable: FuncTable = getEmptyFuncTable();
   _resourceTable: ResourceTable = getEmptyResourceTable();
   _nativeSymbols: NativeSymbolTable = getEmptyNativeSymbolTable();
@@ -302,15 +303,15 @@ export class GlobalDataCollector {
     return this._frameTable;
   }
 
-  getStackTable(): RawStackTable {
-    return this._stackTable;
+  getStackTableBuilder(): RawStackTableBuilder {
+    return this._stackTableBuilder;
   }
 
   // Package up all de-duplicated global tables so that they can be embedded in
   // the profile.
   finish(): { libs: Lib[]; shared: RawProfileSharedData } {
     const shared: RawProfileSharedData = {
-      stackTable: this._stackTable,
+      stackTable: finishRawStackTableBuilder(this._stackTableBuilder),
       frameTable: this._frameTable,
       funcTable: this._funcTable,
       resourceTable: this._resourceTable,

src/profile-logic/import/chrome.ts

@@ -5,12 +5,15 @@
 import type {
   Profile,
   RawThread,
-  RawStackTable,
   IndexIntoStackTable,
   MixedObject,
 } from 'firefox-profiler/types';
 
-import { getEmptyProfile, getEmptyThread } from '../data-structures';
+import {
+  getEmptyProfile,
+  getEmptyThread,
+  type RawStackTableBuilder,
+} from '../data-structures';
 import type { StringTable } from '../../utils/string-table';
 import { ensureExists, coerce } from '../../utils/types';
 import {
@@ -501,7 +504,7 @@ async function processTracingEvents(
   const stringTable = globalDataCollector.getStringTable();
 
   const frameTable = globalDataCollector.getFrameTable();
-  const stackTable = globalDataCollector.getStackTable();
+  const stackTable = globalDataCollector.getStackTableBuilder();
 
   let profileEvents: (ProfileEvent | CpuProfileEvent)[] = (eventsByName.get(
     'Profile'
@@ -868,7 +871,7 @@ function getImageSize(
  * For sanity, check that stacks are ordered where the prefix stack
  * always preceeds the current stack index in the StackTable.
  */
-function assertStackOrdering(stackTable: RawStackTable) {
+function assertStackOrdering(stackTable: RawStackTableBuilder) {
   const visitedStacks = new Set<number | null>([null]);
   for (let i = 0; i < stackTable.length; i++) {
     if (!visitedStacks.has(stackTable.prefix[i])) {

src/profile-logic/import/dhat.ts

@@ -183,7 +183,7 @@ export function attemptToConvertDhat(json: unknown): Profile | null {
   profile.meta.product = dhat.cmd + ' (dhat)';
   profile.meta.importedFrom = `dhat`;
   const globalDataCollector = new GlobalDataCollector();
-  const stackTable = globalDataCollector.getStackTable();
+  const stackTable = globalDataCollector.getStackTableBuilder();
   const frameTable = globalDataCollector.getFrameTable();
   const stringTable = globalDataCollector.getStringTable();
 

src/profile-logic/import/flame-graph.ts

@@ -60,7 +60,7 @@ export function convertFlameGraphProfile(profileText: string): Profile {
   });
 
   const frameTable = globalDataCollector.getFrameTable();
-  const stackTable = globalDataCollector.getStackTable();
+  const stackTable = globalDataCollector.getStackTableBuilder();
   const { samples } = thread;
 
   // Maps to deduplicate stacks, frames, and functions.

src/profile-logic/import/simpleperf.ts

@@ -24,7 +24,9 @@ import {
   getEmptyFuncTable,
   getEmptyResourceTable,
   getEmptyFrameTable,
-  getEmptyRawStackTable,
+  getRawStackTableBuilder,
+  finishRawStackTableBuilder,
+  type RawStackTableBuilder,
   getEmptySamplesTable,
   getEmptyRawMarkerTable,
   getEmptyNativeSymbolTable,
@@ -189,15 +191,15 @@ class FirefoxFrameTable {
 class FirefoxSampleTable {
   strings: StringTable;
 
-  stackTable: RawStackTable = getEmptyRawStackTable();
+  stackTable: RawStackTableBuilder = getRawStackTableBuilder();
   stackMap: Map<string, IndexIntoStackTable> = new Map();
 
   constructor(strings: StringTable) {
     this.strings = strings;
   }
 
   toJson(): RawStackTable {
-    return this.stackTable;
+    return finishRawStackTableBuilder(this.stackTable);
   }
 
   findOrAddStack(

src/profile-logic/insert-stack-labels.ts

@@ -4,7 +4,6 @@
 
 import type {
   IndexIntoFrameTable,
-  IndexIntoStackTable,
   RawStackTable,
   IndexIntoFuncTable,
   Profile,
@@ -197,10 +196,10 @@ export function insertStackLabels(
   );
   let stacksToInsertCount = 0;
   for (let stackIndex = 0; stackIndex < oldStackTable.length; stackIndex++) {
-    const parentStackIndex = oldStackTable.prefix[stackIndex];
+    const prefixOffset = oldStackTable.prefixOffset[stackIndex];
     const inheritedLabelFrameIndex =
-      parentStackIndex !== null
-        ? inheritedLabelFrameIndexAtStack[parentStackIndex]
+      prefixOffset !== 0
+        ? inheritedLabelFrameIndexAtStack[stackIndex - prefixOffset]
         : null;
     const frameIndex = oldStackTable.frame[stackIndex];
     const funcIndex = oldFrameTable.func[frameIndex];
@@ -218,7 +217,7 @@ export function insertStackLabels(
     ) {
       labelFrameIndexToInsertAtStack[stackIndex] = null;
       inheritedLabelFrameIndexAtStack[stackIndex] = null;
-    } else if (parentStackIndex === null) {
+    } else if (prefixOffset === 0) {
       labelFrameIndexToInsertAtStack[stackIndex] = rootLabelFrameIndex;
       inheritedLabelFrameIndexAtStack[stackIndex] = rootLabelFrameIndex;
       stacksToInsertCount++;
@@ -230,7 +229,7 @@ export function insertStackLabels(
 
   // Now compute the new stack table.
   const newStackCount = oldStackTable.length + stacksToInsertCount;
-  const newPrefixCol = new Array<IndexIntoStackTable | null>(newStackCount);
+  const newPrefixOffsetCol = new Int32Array(newStackCount);
   const newFrameCol = new Array<IndexIntoFrameTable>(newStackCount);
   const oldStackToNewStackPlusOne = new Int32Array(oldStackTable.length);
   let nextNewStackIndex = 0;
@@ -241,18 +240,22 @@ export function insertStackLabels(
   ) {
     const labelFrameIndexToInsert =
       labelFrameIndexToInsertAtStack[oldStackIndex];
-    const oldPrefix = oldStackTable.prefix[oldStackIndex];
+    const oldPrefixOffset = oldStackTable.prefixOffset[oldStackIndex];
+    const oldPrefix =
+      oldPrefixOffset !== 0 ? oldStackIndex - oldPrefixOffset : -1;
     let newPrefix =
-      oldPrefix !== null ? oldStackToNewStackPlusOne[oldPrefix] - 1 : null;
+      oldPrefix !== -1 ? oldStackToNewStackPlusOne[oldPrefix] - 1 : -1;
     const frameIndex = oldStackTable.frame[oldStackIndex];
     if (labelFrameIndexToInsert !== null) {
       const insertedStackIndex = nextNewStackIndex++;
-      newPrefixCol[insertedStackIndex] = newPrefix;
+      newPrefixOffsetCol[insertedStackIndex] =
+        newPrefix === -1 ? 0 : insertedStackIndex - newPrefix;
       newFrameCol[insertedStackIndex] = labelFrameIndexToInsert;
       newPrefix = insertedStackIndex;
     }
     const newStackIndex = nextNewStackIndex++;
-    newPrefixCol[newStackIndex] = newPrefix;
+    newPrefixOffsetCol[newStackIndex] =
+      newPrefix === -1 ? 0 : newStackIndex - newPrefix;
     newFrameCol[newStackIndex] = frameIndex;
     oldStackToNewStackPlusOne[oldStackIndex] = newStackIndex + 1;
   }
@@ -266,7 +269,7 @@ export function insertStackLabels(
   }
 
   const stackTable: RawStackTable = {
-    prefix: newPrefixCol,
+    prefixOffset: newPrefixOffsetCol,
     frame: newFrameCol,
     length: newStackCount,
   };