pivot: track row expansion by dimension values, not position

Wire the value-based expand keys through the pivot so expansion survives
adding a field, sorting, and data refreshes (#9781).

- PivotTable sets a hierarchical value-based getRowId (parent id + this
  row's value), with a reserved id for the grand-totals row.
- getValuesForExpandedKey / addExpandedDataToPivot resolve a key by
  matching each row's own value (stored under rowDimensions[0]) instead of
  positional indices, so the totals-row offset is gone.
- queryExpandedRowMeasureValues derives depth from the key's segment count.
- getFiltersForCell / getValuesForFlatTable / getRawRowValues and the
  show-more and ancestor-highlight paths use the value keys.
- Drop the defensive expanded={} resets on add/sort/columns/rows; stale
  keys are inert. Stop deserializing the legacy positional pivotExpanded
  proto field.
- Tests updated/added for value-based resolution.

Fixes #9781

Author: mahdi13

web-common/src/features/dashboards/pivot/PivotTable.svelte

@@ -18,6 +18,13 @@
     isShowMoreRow,
     splitPivotChips,
   } from "@rilldata/web-common/features/dashboards/pivot/pivot-utils";
+  import {
+    buildExpandKey,
+    childExpandKey,
+    encodeExpandKeyValue,
+    parentExpandKey,
+    PIVOT_TOTALS_ROW_ID,
+  } from "@rilldata/web-common/features/dashboards/pivot/pivot-expand-keys";
   import { copyToClipboard } from "@rilldata/web-common/lib/actions/copy-to-clipboard";
   import {
     createVirtualizer,
@@ -89,15 +96,32 @@
   export let clickSelection: PivotClickSelectionState | undefined = undefined;
 
   const options: Readable<TableOptions<PivotDataRow>> = derived(
-    [pivotDataStore, pivotState],
-    ([pivotData, state]) => {
+    [pivotDataStore, pivotState, config],
+    ([pivotData, state, cfg]) => {
       let tableData = [...pivotData.data];
       if (pivotData.totalsRowData) {
         tableData = [pivotData.totalsRowData, ...pivotData.data];
       }
+      const totalsRowData = pivotData.totalsRowData;
+      const rowDims = cfg.rowDimensionNames;
       return {
         data: tableData,
         columns: pivotData.columnDef,
+        // Value-based row ids: a row is keyed by the dimension values from the
+        // root to it, not by position, so expansion survives adding a field,
+        // sorting, and data refreshes. Kept hierarchical (parent id + this
+        // row's value) so depth and parent-address semantics still hold.
+        getRowId: (row, _index, parent) => {
+          if (totalsRowData && row === totalsRowData)
+            return PIVOT_TOTALS_ROW_ID;
+          if (cfg.isFlat) {
+            return buildExpandKey(rowDims.map((dim) => row[dim]));
+          }
+          const anchor = rowDims[0];
+          return parent
+            ? childExpandKey(parent.id, row[anchor])
+            : encodeExpandKeyValue(row[anchor]);
+        },
         state: {
           expanded: state.expanded,
           sorting: state.sorting,
@@ -237,7 +261,7 @@
     if (needsDomains) {
       for (const row of flatRows) {
         // Always skip the prepended grand-totals row.
-        if (hasTotalsRow && row.id === "0") continue;
+        if (hasTotalsRow && row.id === PIVOT_TOTALS_ROW_ID) continue;
         const target = row.subRows.length > 0 ? parentValues : leafValues;
         for (const cell of row.getAllCells()) {
           const meta = cell.column.columnDef.meta;
@@ -340,21 +364,20 @@
 
       if (!nextLimit) return;
 
-      // Check if this is the outermost dimension or a nested dimension
-      // Outermost dimension has rowId like "0", "1", etc. (no dots)
-      // Nested dimensions have rowId like "0.1", "0.1.2", etc.
-      const isOutermostDimension = !rowId.includes(".");
+      // The outermost "Show more" row sits at the top level (its parent key
+      // is the root ""); a nested one has a real parent node whose child
+      // limit we raise.
+      const parentKey = parentExpandKey(rowId);
 
-      if (isOutermostDimension) {
+      if (parentKey === "") {
         // Handle outermost dimension "Show more" click
         if (setPivotOutermostRowLimit) {
           setPivotOutermostRowLimit(nextLimit);
         }
       } else {
         // Handle nested dimension "Show more" click
-        const expandIndex = rowId.split(".").slice(0, -1).join(".");
-        if (expandIndex && setPivotRowLimitForExpanded) {
-          setPivotRowLimitForExpanded(expandIndex, nextLimit);
+        if (setPivotRowLimitForExpanded) {
+          setPivotRowLimitForExpanded(parentKey, nextLimit);
         }
       }
       return;

web-common/src/features/dashboards/pivot/pivot-expand-keys.ts

@@ -24,6 +24,11 @@ export const EXPAND_KEY_SEP = String.fromCharCode(0);
 // collide with a genuinely-absent deeper level. Contains no separator.
 const NULL_SENTINEL = "<null>";
 
+// Reserved id for the prepended grand-totals row. Begins with the separator,
+// which no encoded top-level value ever does, so it can't collide with a real
+// row id.
+export const PIVOT_TOTALS_ROW_ID = EXPAND_KEY_SEP + "totals";
+
 export function encodeExpandKeyValue(value: unknown): string {
   if (value === null || value === undefined) return NULL_SENTINEL;
   return String(value).replaceAll(EXPAND_KEY_SEP, "");

web-common/src/features/dashboards/pivot/pivot-expansion.spec.ts

@@ -1,13 +1,20 @@
 import { describe, expect, it } from "vitest";
 import { LOADING_CELL } from "@rilldata/web-common/features/dashboards/pivot/pivot-constants";
 import { createAndExpression } from "@rilldata/web-common/features/dashboards/stores/filter-utils";
-import { addExpandedDataToPivot } from "./pivot-expansion";
+import { buildExpandKey } from "./pivot-expand-keys";
+import {
+  addExpandedDataToPivot,
+  getValuesForExpandedKey,
+} from "./pivot-expansion";
 import { type PivotDataRow, type PivotDataStoreConfig } from "./types";
 
-function getConfig(showTotalsRow: boolean): PivotDataStoreConfig {
+function getConfig(
+  showTotalsRow: boolean,
+  rowDimensionNames = ["publisher", "campaign"],
+): PivotDataStoreConfig {
   return {
     measureNames: ["impressions"],
-    rowDimensionNames: ["publisher", "campaign"],
+    rowDimensionNames,
     colDimensionNames: [],
     allMeasures: [],
     allDimensions: [],
@@ -38,76 +45,129 @@ function getConfig(showTotalsRow: boolean): PivotDataStoreConfig {
   } as unknown as PivotDataStoreConfig;
 }
 
-describe("pivot expansion", () => {
-  it("adds expanded rows at the correct index when totals row is hidden", () => {
+describe("pivot expansion (value-based keys)", () => {
+  it("fills the node matched by dimension value, regardless of totals row", () => {
+    for (const showTotals of [false, true]) {
+      const tableData: PivotDataRow[] = [
+        { publisher: "A", subRows: [{ publisher: LOADING_CELL }] },
+        { publisher: "B", subRows: [{ publisher: LOADING_CELL }] },
+      ];
+
+      addExpandedDataToPivot(
+        getConfig(showTotals),
+        tableData,
+        ["publisher", "campaign"],
+        {},
+        [
+          {
+            isFetching: false,
+            expandIndex: buildExpandKey(["B"]),
+            rowDimensionValues: ["B"],
+            totals: [{ campaign: "campaign-1", impressions: 10 }],
+            data: [],
+          },
+        ],
+      );
+
+      // Row "A" is untouched; row "B" (matched by value, not position) is filled.
+      expect(tableData[0].subRows?.[0]?.publisher).toBe(LOADING_CELL);
+      expect(tableData[1].subRows?.[0]).toMatchObject({
+        publisher: "campaign-1",
+        campaign: "campaign-1",
+        impressions: 10,
+      });
+    }
+  });
+
+  it("resolves a nested value path to the correct deep node", () => {
     const tableData: PivotDataRow[] = [
       {
         publisher: "A",
-        subRows: [{ publisher: LOADING_CELL }],
-      },
-      {
-        publisher: "B",
-        subRows: [{ publisher: LOADING_CELL }],
+        subRows: [
+          { publisher: "camp1", subRows: [{ publisher: LOADING_CELL }] },
+        ],
       },
     ];
 
     addExpandedDataToPivot(
-      getConfig(false),
+      getConfig(true, ["publisher", "campaign", "adgroup"]),
       tableData,
-      ["publisher", "campaign"],
+      ["publisher", "campaign", "adgroup"],
       {},
       [
         {
           isFetching: false,
-          expandIndex: "1",
-          rowDimensionValues: ["B"],
-          totals: [{ campaign: "campaign-1", impressions: 10 }],
+          expandIndex: buildExpandKey(["A", "camp1"]),
+          rowDimensionValues: ["A", "camp1"],
+          totals: [{ adgroup: "ad-1", impressions: 5 }],
           data: [],
         },
       ],
     );
 
-    expect(tableData[0].subRows?.[0]?.publisher).toBe(LOADING_CELL);
-    expect(tableData[1].subRows?.[0]).toMatchObject({
-      publisher: "campaign-1",
-      campaign: "campaign-1",
-      impressions: 10,
+    expect(tableData[0].subRows?.[0]?.subRows?.[0]).toMatchObject({
+      publisher: "ad-1",
+      adgroup: "ad-1",
+      impressions: 5,
     });
   });
 
-  it("keeps the totals row offset when totals row is visible", () => {
+  it("does nothing when the value path matches no row", () => {
     const tableData: PivotDataRow[] = [
-      {
-        publisher: "A",
-        subRows: [{ publisher: LOADING_CELL }],
-      },
-      {
-        publisher: "B",
-        subRows: [{ publisher: LOADING_CELL }],
-      },
+      { publisher: "A", subRows: [{ publisher: LOADING_CELL }] },
     ];
-
     addExpandedDataToPivot(
-      getConfig(true),
+      getConfig(false),
       tableData,
       ["publisher", "campaign"],
       {},
       [
         {
           isFetching: false,
-          expandIndex: "2",
-          rowDimensionValues: ["B"],
-          totals: [{ campaign: "campaign-1", impressions: 10 }],
+          expandIndex: buildExpandKey(["does-not-exist"]),
+          rowDimensionValues: ["does-not-exist"],
+          totals: [{ campaign: "c", impressions: 1 }],
           data: [],
         },
       ],
     );
-
     expect(tableData[0].subRows?.[0]?.publisher).toBe(LOADING_CELL);
-    expect(tableData[1].subRows?.[0]).toMatchObject({
-      publisher: "campaign-1",
-      campaign: "campaign-1",
-      impressions: 10,
-    });
+  });
+});
+
+describe("getValuesForExpandedKey", () => {
+  const tableData: PivotDataRow[] = [
+    {
+      publisher: "A",
+      subRows: [{ publisher: "camp1" }, { publisher: "camp2" }],
+    },
+    { publisher: "B", subRows: [{ publisher: "camp3" }] },
+  ];
+
+  it("returns the actual values along the matched path", () => {
+    expect(
+      getValuesForExpandedKey(
+        tableData,
+        ["publisher", "campaign"],
+        buildExpandKey(["B"]),
+      ),
+    ).toEqual(["B"]);
+    expect(
+      getValuesForExpandedKey(
+        tableData,
+        ["publisher", "campaign"],
+        buildExpandKey(["A", "camp2"]),
+      ),
+    ).toEqual(["A", "camp2"]);
+  });
+
+  it("stops at the deepest resolvable segment", () => {
+    expect(
+      getValuesForExpandedKey(
+        tableData,
+        ["publisher", "campaign"],
+        buildExpandKey(["A", "missing"]),
+      ),
+    ).toEqual(["A"]);
   });
 });