qid as number, number and cleanup fromArrow

Author: bmschmidt

dev/FourClasses.svelte

@@ -8,8 +8,8 @@
   import SelectPoints from './svelte/SelectPoints.svelte';
   const startSize = 2;
   const prefs = {
-    source_url: '/newtiles',
-    max_points: 10,
+    source_url: '/tiles',
+    max_points: 10000,
     alpha: 35, // Target saturation for the full page.
     zoom_balance: 0.22, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
     point_size: startSize, // Default point size before application of size scaling

dev/svelte/SelectPoints.svelte

@@ -8,8 +8,8 @@
     const selection = await scatterplot.deeptable.select_data({
       name: Math.random().toFixed(8),
       tileFunction: async (tile) => {
-        const b = new Bitmask(tile.manifest.nPoints);
-        for (let i = 0; i < tile.manifest.nPoints; i++) {
+        const b = new Bitmask(tile.metadata.nPoints);
+        for (let i = 0; i < tile.metadata.nPoints; i++) {
           if (Math.random() < 0.001) {
             b.set(i);
           }

src/Deeptable.ts

@@ -37,7 +37,7 @@ import type {
 import { DataSelection } from './selection';
 import { Some, TupleMap } from './utilityFunctions';
 import { getNestedVector } from './regl_rendering';
-import { tileKey_to_tix } from './tixrixqid';
+import { Qid, tileKey_to_tix } from './tixrixqid';
 
 type TransformationStatus = 'queued' | 'in progress' | 'complete' | 'failed';
 
@@ -962,6 +962,25 @@ export class Deeptable {
       }
     };
   }
+
+  getQids(qids: Qid[]): StructRowProxy[] {
+    // For a set of tile row tuples, returns underlying
+    // structRowProxies. Note that this makes no concrete guarantee about
+    // **which** fields will be loaded in the underlying arrow arrays.
+    const matches : StructRowProxy[] = []
+    for (const [tix, rix] of qids) {
+      if (this.flatTree[tix] === undefined) {
+        // If we can't immediately find the tile, check if it's hiding
+        const tile = this.map(t => t).find(t => t.tix === tix)
+        this.flatTree[tix] = tile || null;        
+      }
+      if (this.flatTree[tix] === null) {
+        throw new Error("Missing tile index " + tix)
+      }
+      matches.push(this.flatTree[tix].record_batch.get(rix))
+    }
+    return matches
+  }
 }
 
 function area(rect: Rectangle) {

src/interaction.ts

@@ -15,6 +15,7 @@ import type { Deeptable } from './Deeptable';
 import type * as DS from './types';
 import type { Scatterplot } from './scatterplot';
 import { PositionalAesthetic } from './aesthetics/ScaledAesthetic';
+import { Qid } from './tixrixqid';
 type Annotation = {
   x: number;
   y: number;
@@ -181,10 +182,12 @@ export class Zoom {
     this.zoomer = zoomer;
   }
 
-  set_highlit_points(data: StructRowProxy[]) {
+  set_highlit_points(dd: Qid[]) {
     const { x_, y_ } = this.scales();
     const xdim = this.scatterplot.dim('x') as PositionalAesthetic;
     const ydim = this.scatterplot.dim('y') as PositionalAesthetic;
+    
+    const data = this.scatterplot.deeptable.getQids(dd)
     this.scatterplot.highlit_point_change(data, this.scatterplot);
 
     const annotations: Annotation[] = data.map((d) => {
@@ -226,10 +229,6 @@ export class Zoom {
       });
   }
 
-  set_highlit_point(point: StructRowProxy) {
-    this.set_highlit_points([point]);
-  }
-
   add_mouseover() {
     let last_fired = 0;
     const renderer: ReglRenderer = this.renderers.get('regl') as ReglRenderer;

src/regl_rendering.ts

@@ -33,7 +33,6 @@ import {
   Data,
   Dictionary,
   Struct,
-  StructRowProxy,
   Type,
   Utf8,
   Vector,
@@ -43,6 +42,7 @@ import { StatefulAesthetic } from './aesthetics/StatefulAesthetic';
 import { Filter, Foreground } from './aesthetics/BooleanAesthetic';
 import { ZoomTransform } from 'd3-zoom';
 import { Some, TupleMap, TupleSet } from './utilityFunctions';
+import { Qid } from './tixrixqid';
 // eslint-disable-next-line import/prefer-default-export
 export class ReglRenderer extends Renderer {
   public regl: Regl;
@@ -546,7 +546,7 @@ export class ReglRenderer extends Renderer {
     return v;
   }
 
-  color_pick(x: number, y: number): null | StructRowProxy {
+  color_pick(x: number, y: number): null | Qid {
     if (y === 0) {
       // Not sure why, but this makes things complainy.
       // console.warn('that thing again.');
@@ -561,15 +561,7 @@ export class ReglRenderer extends Renderer {
     if (row_number === -1) {
       return null;
     }
-    for (const tile of this.visible_tiles()) {
-      if (tile.tix === tile_number) {
-        return tile.record_batch.get(row_number);
-      }
-    }
-    return null;
-    //    const p = this.tileSet.findPoint(point_as_int);
-    //    if (p.length === 0) { return; }
-    //    return p[0];
+    return [tile_number, row_number]
   }
 
   color_pick_single(

src/tile.ts

@@ -78,7 +78,7 @@ export class Tile {
    * @param key Either the string identifier of the tile,
    * OR a `TileManifest` object including an identifier.   *
    * @param parent The parent tile -- used to navigate through the tree.
-   * @param deeptable The full atlas deeptable of which this tile is a part.
+   * @param deeptable The full deepscatter deeptable of which this tile is a part.
    */
   constructor(key: string, parent: Tile | null, deeptable: Deeptable) {
     // If it's just initiated with a key, build that into a minimal manifest.
@@ -295,16 +295,16 @@ export class Tile {
   }
 
   set metadata(
-    manifest: TileMetadata | (TileMetadata & { extent: Rectangle }),
+    metadata: TileMetadata | (TileMetadata & { extent: Rectangle }),
   ) {
     // Setting the manifest is the thing that spawns children.
-    this.highest_known_ix = manifest.max_ix;
+    this.highest_known_ix = metadata.max_ix;
     this._metadata = {
-      ...manifest,
+      ...metadata,
       extent:
-        typeof manifest.extent === 'string'
-          ? (JSON.parse(manifest.extent) as Rectangle)
-          : manifest.extent,
+        typeof metadata.extent === 'string'
+          ? (JSON.parse(metadata.extent) as Rectangle)
+          : metadata.extent,
     };
   }
 

src/tixrixqid.ts

@@ -8,7 +8,7 @@ import type {
 } from 'apache-arrow';
 
 import type { Tile } from './deepscatter';
-import { Bitmask, DataSelection, Deeptable } from './deepscatter';
+import { DataSelection, Deeptable } from './deepscatter';
 
 // The type below indicates that a Qid is not valid if
 // there are zero rows selected in the tile.

src/types.ts

@@ -11,6 +11,7 @@ import type {
   Timestamp,
   Utf8,
   Vector,
+  TypeMap,
 } from 'apache-arrow';
 import type { Renderer } from './rendering';
 import type { Deeptable } from './Deeptable';
@@ -45,6 +46,9 @@ export type PointFunction<T = number> = (p: StructRowProxy) => T;
  * A proxy class that wraps around tile get calls. Used to avoid
  * putting Nomic login logic in deepscatter while fetching
  * tiles with authentication.
+ * 
+ * An API call type must return Uint8Arrays that represent arrow 
+ * tables of the type produced by quadfeather.
  *
  */
 export interface TileProxy {
@@ -101,7 +105,7 @@ export type DeeptableCreateParams = {
 
   // A manifest listing all the tiles in the deeptable, of the type created by
   // quadfeather v2.0.0 or greater.
-  tileManifest?: Table;
+  tileManifest?: Table<{key: Utf8}>;
 
   // A URL for an arrow file manifest. The schema for this manifest
   // is not yet publically documented: I hope to bundle it into the

src/wrap_arrow.ts

@@ -8,12 +8,15 @@ import {
   vectorFromArray,
   Vector,
   Float,
+  tableFromArrays,
+  Utf8,
 } from 'apache-arrow';
 import { Deeptable } from './Deeptable';
 import { add_or_delete_column } from './Deeptable';
 import type * as DS from './types';
-import { extent } from 'd3-array';
+import { extent, extent, range } from 'd3-array';
 import { Rectangle } from './tile';
+import { tixToZxy } from './tixrixqid';
 
 /**
  * This function is used to wrap an arrow table into a
@@ -31,22 +34,36 @@ export function wrapArrowTable(
 ): Deeptable {
   let tb = tableFromIPC(tbArray);
   let batches = tb.batches;
+  const minIx = []
+  const maxIx = []
+  // Extents of each tile, as JSON.
+  const extents : string[] = []
   if (tb.getChild('ix') === null) {
     let rowNum = 0;
     batches = batches.map((batch) => {
       if (batch.numRows > 2 ** 16) {
         throw new Error(
-          'Arrow record batches temporarily limited to 2^16 rows.',
+          'Arrow record batches limited to 2^16 rows.',
         );
       }
       const array = new Int32Array(batch.numRows);
       for (let i = 0; i < batch.numRows; i++) {
         array[i] = rowNum++;
       }
-      return add_or_delete_column(batch, 'ix', vectorFromArray(array));
+      return add_or_delete_column(batch, 'ix', vectorFromArray(array));      
     });
     tb = new Table(batches);
   }
+  for (const batch of batches) {
+    minIx.push(batch.get(0)['ix'])
+    maxIx.push(batch.get(batch.numRows - 1)['ix'])
+    extents.push(
+      JSON.stringify({
+        x: extent(batch.getChild('x')),
+        y: extent(batch.getChild('y'))
+      })
+    )
+  }
 
   const proxy = new ArrowProxy(batches);
 
@@ -65,12 +82,25 @@ export function wrapArrowTable(
     y: extent([...(y as Iterable<number>)]),
   } as Rectangle;
 
+
+  const tileManifest = tableFromArrays({
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    key: vectorFromArray(range(batches.length).map(t => tixToZxy(t).join('/')), new Utf8()),
+    min_ix: minIx,
+    max_ix: maxIx,
+    nPoints: batches.map(d => d.numRows),
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    extent: vectorFromArray(extents, new Utf8())
+  })
+
   return new Deeptable({
     baseUrl: `feather://table`,
     plot,
     tileProxy: proxy,
     tileStructure: 'other',
     extent: dataExtent,
+    // @ts-expect-error missing types for tableFromArrays in arrow js
+    tileManifest
   });
 }
 

tests/dataset.spec.js

@@ -28,7 +28,7 @@ test('Columns can be deleted and replaced', async () => {
 
   dataset.transformations['integers'] = async function (tile) {
     await tile.populateManifest();
-    return new Float32Array(tile.manifest.nPoints);
+    return new Float32Array(tile.metadata.nPoints);
   };
 
   dataset.deleteColumn('integers');