From 6647017dcaba0c7d1207b77733098c9e7e9a7ac2 Mon Sep 17 00:00:00 2001 From: Ib Green Date: Tue, 2 May 2023 09:46:19 -0400 Subject: [PATCH] feat: Typed loaders (#2448) --- modules/arrow/src/arrow-loader.ts | 16 +++++------ modules/arrow/src/arrow-writer.ts | 19 ++++++------- modules/arrow/src/lib/encode-arrow.ts | 2 +- modules/bson/src/bson-loader.ts | 2 +- modules/bson/src/bson-writer.ts | 12 +++++---- modules/gltf/src/glb-loader.ts | 2 +- modules/gltf/src/glb-writer.ts | 12 ++++++--- modules/gltf/src/lib/encoders/encode-glb.ts | 9 ++++++- modules/gltf/src/lib/encoders/encode-gltf.ts | 27 ++++++++++++------- modules/gltf/src/lib/types/glb-types.ts | 2 +- .../lib/arrow/convert-schema-from-parquet.ts | 2 +- modules/ply/src/index.ts | 2 -- .../lib/table/simple-table/table-accessors.ts | 15 +++++++---- 13 files changed, 75 insertions(+), 47 deletions(-) diff --git a/modules/arrow/src/arrow-loader.ts b/modules/arrow/src/arrow-loader.ts index 2283ed28b9..fd75ef35fc 100644 --- a/modules/arrow/src/arrow-loader.ts +++ b/modules/arrow/src/arrow-loader.ts @@ -1,4 +1,6 @@ +// loaders.gl, MIT license import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils'; +import type {ArrowTable} from '@loaders.gl/schema'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. @@ -10,14 +12,8 @@ export type ArrowLoaderOptions = LoaderOptions & { }; }; -const DEFAULT_ARROW_LOADER_OPTIONS = { - arrow: { - shape: 'columnar-table' - } -}; - /** ArrowJS table loader */ -export const ArrowLoader = { +export const ArrowLoader: Loader = { name: 'Apache Arrow', id: 'arrow', module: 'arrow', @@ -32,7 +28,11 @@ export const ArrowLoader = { ], binary: true, tests: ['ARROW'], - options: DEFAULT_ARROW_LOADER_OPTIONS + options: { + arrow: { + shape: 'columnar-table' + } + } }; export const _typecheckArrowLoader: Loader = ArrowLoader; diff --git a/modules/arrow/src/arrow-writer.ts b/modules/arrow/src/arrow-writer.ts index 0f8bfe784a..7ffa8a65e8 100644 --- a/modules/arrow/src/arrow-writer.ts +++ b/modules/arrow/src/arrow-writer.ts @@ -1,15 +1,18 @@ -// import type {Writer} from '@loaders.gl/loader-utils'; -import type {WriterOptions} from '@loaders.gl/loader-utils'; +// import type {} from '@loaders.gl/loader-utils'; +import type {Writer, WriterOptions} from '@loaders.gl/loader-utils'; +import {ColumnarTable} from './lib/encode-arrow'; import {encodeArrowSync} from './lib/encode-arrow'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; -type ArrowWriterOptions = WriterOptions; +type ArrowWriterOptions = WriterOptions & { + arrow?: {}; +}; /** Apache Arrow writer */ -export const ArrowWriter = { +export const ArrowWriter: Writer = { name: 'Apache Arrow', id: 'arrow', module: 'arrow', @@ -20,11 +23,9 @@ export const ArrowWriter = { 'application/vnd.apache.arrow.stream', 'application/octet-stream' ], - encodeSync, + encodeSync(data, options?) { + return encodeArrowSync(data); + }, binary: true, options: {} }; - -function encodeSync(data, options?: ArrowWriterOptions) { - return encodeArrowSync(data); -} diff --git a/modules/arrow/src/lib/encode-arrow.ts b/modules/arrow/src/lib/encode-arrow.ts index 04210d4746..36c6c306a1 100644 --- a/modules/arrow/src/lib/encode-arrow.ts +++ b/modules/arrow/src/lib/encode-arrow.ts @@ -1,7 +1,7 @@ import {Table, Vector, tableToIPC, vectorFromArray} from 'apache-arrow'; import {AnyArrayType, VECTOR_TYPES} from '../types'; -type ColumnarTable = { +export type ColumnarTable = { name: string; array: AnyArrayType; type: number; diff --git a/modules/bson/src/bson-loader.ts b/modules/bson/src/bson-loader.ts index ceb237cb32..6e0494bdc9 100644 --- a/modules/bson/src/bson-loader.ts +++ b/modules/bson/src/bson-loader.ts @@ -19,7 +19,7 @@ const DEFAULT_BSON_LOADER_OPTIONS = { bson: {} }; -export const BSONLoader: LoaderWithParser = { +export const BSONLoader: LoaderWithParser, never, BSONLoaderOptions> = { name: 'BSON', id: 'bson', module: 'bson', diff --git a/modules/bson/src/bson-writer.ts b/modules/bson/src/bson-writer.ts index 56a9f38d2a..a2cbe749a5 100644 --- a/modules/bson/src/bson-writer.ts +++ b/modules/bson/src/bson-writer.ts @@ -1,23 +1,25 @@ // loaders.gl, MIT license import type {Writer, WriterOptions} from '@loaders.gl/loader-utils'; +import type {EncodeBSONOptions} from './lib/encoders/encode-bson'; import {encodeBSONSync} from './lib/encoders/encode-bson'; -// import type {Writer} from '@loaders.gl/loader-utils'; // __VERSION__ is injected by babel-plugin-version-inline // @ts-ignore TS2304: Cannot find name '__VERSION__'. const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest'; -export const BSONWriter: Writer = { +export type BSONWriterOptions = WriterOptions & { + bson?: EncodeBSONOptions +} + +export const BSONWriter: Writer, never, BSONWriterOptions> = { name: 'BSON', id: 'bson', module: 'bson', version: VERSION, extensions: ['bson'], options: { - image: { - mimeType: 'application/bson' - } + bson: {} }, async encode(data: Record, options?: WriterOptions): Promise { return encodeBSONSync(data, {}); // options diff --git a/modules/gltf/src/glb-loader.ts b/modules/gltf/src/glb-loader.ts index 11c1fe4c3d..d0fbfe2287 100644 --- a/modules/gltf/src/glb-loader.ts +++ b/modules/gltf/src/glb-loader.ts @@ -13,7 +13,7 @@ export type GLBLoaderOptions = LoaderOptions & { * GLB Loader - * GLB is the binary container format for GLTF */ -export const GLBLoader: LoaderWithParser = { +export const GLBLoader: LoaderWithParser = { name: 'GLB', id: 'glb', module: 'gltf', diff --git a/modules/gltf/src/glb-writer.ts b/modules/gltf/src/glb-writer.ts index 538046f221..b267b73a34 100644 --- a/modules/gltf/src/glb-writer.ts +++ b/modules/gltf/src/glb-writer.ts @@ -1,12 +1,18 @@ -import type {Writer} from '@loaders.gl/loader-utils'; -import {VERSION} from './lib/utils/version'; +import type {Writer, WriterOptions} from '@loaders.gl/loader-utils'; +import type {GLB} from './lib/types/glb-types'; +import type {GLBEncodeOptions} from './lib/encoders/encode-glb'; import encodeGLBSync from './lib/encoders/encode-glb'; +import {VERSION} from './lib/utils/version'; + +export type GLBWriterOptions = WriterOptions & { + glb?: GLBEncodeOptions; +}; /** * GLB exporter * GLB is the binary container format for GLTF */ -export const GLBWriter = { +export const GLBWriter: Writer = { name: 'GLB', id: 'glb', module: 'gltf', diff --git a/modules/gltf/src/lib/encoders/encode-glb.ts b/modules/gltf/src/lib/encoders/encode-glb.ts index 00b318d142..732ea790ee 100644 --- a/modules/gltf/src/lib/encoders/encode-glb.ts +++ b/modules/gltf/src/lib/encoders/encode-glb.ts @@ -10,10 +10,17 @@ const MAGIC_BIN = 0x004e4942; // BIN\0 in ASCII const LE = true; // Binary GLTF is little endian. +export type GLBEncodeOptions = {}; + // Encode the full GLB buffer with header etc // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0# // glb-file-format-specification -export default function encodeGLBSync(glb, dataView, byteOffset = 0, options = {}) { +export default function encodeGLBSync( + glb, + dataView, + byteOffset = 0, + options: GLBEncodeOptions = {} +) { const {magic = MAGIC_glTF, version = 2, json = {}, binary} = glb; const byteOffsetStart = byteOffset; diff --git a/modules/gltf/src/lib/encoders/encode-gltf.ts b/modules/gltf/src/lib/encoders/encode-gltf.ts index 9307097771..251f5e69d3 100644 --- a/modules/gltf/src/lib/encoders/encode-gltf.ts +++ b/modules/gltf/src/lib/encoders/encode-gltf.ts @@ -1,15 +1,24 @@ import encodeGLBSync from './encode-glb'; -// Encode the full glTF file as a binary GLB file -// Returns an ArrayBuffer that represents the complete GLB image that can be saved to file -// -// TODO - Does not support encoding to non-GLB versions of glTF format -// - Encode as a textual JSON file with binary data in base64 data URLs. -// - Encode as a JSON with all images (and buffers?) in separate binary files -// -// glb-file-format-specification -// https://github.com/KhronosGroup/glTF/tree/master/specification/2.0# +export type GLTFEncodeOptions = Record; +/** + * Encode the full glTF file as a binary GLB file + * Returns an ArrayBuffer that represents the complete GLB image that can be saved to file + * + * @todo - Does not support encoding to non-GLB versions of glTF format. Other formats + * - Encode as a textual JSON file with binary data in base64 data URLs. + * - Encode as a JSON with all images (and buffers?) in separate binary files + * + * glb-file-format-specification + * @see https://github.com/KhronosGroup/glTF/tree/master/specification/2.0# + * + * @param gltf + * @param arrayBuffer + * @param byteOffset + * @param options + * @returns + */ export function encodeGLTFSync(gltf, arrayBuffer, byteOffset, options) { convertBuffersToBase64(gltf); diff --git a/modules/gltf/src/lib/types/glb-types.ts b/modules/gltf/src/lib/types/glb-types.ts index dc4287e687..19435a0d7f 100644 --- a/modules/gltf/src/lib/types/glb-types.ts +++ b/modules/gltf/src/lib/types/glb-types.ts @@ -16,6 +16,6 @@ export type GLB = { }; // Per spec we must iterate over chunks, ignoring all except JSON and BIN - json: {}; + json: Record; binChunks: GLBBinChunk[]; }; diff --git a/modules/parquet/src/lib/arrow/convert-schema-from-parquet.ts b/modules/parquet/src/lib/arrow/convert-schema-from-parquet.ts index a8d409fa99..3c5063a8ce 100644 --- a/modules/parquet/src/lib/arrow/convert-schema-from-parquet.ts +++ b/modules/parquet/src/lib/arrow/convert-schema-from-parquet.ts @@ -4,7 +4,7 @@ import {Schema, Field, DataType} from '@loaders.gl/schema'; import type {ParquetSchema} from '../../parquetjs/schema/schema'; import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare'; -import {FileMetaData} from '@loaders.gl/parquet/parquetjs/parquet-thrift'; +import {FileMetaData} from '../../parquetjs/parquet-thrift'; export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = { BOOLEAN: 'bool', diff --git a/modules/ply/src/index.ts b/modules/ply/src/index.ts index 6ca0f6a128..b62c47f9b0 100644 --- a/modules/ply/src/index.ts +++ b/modules/ply/src/index.ts @@ -25,5 +25,3 @@ export const PLYLoader: LoaderWithParser = { parseSync: (arrayBuffer, options) => parsePLY(arrayBuffer, options?.ply), parseInBatches: (arrayBuffer, options) => parsePLYInBatches(arrayBuffer, options?.ply) }; - -export const _typecheckPLYLoader: LoaderWithParser = PLYLoader; diff --git a/modules/schema/src/lib/table/simple-table/table-accessors.ts b/modules/schema/src/lib/table/simple-table/table-accessors.ts index 793f12974c..0995648c49 100644 --- a/modules/schema/src/lib/table/simple-table/table-accessors.ts +++ b/modules/schema/src/lib/table/simple-table/table-accessors.ts @@ -14,15 +14,16 @@ export function getTableLength(table: Table): number { case 'geojson-row-table': return table.data.length; + case 'arrow-table': + return table.data.numRows; + case 'columnar-table': for (const column of Object.values(table.data)) { return column.length || 0; } return 0; - - case 'arrow-table': default: - return table.data.numRows; + throw new Error('table'); } } @@ -48,8 +49,9 @@ export function getTableNumCols(table: Table): number { return Object.keys(table.data).length; case 'arrow-table': - default: return table.data.numCols; + default: + throw new Error('table'); } } @@ -69,7 +71,10 @@ export function getTableCell(table: Table, rowIndex: number, columnName: string) return column[rowIndex]; case 'arrow-table': - return table.data.getChild(columnName)?.get(rowIndex); + const arrowColumnIndex = table.data.schema.fields.findIndex( + (field) => field.name === columnName + ); + return table.data.getChildAt(arrowColumnIndex)?.get(rowIndex); default: throw new Error('todo');