Skip to content

Commit

Permalink
fixing and adding payload discrimination for VAAs
Browse files Browse the repository at this point in the history
  • Loading branch information
@nonergodic
nonergodic committed Oct 13, 2023
1 parent a69ec0b commit fe719a6
Show file tree
Hide file tree
Showing 15 changed files with 266 additions and 254 deletions.
6 changes: 3 additions & 3 deletions core/base/__tests__/layout.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ describe("Layout tests", function () {
[{name: "type", binary: "uint", size: 1, custom: 0}],
[{name: "type", binary: "uint", size: 1, custom: 2}],
]);

expect(discriminator(Uint8Array.from([0]))).toBe(0);
expect(discriminator(Uint8Array.from([2]))).toBe(1);
expect(discriminator(Uint8Array.from([1]))).toBe(null);
Expand Down Expand Up @@ -188,7 +188,7 @@ describe("Layout tests", function () {
{name: "data", binary: "uint", size: 1}],
[{name: "type", binary: "uint", size: 1}],
]);

expect(discriminator(Uint8Array.from([0, 7]))).toBe(0);
expect(discriminator(Uint8Array.from([0]))).toBe(1);
expect(discriminator(Uint8Array.from([1]))).toBe(1);
Expand Down Expand Up @@ -235,7 +235,7 @@ describe("Layout tests", function () {
[{name: "type", binary: "uint", size: 1}],
[{name: "type", binary: "uint", size: 1}],
[
{name: "type", binary: "uint", size: 1},
{name: "type", binary: "uint", size: 1},
{name: "data", binary: "uint", size: 1}
],
] as readonly Layout[];
Expand Down
32 changes: 17 additions & 15 deletions core/base/src/utils/layout/deserialize.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@ import {
Layout,
LayoutItem,
LayoutToType,
LayoutItemToType,
FixedPrimitiveBytesLayoutItem,
FixedValueBytesLayoutItem,
CustomConversion,
UintSizeToPrimitive,
UintType,
BytesType,
isUintType,
isBytesType,
numberMaxSize,
} from "./layout";

Expand Down Expand Up @@ -109,9 +112,9 @@ function deserializeLayoutItem(
let fixedFrom;
let fixedTo;
if (item.custom !== undefined) {
if (item.custom instanceof Uint8Array)
if (isBytesType(item.custom))
fixedFrom = item.custom;
else if (item.custom.from instanceof Uint8Array) {
else if (isBytesType(item.custom.from)) {
fixedFrom = item.custom.from;
fixedTo = item.custom.to;
}
Expand Down Expand Up @@ -139,7 +142,7 @@ function deserializeLayoutItem(
return [fixedTo ?? fixedFrom, newOffset];
}

type narrowedCustom = CustomConversion<Uint8Array, any>;
type narrowedCustom = CustomConversion<BytesType, any>;
return [
item.custom !== undefined ? (item.custom as narrowedCustom).to(value) : value,
newOffset
Expand All @@ -148,21 +151,20 @@ function deserializeLayoutItem(
case "uint": {
const [value, newOffset] = deserializeUint(encoded, offset, item.size);

if (item.custom !== undefined) {
if (typeof item.custom === "number" || typeof item.custom === "bigint") {
checkUintEquals(item.custom, value);
return [item.custom, newOffset];
}
else if (typeof item.custom.from === "number" || typeof item.custom.from === "bigint") {
checkUintEquals(item.custom.from, value);
return [item.custom.to, newOffset];
}
if (isUintType(item.custom)) {
checkUintEquals(item.custom, value);
return [item.custom, newOffset];
}

if (isUintType(item?.custom?.from)) {
checkUintEquals(item!.custom!.from, value);
return [item!.custom!.to, newOffset];
}

//narrowing to CustomConver<number | bigint, any> is a bit hacky here, since the true type
//narrowing to CustomConver<UintType, any> is a bit hacky here, since the true type
// would be CustomConver<number, any> | CustomConver<bigint, any>, but then we'd have to
// further tease that apart still for no real gain...
type narrowedCustom = CustomConversion<number | bigint, any>;
type narrowedCustom = CustomConversion<UintType, any>;
return [
item.custom !== undefined ? (item.custom as narrowedCustom).to(value) : value,
newOffset
Expand Down
55 changes: 30 additions & 25 deletions core/base/src/utils/layout/discriminate.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ import {
Layout,
LayoutItem,
LengthPrefixedBytesLayoutItem,
isPrimitiveType,
isUintType,
isBytesType,
} from "./layout";

import { serializeUint } from "./serialize";
Expand All @@ -20,7 +21,7 @@ type FixedBytes = (readonly [BytePos, Uint8Array])[];
// bound or Infinity) in anticipation of a future switch layout item that might contain multiple
// sublayouts which, unlike arrays currently, could all be bounded but potentially with
// different sizes
type Bounds = readonly [Size, Size];
type Bounds = [Size, Size];

function arrayToBitset(arr: readonly number[]): Bitset {
return arr.reduce((bit, i) => bit | BigInt(1) << BigInt(i), BigInt(0));
Expand All @@ -44,11 +45,11 @@ function count(candidates: Candidates) {

function layoutItemMeta(
item: LayoutItem,
offset: BytePos,
offset: BytePos | null,
fixedBytes: FixedBytes,
): Bounds {
function knownFixed(size: Size, serialized: Uint8Array): Bounds {
if (Number.isFinite(offset))
if (offset !== null)
fixedBytes.push([offset, serialized]);

return [size, size];
Expand All @@ -65,21 +66,28 @@ function layoutItemMeta(
if ("size" in item && item.size !== undefined)
return [item.size, item.size];

if (item?.custom instanceof Uint8Array)
if (isBytesType(item?.custom))
return knownFixed(item.custom.length, item.custom);

if (item?.custom?.from instanceof Uint8Array)
return knownFixed(item.custom.from.length, item.custom.from);
if (isBytesType(item?.custom?.from))
return knownFixed(item!.custom!.from.length, item!.custom!.from);

//TODO typescript should be able to infer that at this point the only possible remaining
// type for item is LengthPrefixedBytesLayoutItem, but for some reason it doesn't
item = item as LengthPrefixedBytesLayoutItem;
return [item.lengthSize !== undefined ? item.lengthSize : 0, Infinity];
}
case "uint": {
if (isPrimitiveType(item.custom)) {
const fixedVal =
isUintType(item.custom)
? item.custom
: isUintType(item?.custom?.from)
? item!.custom!.from
: null;

if (fixedVal !== null) {
const serialized = new Uint8Array(item.size);
serializeUint(serialized, 0, item.custom, item.size)
serializeUint(serialized, 0, fixedVal, item.size)
return knownFixed(item.size, serialized);
}

Expand All @@ -90,16 +98,17 @@ function layoutItemMeta(

function createLayoutMeta(
layout: Layout,
offset: BytePos,
offset: BytePos | null,
fixedBytes: FixedBytes
): Bounds {
let bounds = [0, 0] as Bounds;
for (const item of layout) {
bounds = layoutItemMeta(item, offset, fixedBytes)
//we need the cast because of course mapping tuples to tuples is an unsolved problem in TS:
//https://stackoverflow.com/questions/57913193/how-to-use-array-map-with-tuples-in-typescript#answer-57913509
.map((b, i) => bounds[i] + b) as unknown as Bounds;
offset = bounds[0] === bounds[1] ? bounds[0] : Infinity;
const itemSize = layoutItemMeta(item, offset, fixedBytes);
bounds[0] += itemSize[0];
bounds[1] += itemSize[1];
//if the bounds don't agree then we can't reliably predict the offset of subsequent items
if (offset !== null)
offset = itemSize[0] === itemSize[1] ? offset + itemSize[0] : null;
}
return bounds;
}
Expand Down Expand Up @@ -279,15 +288,9 @@ function generateLayoutDiscriminator(
type Strategy = [BytePos, Candidates, Map<number, Candidates>] | "size" | "indistinguishable";

let distinguishable = true;
let firstStrategy: Strategy | undefined;
const strategies = new Map<Candidates, Strategy>();
const candidatesBySize = new Map<Size, Candidates[]>();
const addStrategy = (candidates: Candidates, strategy: Strategy) => {
if (firstStrategy === undefined) {
firstStrategy = strategy;
return;
}

strategies.set(candidates, strategy);
if (!candidatesBySize.has(count(candidates)))
candidatesBySize.set(count(candidates), []);
Expand Down Expand Up @@ -394,8 +397,12 @@ function generateLayoutDiscriminator(

return [distinguishable, (encoded: Uint8Array) => {
let candidates = allLayouts;
let strategy = firstStrategy!;
while (strategy !== "indistinguishable") {

for (
let strategy = strategies.get(candidates)!;
strategy !== "indistinguishable";
strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates)
) {
if (strategy === "size")
candidates &= layoutsWithSize(encoded.length);
else {
Expand All @@ -414,8 +421,6 @@ function generateLayoutDiscriminator(

if (count(candidates) <= 1)
return bitsetToArray(candidates);

strategy = strategies.get(candidates) ?? findSmallestSuperSetStrategy(candidates);
}

return bitsetToArray(candidates);
Expand Down
48 changes: 22 additions & 26 deletions core/base/src/utils/layout/layout.ts
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
//TODO:
// * make FixedItem recursive
// * implement a swtich layout item that maps different values (versions) to different sublayouts
// * implement a method that determines the total size of a layout, if all items have known size
// * implement a method that determines the offsets of items in a layout (if all preceding items
// have known, fixed size (i.e. no arrays))
// * leverage the above to implement deserialization of just a set of fields of a layout
// * implement a method that takes several layouts and a serialized piece of data and quickly
// determines which layouts this payload conforms to (might be 0 or even all!). Should leverage
// the above methods and fixed values in the layout to quickly exclude candidates.
// * implement a method that allows "raw" serialization and deserialization" i.e. that skips all the
// custom conversions (should only be used for testing!) or even just partitions i.e. slices
// the encoded Uint8Array

export type PrimitiveType = number | bigint | Uint8Array;
// * implement a switch layout item that maps different values (versions) to different sublayouts

export type UintType = number | bigint;
export const isUintType = (x: any): x is UintType =>
typeof x === "number" || typeof x === "bigint";

export type BytesType = Uint8Array;
export const isBytesType = (x: any): x is BytesType => x instanceof Uint8Array;

export type PrimitiveType = UintType | BytesType;
export const isPrimitiveType = (x: any): x is PrimitiveType =>
typeof x === "number" || typeof x === "bigint" || x instanceof Uint8Array;
isUintType(x) || isBytesType(x);

export type BinaryLiterals = "uint" | "bytes" | "array" | "object";

Expand Down Expand Up @@ -57,34 +53,34 @@ interface OptionalToFromCustom<T extends PrimitiveType> {
};

//size: number of bytes used to encode the item
interface UintLayoutItemBase<T extends number | bigint> extends LayoutItemBase<"uint"> {
interface UintLayoutItemBase<T extends UintType> extends LayoutItemBase<"uint"> {
size: T extends bigint ? number : NumberSize,
};

export interface PrimitiveFixedUintLayoutItem<T extends number | bigint>
export interface PrimitiveFixedUintLayoutItem<T extends UintType>
extends UintLayoutItemBase<T>, PrimitiveFixedCustom<T> {};

export interface OptionalToFromUintLayoutItem<T extends number | bigint>
export interface OptionalToFromUintLayoutItem<T extends UintType>
extends UintLayoutItemBase<T>, OptionalToFromCustom<T> {};

export interface FixedPrimitiveBytesLayoutItem
extends LayoutItemBase<"bytes">, PrimitiveFixedCustom<Uint8Array> {};
extends LayoutItemBase<"bytes">, PrimitiveFixedCustom<BytesType> {};

export interface FixedValueBytesLayoutItem extends LayoutItemBase<"bytes"> {
readonly custom: FixedConversion<Uint8Array, any>,
readonly custom: FixedConversion<BytesType, any>,
};

export interface FixedSizeBytesLayoutItem extends LayoutItemBase<"bytes"> {
readonly size: number,
readonly custom?: CustomConversion<Uint8Array, any>,
readonly custom?: CustomConversion<BytesType, any>,
};

//length size: number of bytes used to encode the preceeding length field which in turn
// hold either the number of bytes (for bytes) or elements (for array)
// undefined means it will consume the rest of the data
export interface LengthPrefixedBytesLayoutItem extends LayoutItemBase<"bytes"> {
readonly lengthSize?: NumberSize,
readonly custom?: CustomConversion<Uint8Array, any>,
readonly custom?: CustomConversion<BytesType, any>,
};

export interface ArrayLayoutItem extends LayoutItemBase<"array"> {
Expand Down Expand Up @@ -124,17 +120,17 @@ export type LayoutItemToType<I extends LayoutItem> =
: [I] extends [ObjectLayoutItem]
? LayoutToType<I["layout"]>
: [I] extends [UintLayoutItem]
? I["custom"] extends number | bigint
? I["custom"] extends UintType
? I["custom"]
: I["custom"] extends CustomConversion<any, infer ToType>
? ToType
: I["custom"] extends FixedConversion<any, infer ToType>
? ToType
: UintSizeToPrimitive<I["size"]>
: [I] extends [BytesLayoutItem]
? I["custom"] extends CustomConversion<Uint8Array, infer ToType>
? I["custom"] extends CustomConversion<BytesType, infer ToType>
? ToType
: I["custom"] extends FixedConversion<Uint8Array, infer ToType>
: I["custom"] extends FixedConversion<BytesType, infer ToType>
? ToType
: Uint8Array
: BytesType //this also covers FixedValueBytesLayoutItem (Uint8Arrays don't support literals)
: never;
Loading

0 comments on commit fe719a6

Please sign in to comment.