From c683916927f9824cd9020042c4e76a3ecf423b2d Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 22:46:33 +0900 Subject: [PATCH 1/7] add JavaScriptCodec to handle basic JavaScript objects --- src/JavaScriptCodec.ts | 59 +++++++++++++++++++++++++++++++++++ test/javascript-codec.test.ts | 28 +++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 src/JavaScriptCodec.ts create mode 100644 test/javascript-codec.test.ts diff --git a/src/JavaScriptCodec.ts b/src/JavaScriptCodec.ts new file mode 100644 index 00000000..330824c2 --- /dev/null +++ b/src/JavaScriptCodec.ts @@ -0,0 +1,59 @@ +import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; +import { encode } from "./encode"; +import { decode } from "./decode"; + +export const JavaScriptCodecType = 0; + +export function encodeJavaScriptData(input: unknown): Uint8Array | null { + if (input instanceof Map) { + return encode(["Map", [...input]]); + } else if (input instanceof Set) { + return encode(["Set", [...input]]); + } else if (input instanceof Date) { + // Not a MessagePack timestamp because + // it may be overrided by users + return encode(["Date", input.getTime()]); + } else if (input instanceof RegExp) { + return encode(["RegExp", [input.source, input.flags]]); + } else { + return null; + } +} + +export function decodeJavaScriptData(data: Uint8Array) { + const [constructor, source] = decode(data) as [string, any]; + + switch (constructor) { + case "undefined": { + return undefined; + } + case "Map": { + return new Map(source); + } + case "Set": { + return new Set(source); + } + case "Date": { + return new Date(source); + } + case "RegExp": { + const [pattern, flags] = source; + return new RegExp(pattern, flags); + } + default: { + throw new Error(`Unknown constructor: ${constructor}`); + } + } +} + +export const JavaScriptCodec: ExtensionCodecType = (() => { + const ext = new ExtensionCodec(); + + ext.register({ + type: JavaScriptCodecType, + encode: encodeJavaScriptData, + decode: decodeJavaScriptData, + }); + + return ext; +})(); diff --git a/test/javascript-codec.test.ts b/test/javascript-codec.test.ts new file mode 100644 index 00000000..0d5ea8c4 --- /dev/null +++ b/test/javascript-codec.test.ts @@ -0,0 +1,28 @@ +import assert from "assert"; +import { encode, decode } from "@msgpack/msgpack"; +import { JavaScriptCodec } from "src/JavaScriptCodec"; + +describe("JavaScriptCodec", () => { + context("mixed", () => { + // this data comes from https://github.com/yahoo/serialize-javascript + + it("encodes and decodes the object", () => { + const object = { + str: "string", + num: 0, + obj: { foo: "foo", bar: "bar" }, + arr: [1, 2, 3], + bool: true, + nil: null, + // undef: undefined, + date: new Date("Thu, 28 Apr 2016 22:02:17 GMT"), + map: new Map([["foo", 10], ["bar", 20]]), + set: new Set([123, 456]), + regexp: /foo\n/i, + }; + const encoded = encode(object, { extensionCodec: JavaScriptCodec }); + + assert.deepStrictEqual(decode(encoded, { extensionCodec: JavaScriptCodec }), object); + }); + }); +}); From 0bfb144faffe563cfac977c4ee51c4832001903d Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 22:50:50 +0900 Subject: [PATCH 2/7] use const enum for JavaScript data types --- src/JavaScriptCodec.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/JavaScriptCodec.ts b/src/JavaScriptCodec.ts index 330824c2..a5ff1279 100644 --- a/src/JavaScriptCodec.ts +++ b/src/JavaScriptCodec.ts @@ -4,44 +4,48 @@ import { decode } from "./decode"; export const JavaScriptCodecType = 0; +const enum JSData { + Map, + Set, + Date, + RegExp, +} + export function encodeJavaScriptData(input: unknown): Uint8Array | null { if (input instanceof Map) { - return encode(["Map", [...input]]); + return encode([JSData.Map, [...input]]); } else if (input instanceof Set) { - return encode(["Set", [...input]]); + return encode([JSData.Set, [...input]]); } else if (input instanceof Date) { // Not a MessagePack timestamp because // it may be overrided by users - return encode(["Date", input.getTime()]); + return encode([JSData.Date, input.getTime()]); } else if (input instanceof RegExp) { - return encode(["RegExp", [input.source, input.flags]]); + return encode([JSData.RegExp, [input.source, input.flags]]); } else { return null; } } export function decodeJavaScriptData(data: Uint8Array) { - const [constructor, source] = decode(data) as [string, any]; + const [jsDataType, source] = decode(data) as [JSData, any]; - switch (constructor) { - case "undefined": { - return undefined; - } - case "Map": { + switch (jsDataType) { + case JSData.Map: { return new Map(source); } - case "Set": { + case JSData.Set: { return new Set(source); } - case "Date": { + case JSData.Date: { return new Date(source); } - case "RegExp": { + case JSData.RegExp: { const [pattern, flags] = source; return new RegExp(pattern, flags); } default: { - throw new Error(`Unknown constructor: ${constructor}`); + throw new Error(`Unknown data type: ${jsDataType}`); } } } From 47ae1d4dfd8774753f4a3706188cf661f6207d14 Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 22:55:05 +0900 Subject: [PATCH 3/7] add BigInt support in JavaScriptCodec --- src/JavaScriptCodec.ts | 6 ++++++ test/javascript-codec.test.ts | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/JavaScriptCodec.ts b/src/JavaScriptCodec.ts index a5ff1279..a30958ea 100644 --- a/src/JavaScriptCodec.ts +++ b/src/JavaScriptCodec.ts @@ -9,6 +9,7 @@ const enum JSData { Set, Date, RegExp, + BigInt, } export function encodeJavaScriptData(input: unknown): Uint8Array | null { @@ -22,6 +23,8 @@ export function encodeJavaScriptData(input: unknown): Uint8Array | null { return encode([JSData.Date, input.getTime()]); } else if (input instanceof RegExp) { return encode([JSData.RegExp, [input.source, input.flags]]); + } else if (typeof input === "bigint") { + return encode([JSData.BigInt, input.toString()]); } else { return null; } @@ -44,6 +47,9 @@ export function decodeJavaScriptData(data: Uint8Array) { const [pattern, flags] = source; return new RegExp(pattern, flags); } + case JSData.BigInt: { + return BigInt(source); + } default: { throw new Error(`Unknown data type: ${jsDataType}`); } diff --git a/test/javascript-codec.test.ts b/test/javascript-codec.test.ts index 0d5ea8c4..445f36b6 100644 --- a/test/javascript-codec.test.ts +++ b/test/javascript-codec.test.ts @@ -14,11 +14,12 @@ describe("JavaScriptCodec", () => { arr: [1, 2, 3], bool: true, nil: null, - // undef: undefined, + // undef: undefined, // not supported date: new Date("Thu, 28 Apr 2016 22:02:17 GMT"), map: new Map([["foo", 10], ["bar", 20]]), set: new Set([123, 456]), regexp: /foo\n/i, + bigint: typeof(BigInt) !== "undefined" ? BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1) : null, }; const encoded = encode(object, { extensionCodec: JavaScriptCodec }); From f156faebc615b3e265414804b15a70cc40b5d99e Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 22:57:03 +0900 Subject: [PATCH 4/7] export JavaScriptCodec stuff in index.ts --- src/index.ts | 2 ++ test/javascript-codec.test.ts | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 50b50813..1d56fefb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,4 +20,6 @@ export { decodeTimestampExtension, } from "./timestamp"; +export { JavaScriptCodec, JavaScriptCodecType, encodeJavaScriptData, decodeJavaScriptData } from "./JavaScriptCodec"; + export { WASM_AVAILABLE as __WASM_AVAILABLE } from "./wasmFunctions"; diff --git a/test/javascript-codec.test.ts b/test/javascript-codec.test.ts index 445f36b6..2769ec80 100644 --- a/test/javascript-codec.test.ts +++ b/test/javascript-codec.test.ts @@ -1,6 +1,5 @@ import assert from "assert"; -import { encode, decode } from "@msgpack/msgpack"; -import { JavaScriptCodec } from "src/JavaScriptCodec"; +import { encode, decode, JavaScriptCodec } from "@msgpack/msgpack"; describe("JavaScriptCodec", () => { context("mixed", () => { From 4cef7e1d1e8ad3c5e818a999dab9bcec1ff0c801 Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 23:07:19 +0900 Subject: [PATCH 5/7] rename --- src/JavaScriptCodec.ts | 4 ++-- src/index.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/JavaScriptCodec.ts b/src/JavaScriptCodec.ts index a30958ea..04fbf5d1 100644 --- a/src/JavaScriptCodec.ts +++ b/src/JavaScriptCodec.ts @@ -2,7 +2,7 @@ import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; import { encode } from "./encode"; import { decode } from "./decode"; -export const JavaScriptCodecType = 0; +export const EXT_JAVASCRIPT = 0; const enum JSData { Map, @@ -60,7 +60,7 @@ export const JavaScriptCodec: ExtensionCodecType = (() => { const ext = new ExtensionCodec(); ext.register({ - type: JavaScriptCodecType, + type: EXT_JAVASCRIPT, encode: encodeJavaScriptData, decode: decodeJavaScriptData, }); diff --git a/src/index.ts b/src/index.ts index 1d56fefb..b1eb94a0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,6 @@ export { decodeTimestampExtension, } from "./timestamp"; -export { JavaScriptCodec, JavaScriptCodecType, encodeJavaScriptData, decodeJavaScriptData } from "./JavaScriptCodec"; +export { JavaScriptCodec, EXT_JAVASCRIPT, encodeJavaScriptData, decodeJavaScriptData } from "./JavaScriptCodec"; export { WASM_AVAILABLE as __WASM_AVAILABLE } from "./wasmFunctions"; From a5a5d1bc7f19a7e7ca948012f1f5c855da3c5d15 Mon Sep 17 00:00:00 2001 From: "FUJI Goro (gfx)" Date: Tue, 11 Jun 2019 23:11:29 +0900 Subject: [PATCH 6/7] try to encode an object with custom encoder to override built-in's --- package.json | 2 +- src/ExtensionCodec.ts | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index bf1bd324..b197577c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test:cover:wasm": "npx nyc --no-clean npm run test:wasm", "test:cover:td": "npx nyc --no-clean npm run test:td", "cover:clean": "rimraf .nyc_output coverage/", - "cover:report": "nyc report --reporter=text-summary --reporter=html --reporter=json", + "cover:report": "npx nyc report --reporter=text-summary --reporter=html --reporter=json", "test:browser": "karma start --single-run", "test:browser:firefox": "karma start --single-run --browsers FirefoxHeadless", "test:browser:chrome": "karma start --single-run --browsers ChromeHeadless", diff --git a/src/ExtensionCodec.ts b/src/ExtensionCodec.ts index 3f12cb3b..ece0e9c2 100644 --- a/src/ExtensionCodec.ts +++ b/src/ExtensionCodec.ts @@ -50,25 +50,25 @@ export class ExtensionCodec implements ExtensionCodecType { } public tryToEncode(object: unknown): ExtData | null { - // built-in extensions - for (let i = 0; i < this.builtInEncoders.length; i++) { - const encoder = this.builtInEncoders[i]; + // custom extensions + for (let i = 0; i < this.encoders.length; i++) { + const encoder = this.encoders[i]; if (encoder != null) { const data = encoder(object); if (data != null) { - const type = -1 - i; + const type = i; return new ExtData(type, data); } } } - // custom extensions - for (let i = 0; i < this.encoders.length; i++) { - const encoder = this.encoders[i]; + // built-in extensions + for (let i = 0; i < this.builtInEncoders.length; i++) { + const encoder = this.builtInEncoders[i]; if (encoder != null) { const data = encoder(object); if (data != null) { - const type = i; + const type = -1 - i; return new ExtData(type, data); } } From 9737a9b83c1c504d1ed1c20c0d76d0a1a67d56c2 Mon Sep 17 00:00:00 2001 From: FUJI Goro Date: Sun, 3 Jan 2021 15:00:58 +0900 Subject: [PATCH 7/7] WIP for structured clone algo --- src/JavaScriptCodec.ts | 166 ++++++++++++++++++++++++++-------- src/index.ts | 14 ++- test/javascript-codec.test.ts | 38 ++++++-- 3 files changed, 174 insertions(+), 44 deletions(-) diff --git a/src/JavaScriptCodec.ts b/src/JavaScriptCodec.ts index 3e47cf6d..43feb52d 100644 --- a/src/JavaScriptCodec.ts +++ b/src/JavaScriptCodec.ts @@ -1,57 +1,151 @@ +// Implementation of "Structured Clone" algorithm in MessagPack +// https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm + import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec"; import { encode } from "./encode"; import { decode } from "./decode"; export const EXT_JAVASCRIPT = 0; -const enum JSData { - Map, - Set, - Date, - RegExp, - BigInt, +const enum JS { + // defined in "structured clone algorithm" + // commente-outed ones are TODOs + + // Boolean = "Boolean", + // String = "String", + Date = "Date", + RegExp = "RegExp", + // Blob = "Blob", + // File = "File", + // FileList = "FileList", + ArrayBuffer = "ArrayBuffer", + Int8Array = "Int8Array", + Uint8Array = "Uint8Array", + Uint8ClampedArray = "Uint8ClampedArray", + Int16Array = "Int16Array", + Uint16Array = "Uint16Array", + Int32Array = "Int32Array", + Uint32Array = "Uint32Array", + Float32Array = "Float32Array", + Float64Array = "Float64Array", + BigInt64Array = "BigInt64Array", + BigUint64Array = "BigUint64Array", + DataView = "DataView", + // ImageBitMap = "ImageBitMap", + // ImageData = "ImageData", + Map = "Map", + Set = "Set", + + // and more + BigInt = "BigInt", } -export function encodeJavaScriptData(input: unknown): Uint8Array | null { - if (input instanceof Map) { - return encode([JSData.Map, [...input]]); - } else if (input instanceof Set) { - return encode([JSData.Set, [...input]]); - } else if (input instanceof Date) { - // Not a MessagePack timestamp because - // it may be overrided by users - return encode([JSData.Date, input.getTime()]); - } else if (input instanceof RegExp) { - return encode([JSData.RegExp, [input.source, input.flags]]); - } else if (typeof input === "bigint") { - return encode([JSData.BigInt, input.toString()]); +export function encodeJavaScriptStructure(input: unknown): Uint8Array | null { + if (!(input instanceof Object)) { + if (typeof input === "bigint") { + return encode([JS.BigInt, input.toString()]); + } else { + return null; + } + } + const type = input.constructor.name; + + if (ArrayBuffer.isView(input)) { + if (type === JS.Uint8Array) { + return null; // fall through to the default encoder + } else if (type === JS.DataView || type === JS.Int8Array || type === JS.Uint8ClampedArray) { + // handles them as a byte buffer + const v = new Uint8Array(input.buffer, input.byteOffset, input.byteLength) + return encode([type, v]); + } else { + // handles them as a number array for portability + return encode([type, ...(input as unknown as Iterable)]); + } + } else if (type === JS.ArrayBuffer) { + const bufferView = new Uint8Array(input as ArrayBuffer); + return encode([type, bufferView]); + } else if (type === JS.Map) { + return encode([JS.Map, ...input as Map]); + } else if (type === JS.Set) { + return encode([JS.Set, ...input as Set]); + } else if (type === JS.Date) { + return encode([JS.Date, (input as Date).getTime()]); + } else if (type === JS.RegExp) { + return encode([JS.RegExp, (input as RegExp).source, (input as RegExp).flags]); } else { return null; } } -export function decodeJavaScriptData(data: Uint8Array) { - const [jsDataType, source] = decode(data) as [JSData, any]; - - switch (jsDataType) { - case JSData.Map: { - return new Map(source); - } - case JSData.Set: { - return new Set(source); +export function decodeJavaScriptStructure(data: Uint8Array) { + const [type, ...source] = decode(data) as [JS, ...any]; + switch (type) { + case JS.BigInt: { + const [str] = source; + return BigInt(str); } - case JSData.Date: { - return new Date(source); + case JS.Date: { + const [millis] = source; + return new Date(millis); } - case JSData.RegExp: { + case JS.RegExp: { const [pattern, flags] = source; return new RegExp(pattern, flags); } - case JSData.BigInt: { - return BigInt(source); + case JS.ArrayBuffer: { + const [buffer] = source as [Uint8Array]; + return buffer.slice(0).buffer; + } + case JS.Int8Array: { + const [v] = source as [Uint8Array]; + return new Int8Array(v.buffer, v.byteOffset, v.byteLength); + } + case JS.Uint8Array: { + // unlikely because it is handled by the default decoder, + // but technically possible with no conflict. + const [v] = source as [Uint8Array]; + return new Uint8Array(v.buffer, v.byteOffset, v.byteLength); + } + case JS.Uint8ClampedArray: { + const [v] = source as [Uint8Array]; + return new Uint8ClampedArray(v.buffer, v.byteOffset, v.byteLength); + } + case JS.Int16Array: { + return Int16Array.from(source as ReadonlyArray); + } + case JS.Uint16Array: { + return Uint16Array.from(source as ReadonlyArray); + } + case JS.Int32Array: { + return Int32Array.from(source as ReadonlyArray); + } + case JS.Uint32Array: { + return Uint32Array.from(source as ReadonlyArray); + } + case JS.Float32Array: { + return Float32Array.from(source as ReadonlyArray); + } + case JS.Float64Array: { + return Float64Array.from(source as ReadonlyArray); + } + case JS.BigInt64Array: { + return BigInt64Array.from(source as ReadonlyArray); + } + case JS.BigUint64Array: { + return BigUint64Array.from(source as ReadonlyArray); + } + case JS.DataView: { + const [v] = source as [Uint8Array]; + return new DataView(v.buffer, v.byteOffset, v.byteLength); + } + case JS.Map: { + return new Map(source); + } + case JS.Set: { + return new Set(source); } default: { - throw new Error(`Unknown data type: ${jsDataType}`); + throw new Error(`Unknown data type: ${type}`); } } } @@ -61,8 +155,8 @@ export const JavaScriptCodec: ExtensionCodecType = (() => { ext.register({ type: EXT_JAVASCRIPT, - encode: encodeJavaScriptData, - decode: decodeJavaScriptData, + encode: encodeJavaScriptStructure, + decode: decodeJavaScriptStructure, }); return ext; diff --git a/src/index.ts b/src/index.ts index 961585e9..9851819f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -45,5 +45,15 @@ export { decodeTimestampExtension, }; -export { JavaScriptCodec, EXT_JAVASCRIPT, encodeJavaScriptData, decodeJavaScriptData } from "./JavaScriptCodec"; - +import { + JavaScriptCodec, + EXT_JAVASCRIPT, + encodeJavaScriptStructure, + decodeJavaScriptStructure, +} from "./JavaScriptCodec"; +export { + JavaScriptCodec, + EXT_JAVASCRIPT, + encodeJavaScriptStructure, + decodeJavaScriptStructure, +}; diff --git a/test/javascript-codec.test.ts b/test/javascript-codec.test.ts index 2769ec80..9bfe094b 100644 --- a/test/javascript-codec.test.ts +++ b/test/javascript-codec.test.ts @@ -3,25 +3,51 @@ import { encode, decode, JavaScriptCodec } from "@msgpack/msgpack"; describe("JavaScriptCodec", () => { context("mixed", () => { - // this data comes from https://github.com/yahoo/serialize-javascript - - it("encodes and decodes the object", () => { + it("encodes and decodes structured data", () => { const object = { + // basic str: "string", num: 0, obj: { foo: "foo", bar: "bar" }, arr: [1, 2, 3], bool: true, nil: null, - // undef: undefined, // not supported + + // JavaScript structures date: new Date("Thu, 28 Apr 2016 22:02:17 GMT"), + regexp: /foo\n/i, + arrayBuffer: Uint8Array.from([0, 1, 2, 0xff]).buffer, + int8Array: Int8Array.from([0, 1, 2, 0xff]), + uint8ClampedArray: Uint8ClampedArray.from([0, 1, 2, 0xff]), + int16Array: Int16Array.from([0, 1, 2, 0xffff]), + uint16Array: Uint16Array.from([0, 1, 2, -1]), + int32Array: Int32Array.from([0, 1, 2, 0xffff]), + uint32Array: Uint32Array.from([0, 1, 2, -1]), + float32Array: Float32Array.from([0, 1, 2, Math.PI, Math.E]), + float64Array: Float64Array.from([0, 1, 2, Math.PI, Math.E]), map: new Map([["foo", 10], ["bar", 20]]), set: new Set([123, 456]), - regexp: /foo\n/i, - bigint: typeof(BigInt) !== "undefined" ? BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1) : null, }; const encoded = encode(object, { extensionCodec: JavaScriptCodec }); + assert.deepStrictEqual(decode(encoded, { extensionCodec: JavaScriptCodec }), object); + }); + }); + + context("bigint and its family", () => { + it("encodes and decodes structured data", function () { + if (typeof BigInt === "undefined" || typeof BigInt64Array === "undefined" || typeof BigUint64Array === "undefined") { + this.skip(); + } + + const object = { + bigint: BigInt(42), + bigintArray: [BigInt(42)], + // TODO: + //bigInt64array: BigInt64Array.from([BigInt(0), BigInt(Number.MIN_SAFE_INTEGER) - BigInt(1)]), + //bigUint64array: BigUint64Array.from([BigInt(0), BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)]), + }; + const encoded = encode(object, { extensionCodec: JavaScriptCodec }); assert.deepStrictEqual(decode(encoded, { extensionCodec: JavaScriptCodec }), object); }); });