diff --git a/CHANGELOG.md b/CHANGELOG.md index d7f5e49cb..1eab74865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ history. ## Unreleased +- `fjs/text/ascii` owns the hex-digit codec: `hexDigitValue`, + `hexDigitCodePoint`, and the `a-f` / `A-F` ranges. The JSON serializer and + both tokenizers use it instead of rederiving the offsets; the DJS tokenizer + no longer decodes a non-hex code point to a garbage digit + [#1522](https://github.com/functionalscript/functionalscript/pull/1522) - `fjs/types/result` exports `okThen`, the pure `Result` bind — `mapOk`'s monad sibling and the pure twin of `fjs/effects`' `okStep`. Its two error types are unioned, so a chain that widens its error needs no rewrapping diff --git a/fjs/djs/tokenizer/module.f.mjs b/fjs/djs/tokenizer/module.f.mjs index 5ee4f3c0e..031dcac6e 100644 --- a/fjs/djs/tokenizer/module.f.mjs +++ b/fjs/djs/tokenizer/module.f.mjs @@ -52,14 +52,12 @@ import { multiply } from '../../types/bigfloat/module.f.mjs' import { asterisk, backspace, ht, lf, ff, cr, quotationMark, solidus, reverseSolidus, - digitRange, digit0, - latinCapitalLetterA, - latinSmallLetterA, latinSmallLetterB, latinSmallLetterF, + hexDigitValue, + latinSmallLetterB, latinSmallLetterF, latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU, - range as asciiRange, } from '../../text/ascii/module.f.mjs' import { codePointListToString, stringToCodePointList } from '../../text/utf16/module.f.mjs' -import { contains } from '../../types/range/module.f.mjs' +import { mapUnwrap } from '../../types/nullable/module.f.mjs' import { concat, empty, filter, flat, flatMap, fold, map, stateScan, toArray } from '../../types/list/module.f.mjs' import { stringifyAsTree } from '../serializer/module.f.mjs' import { sort } from '../../types/object/module.f.mjs' @@ -333,7 +331,13 @@ const filterFunc = tk => { } } -const rangeCapitalAF = asciiRange('AF') +/** + * A `\uXXXX` escape reaches the `unicode` state only after the grammar has + * accepted its four hex digits, so a non-hex code point here is a tokenizer + * bug rather than bad input — assert instead of decoding it to a garbage + * value, which is what the hand-rolled ternary chain used to do. + */ +const unwrapHexDigitValue = mapUnwrap(hexDigitValue) /** @typedef { * | { readonly kind: 'normal' } @@ -358,11 +362,7 @@ const stringDecodeScan = (cp, state) => { default: return [[cp], { kind: 'normal' }] } case 'unicode': { - // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f' - const digit = contains(...digitRange)(cp) ? cp - digit0 - : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10) - : cp - (latinSmallLetterA - 10) - const acc = (state.acc << 4) | digit + const acc = (state.acc << 4) | unwrapHexDigitValue(cp) return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }] } default: diff --git a/fjs/js/tokenizer/module.f.mjs b/fjs/js/tokenizer/module.f.mjs index fb2d0b110..597f0446f 100644 --- a/fjs/js/tokenizer/module.f.mjs +++ b/fjs/js/tokenizer/module.f.mjs @@ -51,8 +51,9 @@ import { digit0, colon, // + hexDigitValue, + // latinCapitalLetterRange, - latinCapitalLetterA, latinCapitalLetterE, // leftSquareBracket, @@ -61,7 +62,6 @@ import { lowLine, // latinSmallLetterRange, - latinSmallLetterA, latinSmallLetterB, latinSmallLetterE, latinSmallLetterF, @@ -150,9 +150,6 @@ const rangeSetTerminalForNumber = [ one(tilde), ] -const rangeSmallAF = range('af') -const rangeCapitalAF = range('AF') - const rangeIdStart = [ latinSmallLetterRange, latinCapitalLetterRange, @@ -596,22 +593,22 @@ const parseUnicodeCharDefault = state => input => { return [{ first: { kind: 'error', message: 'invalid hex value' }, tail: next[0] }, next[1]] } -/** @type {(offset: number) => _CreateToToken<_ParseUnicodeCharState>} */ -const parseUnicodeCharHex = offset => state => input => { - const hexValue = input - offset +/** + * `hexDigitValue` classifies the code point and decodes it in one step, so this + * state needs no range-map dispatch: a `null` value is exactly the non-hex + * input the default handler rejects. + * + * @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List, _TokenizerState]} + */ +const parseUnicodeCharStateOp = state => input => { + const hexValue = hexDigitValue(input) + if (hexValue === null) { return parseUnicodeCharDefault(state)(input) } const newUnicode = state.unicode | (hexValue << (3 - state.hexIndex) * 4) return [empty, state.hexIndex === 3 ? { kind: 'string', value: appendChar(state.value)(newUnicode) } : { kind: 'unicodeChar', value: state.value, unicode: newUnicode, hexIndex: state.hexIndex + 1 }] } -/** @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List, _TokenizerState]} */ -const parseUnicodeCharStateOp = create(parseUnicodeCharDefault)([ - rangeFunc(digitRange)(parseUnicodeCharHex(digit0)), - rangeFunc(rangeSmallAF)(parseUnicodeCharHex(latinSmallLetterA - 10)), - rangeFunc(rangeCapitalAF)(parseUnicodeCharHex(latinCapitalLetterA - 10)) -]) - /** @type {(s: string) => JsToken} */ const idToToken = s => at(s)(keywordMap) ?? { kind: 'id', value: s } diff --git a/fjs/media/json/serializer/module.f.mjs b/fjs/media/json/serializer/module.f.mjs index a9b4e102a..ade51dc06 100644 --- a/fjs/media/json/serializer/module.f.mjs +++ b/fjs/media/json/serializer/module.f.mjs @@ -17,10 +17,9 @@ import { errorMask } from '../../../text/code_point/module.f.mjs' import { backspace, cr, - digit0, ff, + hexDigitCodePoint, ht, - latinSmallLetterA, lf, quotationMark, reverseSolidus, @@ -46,8 +45,7 @@ const escapeTable = /** @type {const} */ ({ }) /** @type {(value: number) => string} */ -const hexDigit = value => - fromCharCode(value < 10 ? digit0 + value : latinSmallLetterA + value - 10) +const hexDigit = value => fromCharCode(hexDigitCodePoint(value)) /** * `\uXXXX` with lowercase hex digits, matching ECMAScript's `UnicodeEscape`. diff --git a/fjs/text/ascii/module.f.mjs b/fjs/text/ascii/module.f.mjs index f1f5d6947..81ae05663 100644 --- a/fjs/text/ascii/module.f.mjs +++ b/fjs/text/ascii/module.f.mjs @@ -1,10 +1,17 @@ /** * Provides ASCII code point constants and helpers for creating numeric code points and inclusive ranges. * + * It also owns the hexadecimal digit codec (`hexDigitValue` / + * `hexDigitCodePoint`), so no consumer has to rederive the `'0'`, `'a' - 10` + * and `'A' - 10` offsets for itself. + * * @module + * + * @import { Nullable } from '../../types/nullable/types.ts' + * @import { Range } from '../../types/range/types.ts' */ -/** @import { Range } from '../../types/range/types.ts' */ +import { contains } from '../../types/range/module.f.mjs' /** @type {(s: string) => (i: number) => number} */ const at = s => i => { @@ -161,6 +168,9 @@ export const latinCapitalLetterE = one('E') /** 0x46 */ export const latinCapitalLetterF = one('F') +/** 0x41..0x46, the uppercase hexadecimal digits. */ +export const latinCapitalLetterAFRange = range('AF') + /** 0x5B */ export const leftSquareBracket = one('[') @@ -196,6 +206,9 @@ export const latinSmallLetterE = one('e') /** 0x66 */ export const latinSmallLetterF = one('f') +/** 0x61..0x66, the lowercase hexadecimal digits. */ +export const latinSmallLetterAFRange = range('af') + /** 0x6E */ export const latinSmallLetterN = one('n') @@ -222,3 +235,38 @@ export const rightCurlyBracket = one('}') /** 0x7E */ export const tilde = one('~') + +// hexadecimal digits + +const isDigit = contains(...digitRange) + +const isLatinSmallLetterAF = contains(...latinSmallLetterAFRange) + +const isLatinCapitalLetterAF = contains(...latinCapitalLetterAFRange) + +/** The distance from a lowercase hexadecimal letter to the value it denotes. */ +const latinSmallLetterAFOffset = latinSmallLetterA - 10 + +/** The distance from an uppercase hexadecimal letter to the value it denotes. */ +const latinCapitalLetterAFOffset = latinCapitalLetterA - 10 + +/** + * The value `0..15` denoted by a hexadecimal digit code point, or `null` when + * the code point is not one of `0-9`, `a-f`, `A-F`. + * + * @type {(codePoint: number) => Nullable} + */ +export const hexDigitValue = codePoint => + isDigit(codePoint) ? codePoint - digit0 + : isLatinSmallLetterAF(codePoint) ? codePoint - latinSmallLetterAFOffset + : isLatinCapitalLetterAF(codePoint) ? codePoint - latinCapitalLetterAFOffset + : null + +/** + * The lowercase hexadecimal digit code point denoting a value in `0..15`. + * `hexDigitValue(hexDigitCodePoint(v))` is `v` for every such value. + * + * @type {(value: number) => number} + */ +export const hexDigitCodePoint = value => + value < 10 ? digit0 + value : latinSmallLetterAFOffset + value diff --git a/fjs/text/ascii/proof.f.mjs b/fjs/text/ascii/proof.f.mjs index 95089b16d..0d0027e27 100644 --- a/fjs/text/ascii/proof.f.mjs +++ b/fjs/text/ascii/proof.f.mjs @@ -1,15 +1,47 @@ -import { one, range } from './module.f.mjs' +import { hexDigitCodePoint, hexDigitValue, one, range } from './module.f.mjs' import { stringify as jsonStringify } from '../../media/json/module.f.mjs' import { sort } from '../../types/object/module.f.mjs' import { assertEq } from '../../asserts/module.f.mjs' const stringify = jsonStringify(sort) +const values = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + export const proof = { range: () => { const r = stringify(range("A")) assertEq(r, '[65,65]') }, + hexDigitValue: { + digit: () => { + assertEq(hexDigitValue(one('0')), 0) + assertEq(hexDigitValue(one('9')), 9) + }, + latinSmallLetterAF: () => { + assertEq(hexDigitValue(one('a')), 10) + assertEq(hexDigitValue(one('f')), 15) + }, + latinCapitalLetterAF: () => { + assertEq(hexDigitValue(one('A')), 10) + assertEq(hexDigitValue(one('F')), 15) + }, + notAHexDigit: () => { + assertEq(hexDigitValue(one('/')), null) + assertEq(hexDigitValue(one(':')), null) + assertEq(hexDigitValue(one('@')), null) + assertEq(hexDigitValue(one('G')), null) + assertEq(hexDigitValue(one('`')), null) + assertEq(hexDigitValue(one('g')), null) + }, + }, + hexDigitCodePoint: { + lowercaseDigits: () => { + assertEq(String.fromCodePoint(...values.map(v => hexDigitCodePoint(v))), '0123456789abcdef') + }, + roundTrip: () => { + assertEq(stringify(values.map(v => hexDigitValue(hexDigitCodePoint(v)))), stringify(values)) + }, + }, throw: { oneThrowsOnEmpty: () => one(''), }, diff --git a/fjs/text/ascii/todo/hex-digit-codec.md b/fjs/text/ascii/todo/hex-digit-codec.md deleted file mode 100644 index 31365b898..000000000 --- a/fjs/text/ascii/todo/hex-digit-codec.md +++ /dev/null @@ -1,46 +0,0 @@ -## Own the hex-digit ↔ value codec - -**Priority:** P3 -**Status:** open - -### Problem - -The mapping between a hex-digit code point and its numeric value — the three -offsets `digit0`, `latinSmallLetterA - 10`, `latinCapitalLetterA - 10` — is -recomputed in three modules, in both directions: - -```js -// fjs/media/json/serializer/module.f.mjs:48-50 (value → char) -const hexDigit = value => - fromCharCode(value < 10 ? digit0 + value : latinSmallLetterA + value - 10) - -// fjs/js/tokenizer/module.f.mjs:599-612 (char → value, range-map dispatch) -const parseUnicodeCharHex = offset => state => input => { ... input - offset ... } - -// fjs/djs/tokenizer/module.f.mjs:360-366 (char → value, ternary chain) -const digit = contains(...digitRange)(cp) ? cp - digit0 - : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10) - : cp - (latinSmallLetterA - 10) -``` - -The `af`/`AF` ranges are likewise built independently in `js/tokenizer:153-154` -and `djs/tokenizer:336`. Note the djs copy's fallthrough: the last branch -assumes lowercase without checking, so a non-hex code point silently yields a -garbage digit, where the js copy has a real reject path. - -### Proposal - -`fjs/text/ascii` already owns `digit0`, `latinSmallLetterA`, -`latinCapitalLetterA` — and `latinCapitalLetterF`/`latinSmallLetterF`, which -exist for no other reason. Add: - -- `hexDigitValue: (cp: number) => Nullable` -- `hexDigitCodePoint: (v: number) => number` -- the shared digit/`af`/`AF` ranges - -and convert the three consumers. - -### Tasks - -- [ ] Add the codec pair and ranges with proof coverage -- [ ] Convert `media/json/serializer`, `js/tokenizer`, `djs/tokenizer`