functionalscript · sergey-shandar · Aug 13, 2026 · Aug 13, 2026 · Aug 13, 2026 · Aug 13, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,6 +20,11 @@ history.
 
 ## Unreleased
 
+- `fjs/text/ascii` owns the hex-digit codec: `hexDigitValue`,
+  `hexDigitCodePoint`, and the `a-f` / `A-F` ranges. The JSON serializer and
+  both tokenizers use it instead of rederiving the offsets; the DJS tokenizer
+  no longer decodes a non-hex code point to a garbage digit
+  [#1522](https://github.com/functionalscript/functionalscript/pull/1522)
 - `fjs/types/result` exports `okThen`, the pure `Result` bind — `mapOk`'s
   monad sibling and the pure twin of `fjs/effects`' `okStep`. Its two error
   types are unioned, so a chain that widens its error needs no rewrapping

diff --git a/fjs/djs/tokenizer/module.f.mjs b/fjs/djs/tokenizer/module.f.mjs
@@ -52,14 +52,12 @@ import { multiply } from '../../types/bigfloat/module.f.mjs'
 import {
     asterisk, backspace, ht, lf, ff, cr,
     quotationMark, solidus, reverseSolidus,
-    digitRange, digit0,
-    latinCapitalLetterA,
-    latinSmallLetterA, latinSmallLetterB, latinSmallLetterF,
+    hexDigitValue,
+    latinSmallLetterB, latinSmallLetterF,
     latinSmallLetterN, latinSmallLetterR, latinSmallLetterT, latinSmallLetterU,
-    range as asciiRange,
 } from '../../text/ascii/module.f.mjs'
 import { codePointListToString, stringToCodePointList } from '../../text/utf16/module.f.mjs'
-import { contains } from '../../types/range/module.f.mjs'
+import { mapUnwrap } from '../../types/nullable/module.f.mjs'
 import { concat, empty, filter, flat, flatMap, fold, map, stateScan, toArray } from '../../types/list/module.f.mjs'
 import { stringifyAsTree } from '../serializer/module.f.mjs'
 import { sort } from '../../types/object/module.f.mjs'
@@ -333,7 +331,13 @@ const filterFunc = tk => {
     }
 }
 
-const rangeCapitalAF = asciiRange('AF')
+/**
+ * A `\uXXXX` escape reaches the `unicode` state only after the grammar has
+ * accepted its four hex digits, so a non-hex code point here is a tokenizer
+ * bug rather than bad input — assert instead of decoding it to a garbage
+ * value, which is what the hand-rolled ternary chain used to do.
+ */
+const unwrapHexDigitValue = mapUnwrap(hexDigitValue)
 
 /** @typedef {
  *   | { readonly kind: 'normal' }
@@ -358,11 +362,7 @@ const stringDecodeScan = (cp, state) => {
                 default:  return [[cp], { kind: 'normal' }]
             }
         case 'unicode': {
-            // convert hex digit char to its numeric value: '0'-'9', 'A'-'F', 'a'-'f'
-            const digit = contains(...digitRange)(cp) ? cp - digit0
-                : contains(...rangeCapitalAF)(cp) ? cp - (latinCapitalLetterA - 10)
-                : cp - (latinSmallLetterA - 10)
-            const acc = (state.acc << 4) | digit
+            const acc = (state.acc << 4) | unwrapHexDigitValue(cp)
             return state.count === 3 ? [[acc], { kind: 'normal' }] : [null, { kind: 'unicode', acc, count: state.count + 1 }]
         }
         default:

diff --git a/fjs/js/tokenizer/module.f.mjs b/fjs/js/tokenizer/module.f.mjs
@@ -51,8 +51,9 @@ import {
     digit0,
     colon,
     //
+    hexDigitValue,
+    //
     latinCapitalLetterRange,
-    latinCapitalLetterA,
     latinCapitalLetterE,
     //
     leftSquareBracket,
@@ -61,7 +62,6 @@ import {
     lowLine,
     //
     latinSmallLetterRange,
-    latinSmallLetterA,
     latinSmallLetterB,
     latinSmallLetterE,
     latinSmallLetterF,
@@ -150,9 +150,6 @@ const rangeSetTerminalForNumber = [
     one(tilde),
 ]
 
-const rangeSmallAF = range('af')
-const rangeCapitalAF = range('AF')
-
 const rangeIdStart = [
     latinSmallLetterRange,
     latinCapitalLetterRange,
@@ -596,22 +593,22 @@ const parseUnicodeCharDefault = state => input => {
     return [{ first: { kind: 'error', message: 'invalid hex value' }, tail: next[0] }, next[1]]
 }
 
-/** @type {(offset: number) => _CreateToToken<_ParseUnicodeCharState>} */
-const parseUnicodeCharHex = offset => state => input => {
-    const hexValue = input - offset
+/**
+ * `hexDigitValue` classifies the code point and decodes it in one step, so this
+ * state needs no range-map dispatch: a `null` value is exactly the non-hex
+ * input the default handler rejects.
+ *
+ * @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List<JsToken>, _TokenizerState]}
+ */
+const parseUnicodeCharStateOp = state => input => {
+    const hexValue = hexDigitValue(input)
+    if (hexValue === null) { return parseUnicodeCharDefault(state)(input) }
     const newUnicode = state.unicode | (hexValue << (3 - state.hexIndex) * 4)
     return [empty, state.hexIndex === 3 ?
         { kind: 'string', value: appendChar(state.value)(newUnicode) } :
         { kind: 'unicodeChar', value: state.value, unicode: newUnicode, hexIndex: state.hexIndex + 1 }]
 }
 
-/** @type {(state: _ParseUnicodeCharState) => (input: number) => readonly [List<JsToken>, _TokenizerState]} */
-const parseUnicodeCharStateOp = create(parseUnicodeCharDefault)([
-    rangeFunc(digitRange)(parseUnicodeCharHex(digit0)),
-    rangeFunc(rangeSmallAF)(parseUnicodeCharHex(latinSmallLetterA - 10)),
-    rangeFunc(rangeCapitalAF)(parseUnicodeCharHex(latinCapitalLetterA - 10))
-])
-
 /** @type {(s: string) => JsToken} */
 const idToToken = s => at(s)(keywordMap) ?? { kind: 'id', value: s }
 

diff --git a/fjs/media/json/serializer/module.f.mjs b/fjs/media/json/serializer/module.f.mjs
@@ -17,10 +17,9 @@ import { errorMask } from '../../../text/code_point/module.f.mjs'
 import {
     backspace,
     cr,
-    digit0,
     ff,
+    hexDigitCodePoint,
     ht,
-    latinSmallLetterA,
     lf,
     quotationMark,
     reverseSolidus,
@@ -46,8 +45,7 @@ const escapeTable = /** @type {const} */ ({
 })
 
 /** @type {(value: number) => string} */
-const hexDigit = value =>
-    fromCharCode(value < 10 ? digit0 + value : latinSmallLetterA + value - 10)
+const hexDigit = value => fromCharCode(hexDigitCodePoint(value))
 
 /**
  * `\uXXXX` with lowercase hex digits, matching ECMAScript's `UnicodeEscape`.

diff --git a/fjs/text/ascii/module.f.mjs b/fjs/text/ascii/module.f.mjs
@@ -1,10 +1,17 @@
 /**
  * Provides ASCII code point constants and helpers for creating numeric code points and inclusive ranges.
  *
+ * It also owns the hexadecimal digit codec (`hexDigitValue` /
+ * `hexDigitCodePoint`), so no consumer has to rederive the `'0'`, `'a' - 10`
+ * and `'A' - 10` offsets for itself.
+ *
  * @module
+ *
+ * @import { Nullable } from '../../types/nullable/types.ts'
+ * @import { Range } from '../../types/range/types.ts'
  */
 
-/** @import { Range } from '../../types/range/types.ts' */
+import { contains } from '../../types/range/module.f.mjs'
 
 /** @type {(s: string) => (i: number) => number} */
 const at = s => i => {
@@ -161,6 +168,9 @@ export const latinCapitalLetterE = one('E')
 /** 0x46 */
 export const latinCapitalLetterF = one('F')
 
+/** 0x41..0x46, the uppercase hexadecimal digits. */
+export const latinCapitalLetterAFRange = range('AF')
+
 /** 0x5B */
 export const leftSquareBracket = one('[')
 
@@ -196,6 +206,9 @@ export const latinSmallLetterE = one('e')
 /** 0x66 */
 export const latinSmallLetterF = one('f')
 
+/** 0x61..0x66, the lowercase hexadecimal digits. */
+export const latinSmallLetterAFRange = range('af')
+
 /** 0x6E */
 export const latinSmallLetterN = one('n')
 
@@ -222,3 +235,38 @@ export const rightCurlyBracket = one('}')
 
 /** 0x7E */
 export const tilde = one('~')
+
+// hexadecimal digits
+
+const isDigit = contains(...digitRange)
+
+const isLatinSmallLetterAF = contains(...latinSmallLetterAFRange)
+
+const isLatinCapitalLetterAF = contains(...latinCapitalLetterAFRange)
+
+/** The distance from a lowercase hexadecimal letter to the value it denotes. */
+const latinSmallLetterAFOffset = latinSmallLetterA - 10
+
+/** The distance from an uppercase hexadecimal letter to the value it denotes. */
+const latinCapitalLetterAFOffset = latinCapitalLetterA - 10
+
+/**
+ * The value `0..15` denoted by a hexadecimal digit code point, or `null` when
+ * the code point is not one of `0-9`, `a-f`, `A-F`.
+ *
+ * @type {(codePoint: number) => Nullable<number>}
+ */
+export const hexDigitValue = codePoint =>
+    isDigit(codePoint) ? codePoint - digit0
+        : isLatinSmallLetterAF(codePoint) ? codePoint - latinSmallLetterAFOffset
+            : isLatinCapitalLetterAF(codePoint) ? codePoint - latinCapitalLetterAFOffset
+                : null
+
+/**
+ * The lowercase hexadecimal digit code point denoting a value in `0..15`.
+ * `hexDigitValue(hexDigitCodePoint(v))` is `v` for every such value.
+ *
+ * @type {(value: number) => number}
+ */
+export const hexDigitCodePoint = value =>
+    value < 10 ? digit0 + value : latinSmallLetterAFOffset + value
diff --git a/fjs/text/ascii/proof.f.mjs b/fjs/text/ascii/proof.f.mjs
@@ -1,15 +1,47 @@
-import { one, range } from './module.f.mjs'
+import { hexDigitCodePoint, hexDigitValue, one, range } from './module.f.mjs'
 import { stringify as jsonStringify } from '../../media/json/module.f.mjs'
 import { sort } from '../../types/object/module.f.mjs'
 import { assertEq } from '../../asserts/module.f.mjs'
 
 const stringify = jsonStringify(sort)
 
+const values = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+
 export const proof = {
     range: () => {
         const r = stringify(range("A"))
         assertEq(r, '[65,65]')
     },
+    hexDigitValue: {
+        digit: () => {
+            assertEq(hexDigitValue(one('0')), 0)
+            assertEq(hexDigitValue(one('9')), 9)
+        },
+        latinSmallLetterAF: () => {
+            assertEq(hexDigitValue(one('a')), 10)
+            assertEq(hexDigitValue(one('f')), 15)
+        },
+        latinCapitalLetterAF: () => {
+            assertEq(hexDigitValue(one('A')), 10)
+            assertEq(hexDigitValue(one('F')), 15)
+        },
+        notAHexDigit: () => {
+            assertEq(hexDigitValue(one('/')), null)
+            assertEq(hexDigitValue(one(':')), null)
+            assertEq(hexDigitValue(one('@')), null)
+            assertEq(hexDigitValue(one('G')), null)
+            assertEq(hexDigitValue(one('`')), null)
+            assertEq(hexDigitValue(one('g')), null)
+        },
+    },
+    hexDigitCodePoint: {
+        lowercaseDigits: () => {
+            assertEq(String.fromCodePoint(...values.map(v => hexDigitCodePoint(v))), '0123456789abcdef')
+        },
+        roundTrip: () => {
+            assertEq(stringify(values.map(v => hexDigitValue(hexDigitCodePoint(v)))), stringify(values))
+        },
+    },
     throw: {
         oneThrowsOnEmpty: () => one(''),
     },

diff --git a/fjs/text/ascii/todo/hex-digit-codec.md b/fjs/text/ascii/todo/hex-digit-codec.md