functionalscript · sergey-shandar · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026 · Aug 14, 2026
diff --git a/changelog/unreleased/1564.md b/changelog/unreleased/1564.md
@@ -0,0 +1,3 @@
+- `js/tokenizer`: removes `tokenizeOp`, a wrapper whose `input === null` branch
+  was unreachable — every caller's own type already guarantees a non-null
+  `number` input; call sites now use `tokenizeCharCodeOp` directly
diff --git a/fjs/js/tokenizer/module.f.mjs b/fjs/js/tokenizer/module.f.mjs
@@ -77,7 +77,7 @@ import {
     rightCurlyBracket,
     dollarSign
 }  from '../../text/ascii/module.f.mjs'
-import { todo, assertEq } from '../../asserts/module.f.mjs'
+import { todo, assertEq, assertStructurallySame } from '../../asserts/module.f.mjs'
 
 const { fromCharCode } = String
 
@@ -352,7 +352,7 @@ const initialStateOp = create(
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const invalidNumberToToken = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
 }
 
@@ -361,14 +361,14 @@ const fullStopToToken = state => input => {
     switch (state.numberKind) {
         case '0':
         case 'int': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: '.' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
 /** @type {_CreateToToken<_ParseNumberState>} */
 const digit0ToToken = state => input => {
     switch (state.numberKind) {
-        case '0': return tokenizeOp(input, { kind: 'invalidNumber' })
+        case '0': return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
         case '.':
         case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addFracDigit(input)(state.b), numberKind: 'fractional' }]
         case 'e':
@@ -382,7 +382,7 @@ const digit0ToToken = state => input => {
 /** @type {_CreateToToken<_ParseNumberState>} */
 const digit19ToToken = state => input => {
     switch (state.numberKind) {
-        case '0': return tokenizeOp(input, { kind: 'invalidNumber' })
+        case '0': return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
         case '.':
         case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: addFracDigit(input)(state.b), numberKind: 'fractional' }]
         case 'e':
@@ -399,7 +399,7 @@ const expToToken = state => input => {
         case '0':
         case 'int':
         case 'fractional': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: 'e' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
@@ -415,7 +415,7 @@ const hyphenMinusToToken = state => input => {
 const plusSignToToken = state => input => {
     switch (state.numberKind) {
         case 'e': return [empty, { kind: 'number', value: appendChar(state.value)(input), b: state.b, numberKind: 'e+' }]
-        default: return tokenizeOp(input, { kind: 'invalidNumber' })
+        default: return tokenizeCharCodeOp(input, { kind: 'invalidNumber' })
     }
 }
 
@@ -427,12 +427,12 @@ const terminalToToken = state => input => {
         case 'e+':
         case 'e-':
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
                 return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
             }
         default:
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
                 return [{ first: bufferToNumberToken(state), tail: next[0] }, next[1]]
             }
     }
@@ -448,7 +448,7 @@ const bigintToToken = state => input => {
             }
         default:
             {
-                const next = tokenizeOp(input, { kind: 'initial' })
+                const next = tokenizeCharCodeOp(input, { kind: 'initial' })
                 return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
             }
     }
@@ -471,7 +471,7 @@ const invalidNumberStateOp = create(
     /** @type {_CreateToToken<_InvalidNumberState>} */ (() => () => [empty, { kind: 'invalidNumber' }])
 )([
     rangeSetFunc(rangeSetTerminalForNumber)(/** @type {_CreateToToken<_InvalidNumberState>} */ (() => input => {
-        const next = tokenizeOp(input, { kind: 'initial' })
+        const next = tokenizeCharCodeOp(input, { kind: 'initial' })
         return [{ first: { kind: 'error', message: 'invalid number' }, tail: next[0] }, next[1]]
     }))
 ])
@@ -495,7 +495,7 @@ const parseStringStateOp = create(
 
 /** @type {_CreateToToken<_ParseEscapeCharState>} */
 const parseEscapeDefault = state => input => {
-    const next = tokenizeOp(input, { kind: 'string', value: state.value })
+    const next = tokenizeCharCodeOp(input, { kind: 'string', value: state.value })
     return [{ first: { kind: 'error', message: 'unescaped character' }, tail: next[0] }, next[1]]
 }
 
@@ -512,7 +512,7 @@ const parseEscapeCharStateOp = create(parseEscapeDefault)([
 
 /** @type {_CreateToToken<_ParseUnicodeCharState>} */
 const parseUnicodeCharDefault = state => input => {
-    const next = tokenizeOp(input, { kind: 'string', value: state.value })
+    const next = tokenizeCharCodeOp(input, { kind: 'string', value: state.value })
     return [{ first: { kind: 'error', message: 'invalid hex value' }, tail: next[0] }, next[1]]
 }
 
@@ -538,7 +538,7 @@ const idToToken = s => at(s)(keywordMap) ?? { kind: 'id', value: s }
 /** @type {_CreateToToken<_ParseIdState>} */
 const parseIdDefault = state => input => {
     const keyWordToken = idToToken(state.value)
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: keyWordToken, tail: next[0] }, next[1]]
 }
 
@@ -556,7 +556,7 @@ const parseOperatorStateOp = state => input => {
         default: {
             if (hasOperatorToken(nextStateValue))
                 return [empty, { kind: 'op', value: nextStateValue }]
-            const next = tokenizeOp(input, { kind: 'initial' })
+            const next = tokenizeCharCodeOp(input, { kind: 'initial' })
             return [{ first: getOperatorToken(state.value), tail: next[0] }, next[1]]
         }
     }
@@ -592,7 +592,7 @@ const parseMultilineCommentAsteriskStateOp = create(
 
 /** @type {_CreateToToken<_ParseWhitespaceState>} */
 const parseWhitespaceDefault = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'ws' }, tail: next[0] }, next[1]]
 }
 
@@ -604,7 +604,7 @@ const parseWhitespaceStateOp = create(parseWhitespaceDefault)([
 
 /** @type {_CreateToToken<_ParseNewLineState>} */
 const parseNewLineDefault = () => input => {
-    const next = tokenizeOp(input, { kind: 'initial' })
+    const next = tokenizeCharCodeOp(input, { kind: 'initial' })
     return [{ first: { kind: 'nl' }, tail: next[0] }, next[1]]
 }
 
@@ -666,9 +666,6 @@ const tokenizeEofOp = state => {
     }
 }
 
-/** @type {StateScan<_CharCodeOrEof, _TokenizerState, List<JsToken>>} */
-const tokenizeOp = (input, state) => input === null ? tokenizeEofOp(state) : tokenizeCharCodeOp(input, state)
-
 /** @type {(metadata: TokenMetadata) => (token: JsToken) => JsTokenWithMetadata} */
 const mapTokenWithMetadata = metadata => token => { return { token, metadata } }
 
@@ -702,6 +699,20 @@ export const proof = {
         const result = getOperatorToken('@')
         assertEq(result.kind, 'error')
     },
+    // `tokenize` appends exactly one trailing `null` after its input, so the
+    // scan reaches `{ kind: 'eof' }` only on that final step — nothing ever
+    // runs tokenizeCharCodeOp/tokenizeEofOp again afterward with that state.
+    // Call each directly to cover their otherwise-unreachable `'eof'` arms.
+    tokenizeCharCodeOpAfterEof: () => {
+        const [tokens, state] = tokenizeCharCodeOp('a'.charCodeAt(0), { kind: 'eof' })
+        assertStructurallySame(toArray(tokens), [{ kind: 'error', message: 'eof' }])
+        assertStructurallySame(state, { kind: 'eof' })
+    },
+    tokenizeEofOpAfterEof: () => {
+        const [tokens, state] = tokenizeEofOp({ kind: 'eof' })
+        assertStructurallySame(toArray(tokens), [{ kind: 'error', message: 'eof' }, { kind: 'eof' }])
+        assertStructurallySame(state, { kind: 'eof' })
+    },
     throw: {
         // union throws when two distinct non-default handlers are merged for the same range;
         // this path is unreachable through the public API (no overlapping ranges in practice).

diff --git a/fjs/js/tokenizer/proof.f.mjs b/fjs/js/tokenizer/proof.f.mjs
@@ -174,6 +174,27 @@ export const proof = {
             const result = stringify(tokenizeString('00'))
             if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
         },
+        () => {
+            // A leading zero followed by another digit (not '0' itself, so
+            // digit19ToToken's own '0'-state arm, distinct from '00' above).
+            const result = stringify(tokenizeString('01'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            // 'e' right after a bare '.' (no fractional digits yet).
+            const result = stringify(tokenizeString('1.e5'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
+        () => {
+            // '+' after the exponent's digits have already started. A trailing
+            // digit (not just eof) matters here: if this branch were merged
+            // into the accepting 'e' arm, the '+' would restart the exponent
+            // and the trailing '3' would complete a *valid* number instead —
+            // 'invalid number' alone doesn't distinguish that from eof cutting
+            // the (still invalid) token short.
+            const result = stringify(tokenizeString('1e5+3'))
+            if (result !== '[{"kind":"error","message":"invalid number"},{"kind":"eof"}]') { throw result }
+        },
         () => {
             // terminal char after invalidNumber state (covers invalidNumberStateOp terminal branch)
             const result = stringify(tokenizeString('00,'))