diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 73e036759..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# Changelog - -The changelog is the [`changelog/`](./changelog/) directory: one file per -released version, plus `changelog/unreleased/` holding one file per pull request -that is not released yet. See [changelog/README.md](./changelog/README.md). diff --git a/changelog/unreleased/1553.md b/changelog/unreleased/1553.md new file mode 100644 index 000000000..d641917b4 --- /dev/null +++ b/changelog/unreleased/1553.md @@ -0,0 +1,5 @@ +- `djs/tokenizer`: `parseDjsMinusState` drops its `case '-'` — `js/tokenizer` + always merges two adjacent `-` characters into one `'--'` token, so + minus-state can never see another `-` — and `metadataAfterTag` drops an + `indexOf` miss check its one call site can never trigger + [#1553](https://github.com/functionalscript/functionalscript/pull/1553) diff --git a/fjs/djs/tokenizer/module.f.mjs b/fjs/djs/tokenizer/module.f.mjs index 031dcac6e..fad3f8669 100644 --- a/fjs/djs/tokenizer/module.f.mjs +++ b/fjs/djs/tokenizer/module.f.mjs @@ -511,10 +511,13 @@ export const tokenizeString = s => { // Finds `tag` in flatTokens and returns the metadata of the next code point after it. // numError/unterminated tags are followed by the poisoning char / EOF-hitting position // in the flattened AST walk order, so this pinpoints roughly where tokenization failed. +// +// The single call site only passes a `tag` it already confirmed via +// `flatTokens.includes(tag)`, so `indexOf` here is never -1. /** @type {(tag: string, flatTokens: readonly _FlatToken[], fallback: TokenMetadata) => TokenMetadata} */ const metadataAfterTag = (tag, flatTokens, fallback) => { const idx = flatTokens.indexOf(tag) - const found = idx < 0 ? undefined : flatTokens.slice(idx + 1).find((/** @type {_FlatToken} */ t) => t instanceof Array) + const found = flatTokens.slice(idx + 1).find((/** @type {_FlatToken} */ t) => t instanceof Array) return found === undefined ? fallback : found[1] } @@ -592,11 +595,16 @@ const parseDjsDefaultState = input => { // Folds a leading '-' into the following number/bigint token, mirroring the old // fjs/djs/tokenizer's minus-state exactly. +// +// No `case '-'` here: the underlying `js/tokenizer` always merges two adjacent +// `-` characters into a single `'--'` token (the decrement operator), so this +// state — entered only after a single, unmerged `-` — can never itself see +// another `'-'`-kind input. Such an input falls through to `default`, which +// handles it exactly like any other non-number/bigint/eof token. /** @type {(input: JsToken) => readonly [List, _DjsScanState]} */ const parseDjsMinusState = input => { switch (input.kind) { case 'eof': return [[{ kind: 'error', message: 'invalid token' }, { kind: 'eof' }], { kind: 'def' }] - case '-': return [[{ kind: 'error', message: 'invalid token' }], { kind: '-' }] case 'bigint': return [[{ kind: 'bigint', value: -1n * input.value }], { kind: 'def' }] case 'number': return [[{ kind: 'number', bf: multiply(input.bf)(-1n), value: `-${input.value}` }], { kind: 'def' }] default: return [{ first: { kind: 'error', message: 'invalid token' }, tail: mapDjsToken(input) }, { kind: 'def' }] diff --git a/fjs/djs/tokenizer/proof.f.mjs b/fjs/djs/tokenizer/proof.f.mjs index 1ab42a4cb..45dc9a6e1 100644 --- a/fjs/djs/tokenizer/proof.f.mjs +++ b/fjs/djs/tokenizer/proof.f.mjs @@ -901,7 +901,10 @@ export const proof = { assertEq(stringify(/** @type {Unknown} */ (result)), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"bigint","value":-1234567890n}},{"metadata":{"column":13,"line":1,"path":""},"token":{"kind":"eof"}}]') }, () => { - // '-' followed by '-': one error, stays in minus-state waiting for what follows + // `js/tokenizer` merges '--' into one decrement-operator token, so + // this never enters (or re-enters) minus-state via a second '-'; + // it's one error straight from the default state's unknown-token + // fallback, mapping the whole '--' token to a single error. const result = toArray(tokenize(stringToList('--'))('')) assertEq(stringify(/** @type {Unknown} */ (result)), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"eof"}}]') }, @@ -914,6 +917,12 @@ export const proof = { const result = toArray(tokenize(stringToList('-'))('')) assertEq(stringify(/** @type {Unknown} */ (result)), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"eof"}}]') }, + () => { + // '-' followed by neither a number/bigint nor eof: one error for + // the dangling '-', then the following token maps through normally. + const result = toArray(tokenize(stringToList('-{'))('')) + assertEq(stringify(/** @type {Unknown} */ (result)), '[{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"error","message":"invalid token"}},{"metadata":{"column":2,"line":1,"path":""},"token":{"kind":"{"}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"eof"}}]') + }, () => { const result = toArray(tokenize(stringToList('[-1234567890n]'))('')) assertEq(stringify(/** @type {Unknown} */ (result)), '[{"metadata":{"column":1,"line":1,"path":""},"token":{"kind":"["}},{"metadata":{"column":3,"line":1,"path":""},"token":{"kind":"bigint","value":-1234567890n}},{"metadata":{"column":14,"line":1,"path":""},"token":{"kind":"]"}},{"metadata":{"column":15,"line":1,"path":""},"token":{"kind":"eof"}}]')