Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions changelog/unreleased/1562.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
- `js/keywords`: new module — the one source of truth for JavaScript keywords
(`reservedWords`, `strictModeReservedWords`, `restrictedNames`, and the
aggregate `keywords`). The JavaScript and DJS tokenizers and the rtti
TypeScript printer derive their sets from it instead of keeping copies:
FunctionalScript is a strict subset of JavaScript, so every consumer must
agree on what a keyword is
[#1562](https://github.com/functionalscript/functionalscript/pull/1562).
14 changes: 4 additions & 10 deletions fjs/djs/tokenizer/module.f.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
unicodeMax,
unicodeRange,
} from '../../bnf/module.f.mjs'
import { keywords } from '../../js/keywords/module.f.mjs'
import { isKeywordToken } from '../../js/tokenizer/module.f.mjs'
import { multiply } from '../../types/bigfloat/module.f.mjs'
import {
Expand Down Expand Up @@ -389,15 +390,8 @@ const decodeNumber = value => {
return [mantissa, exp]
}

const keywords = /** @type {ReadonlySet<string>} */ (new Set([
'true', 'false', 'null', 'undefined',
'arguments', 'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
'debugger', 'default', 'delete', 'do', 'else', 'enum', 'eval', 'export',
'extends', 'finally', 'for', 'function', 'if', 'implements', 'import', 'in',
'instanceof', 'interface', 'let', 'new', 'package', 'private', 'protected',
'public', 'return', 'static', 'super', 'switch', 'this', 'throw', 'try',
'typeof', 'var', 'void', 'while', 'with', 'yield',
]))
/** @type {ReadonlySet<string>} */
const keywordSet = new Set(keywords)

/** @type {(tk: _Token) => JsToken} */
const toJsToken = tk => {
Expand All @@ -413,7 +407,7 @@ const toJsToken = tk => {
return { kind: 'string', value: decodeJsonString(codePoints) }
case 'id': {
const value = codePointListToString(codePoints)
if (keywords.has(value)) return /** @type {JsToken} */ ({ kind: value })
if (keywordSet.has(value)) return /** @type {JsToken} */ ({ kind: value })
return { kind: 'id', value }
}
case 'number': {
Expand Down
72 changes: 72 additions & 0 deletions fjs/js/keywords/module.f.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/**
* The JavaScript keywords — one source of truth.
*
* FunctionalScript is a strict subset of JavaScript: any FunctionalScript
* program must run the same on JavaScript. Every consumer that decides
* whether a name is a keyword — the JavaScript and DJS tokenizers, printers
* that emit identifiers — derives its set from this module instead of
* keeping a copy, so the sets cannot drift apart.
*
* @module
*
* @import { Assert } from '../../asserts/types.ts'
* @import { Equal } from '../../types/ts/types.ts'
*/

/**
* The ECMAScript `ReservedWord` production
* ([ECMA-262 §12.7.2](https://tc39.es/ecma262/#prod-ReservedWord)) — never
* usable as identifiers.
*/
export const reservedWords = /** @type {const} */ ([
'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export',
'extends', 'false', 'finally', 'for', 'function', 'if', 'import', 'in',
'instanceof', 'new', 'null', 'return', 'super', 'switch', 'this',
'throw', 'true', 'try', 'typeof', 'var', 'void', 'while', 'with',
'yield',
])

/**
* Reserved only in strict-mode code — and every module is strict-mode code,
* so FunctionalScript treats them exactly like {@link reservedWords}.
*/
export const strictModeReservedWords = /** @type {const} */ ([
'implements', 'interface', 'let', 'package', 'private', 'protected',
'public', 'static',
])

/**
* Not reserved words, but strict-mode code cannot bind, assign, or shadow
* them.
*/
export const restrictedNames = /** @type {const} */ (['arguments', 'eval'])

/**
* Every name FunctionalScript treats as a keyword, alphabetically: the
* {@link reservedWords}, the {@link strictModeReservedWords}, the
* {@link restrictedNames}, and `undefined` — an ordinary global in
* JavaScript that FunctionalScript keeps as a literal keyword.
*
* The proof verifies this list is exactly the sorted union of the groups,
* and `_KeywordsPinned` ties the two type-level unions together.
*/
export const keywords = /** @type {const} */ ([
'arguments', 'await', 'break', 'case', 'catch', 'class', 'const',
'continue', 'debugger', 'default', 'delete', 'do', 'else', 'enum',
'eval', 'export', 'extends', 'false', 'finally', 'for', 'function',
'if', 'implements', 'import', 'in', 'instanceof', 'interface', 'let',
'new', 'null', 'package', 'private', 'protected', 'public', 'return',
'static', 'super', 'switch', 'this', 'throw', 'true', 'try', 'typeof',
'undefined', 'var', 'void', 'while', 'with', 'yield',
])

/**
* @typedef {Assert<Equal<
* typeof keywords[number],
* | typeof reservedWords[number]
* | typeof strictModeReservedWords[number]
* | typeof restrictedNames[number]
* | 'undefined'
* >>} _KeywordsPinned
*/
13 changes: 13 additions & 0 deletions fjs/js/keywords/proof.f.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { assertEq } from '../../asserts/module.f.mjs'
import { keywords, reservedWords, restrictedNames, strictModeReservedWords } from './module.f.mjs'

export const proof = {
// `keywords` is exactly the sorted union of the groups plus `undefined`
aggregate: () => {
/** @type {readonly string[]} */
const union = [...reservedWords, ...strictModeReservedWords, ...restrictedNames, 'undefined']
// the names are unique, so the comparator never sees an equal pair
assertEq(keywords.join(), union.toSorted((a, b) => a < b ? -1 : 1).join())
assertEq(keywords.length, new Set(keywords).size)
},
}
60 changes: 8 additions & 52 deletions fjs/js/tokenizer/module.f.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import { strictEqual } from '../../types/function/operator/module.f.mjs'
import { merge, fromRange, get } from '../../types/range_map/module.f.mjs'
import { empty, stateScan, flat, toArray, reduce as listReduce, scan, map as listMap } from '../../types/list/module.f.mjs'
import { keywords } from '../keywords/module.f.mjs'
import { at, fromEntries } from '../../types/ordered_map/module.f.mjs'
import { one } from '../../types/range/module.f.mjs'
import {
Expand Down Expand Up @@ -251,60 +252,15 @@ const bufferToNumberToken = ({ numberKind, value, b }) => {
}

/**
* @link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar#keywords
* Derived from the one source of truth for JavaScript keywords,
* `fjs/js/keywords` — FunctionalScript is a strict subset of JavaScript, so
* the tokenizer recognizes exactly that module's `keywords`.
*/
/** @type {List<Entry<JsToken>>} */
const keywordEntries = [
['arguments', { kind: 'arguments' }],
['await', { kind: 'await' }],
['break', { kind: 'break' }],
['case', { kind: 'case' }],
['catch', { kind: 'catch' }],
['class', { kind: 'class' }],
['const', { kind: 'const' }],
['continue', { kind: 'continue' }],
['debugger', { kind: 'debugger' }],
['default', { kind: 'default' }],
['delete', { kind: 'delete' }],
['do', { kind: 'do' }],
['else', { kind: 'else' }],
['enum', { kind: 'enum' }],
['eval', { kind: 'eval' }],
['export', { kind: 'export' }],
['extends', { kind: 'extends' }],
['false', { kind: 'false' }],
['finally', { kind: 'finally' }],
['for', { kind: 'for' }],
['function', { kind: 'function' }],
['if', { kind: 'if' }],
['implements', { kind: 'implements' }],
['import', { kind: 'import' }],
['in', { kind: 'in' }],
['instanceof', { kind: 'instanceof' }],
['interface', { kind: 'interface' }],
['let', { kind: 'let' }],
['new', { kind: 'new' }],
['null', { kind: 'null' }],
['package', { kind: 'package' }],
['private', { kind: 'private' }],
['protected', { kind: 'protected' }],
['public', { kind: 'public' }],
['return', { kind: 'return' }],
['static', { kind: 'static' }],
['super', { kind: 'super' }],
['switch', { kind: 'switch' }],
['this', { kind: 'this' }],
['throw', { kind: 'throw' }],
['true', { kind: 'true' }],
['try', { kind: 'try' }],
['typeof', { kind: 'typeof' }],
['undefined', { kind: 'undefined' }],
['var', { kind: 'var' }],
['void', { kind: 'void' }],
['while', { kind: 'while' }],
['with', { kind: 'with' }],
['yield', { kind: 'yield' }],
]
const keywordEntries = keywords.map(kind =>
// every keyword kind is a `JsToken` kind by construction: `_KeywordToken`
// derives its kinds from this same `keywords` list
[kind, /** @type {JsToken} */ ({ kind })])

const keywordMap = fromEntries(keywordEntries)

Expand Down
19 changes: 11 additions & 8 deletions fjs/js/tokenizer/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import type { RangeMapArray } from '../../types/range_map/types.ts'
import type { List } from '../../types/list/types.ts'
import type { BigFloat } from '../../types/bigfloat/types.ts'
import type { keywords } from '../keywords/module.f.mjs'

export type StringToken = {
readonly kind: 'string'
Expand Down Expand Up @@ -42,14 +43,16 @@ export type _NullToken = {readonly kind: 'null'}
/** @internal */
export type _UndefinedToken = {readonly kind: 'undefined'}

/** @internal */
export type _KeywordToken = |
{ readonly kind: 'arguments' | 'await' | 'break' | 'case' | 'catch' | 'class' | 'const' | 'continue' } |
{ readonly kind: 'debugger' | 'default' | 'delete' | 'do' | 'else' | 'enum' | 'eval' | 'export' } |
{ readonly kind: 'extends' | 'finally' | 'for' | 'function' | 'if' | 'implements' | 'import' | 'in' } |
{ readonly kind: 'instanceof' | 'interface' | 'let' | 'new' | 'package' | 'private' | 'protected' | 'public' } |
{ readonly kind: 'return' | 'static' | 'super' | 'switch' | 'this' | 'throw' | 'try' | 'typeof' } |
{ readonly kind: 'var' | 'void' | 'while' | 'with' | 'yield' }
/**
* A keyword token, its kind drawn from the one source of truth for
* JavaScript keywords, `fjs/js/keywords` — minus the literal keywords
* (`true`/`false`/`null`/`undefined`), which have their own token types.
*
* @internal
*/
export type _KeywordToken = {
readonly kind: Exclude<typeof keywords[number], 'true' | 'false' | 'null' | 'undefined'>
}

export type IdToken = {
readonly kind: 'id'
Expand Down
30 changes: 13 additions & 17 deletions fjs/types/rtti/ts/module.f.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
*/

import { assertNotNullish } from '../../../asserts/module.f.mjs'
import { reservedWords, strictModeReservedWords } from '../../../js/keywords/module.f.mjs'
import { at, definedEntries } from '../../object/module.f.mjs'
import { primitive, union, printer as tsPrinter } from '../../ts/module.f.mjs'
import { cmp, toData, unitBit, unknown as top } from '../data/module.f.mjs'
Expand All @@ -30,27 +31,22 @@ const trueBit = unitBit(true)
const booleanBits = falseBit | trueBit

/**
* Names that cannot name a TypeScript type alias: the predefined type
* names (`TS2457`), the ECMAScript reserved words — those reserved only in
* strict-mode code included, since every module is strict-mode code
* (`TS1214`) — and the type keywords that fail in the alias-name position.
* Names that cannot name a TypeScript type alias: the ECMAScript reserved
* words — from the one source of truth for JavaScript keywords,
* `fjs/js/keywords`, the strict-mode ones included since every module is
* strict-mode code (`TS1214`) — plus TypeScript's predefined type names
* (`TS2457`) and the type keywords that fail in the alias-name position.
*/
const reserved = /** @type {const} */ ([
/** @type {readonly string[]} */
const reserved = [
...reservedWords,
...strictModeReservedWords,
// predefined type names
'any', 'bigint', 'boolean', 'false', 'never', 'null', 'number', 'object',
'string', 'symbol', 'true', 'undefined', 'unknown', 'void',
// ECMAScript reserved words
'await', 'break', 'case', 'catch', 'class', 'const', 'continue',
'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export',
'extends', 'finally', 'for', 'function', 'if', 'import', 'in',
'instanceof', 'new', 'return', 'super', 'switch', 'this', 'throw', 'try',
'typeof', 'var', 'while', 'with',
// reserved in strict-mode code — and every module is strict-mode code
'implements', 'interface', 'let', 'package', 'private', 'protected',
'public', 'static', 'yield',
'any', 'bigint', 'boolean', 'never', 'number', 'object', 'string',
'symbol', 'undefined', 'unknown',
// type-operator keywords, and `intrinsic` (TS2795 outside lib.d.ts)
'infer', 'intrinsic', 'keyof', 'readonly', 'unique',
])
]

/** @type {(c: string) => boolean} */
const isIdStart = c => (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c === '_' || c === '$'
Expand Down
Loading