| /** |
| * @fileoverview Object to handle access and retrieval of tokens. |
| * @author Brandon Mills |
| */ |
| "use strict"; |
| |
| //------------------------------------------------------------------------------ |
| // Requirements |
| //------------------------------------------------------------------------------ |
| |
| const assert = require("assert"); |
| const { isCommentToken } = require("eslint-utils"); |
| const cursors = require("./cursors"); |
| const ForwardTokenCursor = require("./forward-token-cursor"); |
| const PaddedTokenCursor = require("./padded-token-cursor"); |
| const utils = require("./utils"); |
| |
| //------------------------------------------------------------------------------ |
| // Helpers |
| //------------------------------------------------------------------------------ |
| |
| const TOKENS = Symbol("tokens"); |
| const COMMENTS = Symbol("comments"); |
| const INDEX_MAP = Symbol("indexMap"); |
| |
| /** |
| * Creates the map from locations to indices in `tokens`. |
| * |
| * The first/last location of tokens is mapped to the index of the token. |
| * The first/last location of comments is mapped to the index of the next token of each comment. |
| * |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| * @returns {Object} The map from locations to indices in `tokens`. |
| * @private |
| */ |
| function createIndexMap(tokens, comments) { |
| const map = Object.create(null); |
| let tokenIndex = 0; |
| let commentIndex = 0; |
| let nextStart = 0; |
| let range = null; |
| |
| while (tokenIndex < tokens.length || commentIndex < comments.length) { |
| nextStart = (commentIndex < comments.length) ? comments[commentIndex].range[0] : Number.MAX_SAFE_INTEGER; |
| while (tokenIndex < tokens.length && (range = tokens[tokenIndex].range)[0] < nextStart) { |
| map[range[0]] = tokenIndex; |
| map[range[1] - 1] = tokenIndex; |
| tokenIndex += 1; |
| } |
| |
| nextStart = (tokenIndex < tokens.length) ? tokens[tokenIndex].range[0] : Number.MAX_SAFE_INTEGER; |
| while (commentIndex < comments.length && (range = comments[commentIndex].range)[0] < nextStart) { |
| map[range[0]] = tokenIndex; |
| map[range[1] - 1] = tokenIndex; |
| commentIndex += 1; |
| } |
| } |
| |
| return map; |
| } |
| |
| /** |
| * Creates the cursor iterates tokens with options. |
| * |
| * @param {CursorFactory} factory - The cursor factory to initialize cursor. |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| * @param {Object} indexMap - The map from locations to indices in `tokens`. |
| * @param {number} startLoc - The start location of the iteration range. |
| * @param {number} endLoc - The end location of the iteration range. |
| * @param {number|Function|Object} [opts=0] - The option object. If this is a number then it's `opts.skip`. If this is a function then it's `opts.filter`. |
| * @param {boolean} [opts.includeComments=false] - The flag to iterate comments as well. |
| * @param {Function|null} [opts.filter=null] - The predicate function to choose tokens. |
| * @param {number} [opts.skip=0] - The count of tokens the cursor skips. |
| * @returns {Cursor} The created cursor. |
| * @private |
| */ |
| function createCursorWithSkip(factory, tokens, comments, indexMap, startLoc, endLoc, opts) { |
| let includeComments = false; |
| let skip = 0; |
| let filter = null; |
| |
| if (typeof opts === "number") { |
| skip = opts | 0; |
| } else if (typeof opts === "function") { |
| filter = opts; |
| } else if (opts) { |
| includeComments = !!opts.includeComments; |
| skip = opts.skip | 0; |
| filter = opts.filter || null; |
| } |
| assert(skip >= 0, "options.skip should be zero or a positive integer."); |
| assert(!filter || typeof filter === "function", "options.filter should be a function."); |
| |
| return factory.createCursor(tokens, comments, indexMap, startLoc, endLoc, includeComments, filter, skip, -1); |
| } |
| |
| /** |
| * Creates the cursor iterates tokens with options. |
| * |
| * @param {CursorFactory} factory - The cursor factory to initialize cursor. |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| * @param {Object} indexMap - The map from locations to indices in `tokens`. |
| * @param {number} startLoc - The start location of the iteration range. |
| * @param {number} endLoc - The end location of the iteration range. |
| * @param {number|Function|Object} [opts=0] - The option object. If this is a number then it's `opts.count`. If this is a function then it's `opts.filter`. |
| * @param {boolean} [opts.includeComments] - The flag to iterate comments as well. |
| * @param {Function|null} [opts.filter=null] - The predicate function to choose tokens. |
| * @param {number} [opts.count=0] - The maximum count of tokens the cursor iterates. Zero is no iteration for backward compatibility. |
| * @returns {Cursor} The created cursor. |
| * @private |
| */ |
| function createCursorWithCount(factory, tokens, comments, indexMap, startLoc, endLoc, opts) { |
| let includeComments = false; |
| let count = 0; |
| let countExists = false; |
| let filter = null; |
| |
| if (typeof opts === "number") { |
| count = opts | 0; |
| countExists = true; |
| } else if (typeof opts === "function") { |
| filter = opts; |
| } else if (opts) { |
| includeComments = !!opts.includeComments; |
| count = opts.count | 0; |
| countExists = typeof opts.count === "number"; |
| filter = opts.filter || null; |
| } |
| assert(count >= 0, "options.count should be zero or a positive integer."); |
| assert(!filter || typeof filter === "function", "options.filter should be a function."); |
| |
| return factory.createCursor(tokens, comments, indexMap, startLoc, endLoc, includeComments, filter, 0, countExists ? count : -1); |
| } |
| |
| /** |
| * Creates the cursor iterates tokens with options. |
| * This is overload function of the below. |
| * |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| * @param {Object} indexMap - The map from locations to indices in `tokens`. |
| * @param {number} startLoc - The start location of the iteration range. |
| * @param {number} endLoc - The end location of the iteration range. |
| * @param {Function|Object} opts - The option object. If this is a function then it's `opts.filter`. |
| * @param {boolean} [opts.includeComments] - The flag to iterate comments as well. |
| * @param {Function|null} [opts.filter=null] - The predicate function to choose tokens. |
| * @param {number} [opts.count=0] - The maximum count of tokens the cursor iterates. Zero is no iteration for backward compatibility. |
| * @returns {Cursor} The created cursor. |
| * @private |
| */ |
| /** |
| * Creates the cursor iterates tokens with options. |
| * |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| * @param {Object} indexMap - The map from locations to indices in `tokens`. |
| * @param {number} startLoc - The start location of the iteration range. |
| * @param {number} endLoc - The end location of the iteration range. |
| * @param {number} [beforeCount=0] - The number of tokens before the node to retrieve. |
| * @param {boolean} [afterCount=0] - The number of tokens after the node to retrieve. |
| * @returns {Cursor} The created cursor. |
| * @private |
| */ |
| function createCursorWithPadding(tokens, comments, indexMap, startLoc, endLoc, beforeCount, afterCount) { |
| if (typeof beforeCount === "undefined" && typeof afterCount === "undefined") { |
| return new ForwardTokenCursor(tokens, comments, indexMap, startLoc, endLoc); |
| } |
| if (typeof beforeCount === "number" || typeof beforeCount === "undefined") { |
| return new PaddedTokenCursor(tokens, comments, indexMap, startLoc, endLoc, beforeCount | 0, afterCount | 0); |
| } |
| return createCursorWithCount(cursors.forward, tokens, comments, indexMap, startLoc, endLoc, beforeCount); |
| } |
| |
| /** |
| * Gets comment tokens that are adjacent to the current cursor position. |
| * @param {Cursor} cursor - A cursor instance. |
| * @returns {Array} An array of comment tokens adjacent to the current cursor position. |
| * @private |
| */ |
| function getAdjacentCommentTokensFromCursor(cursor) { |
| const tokens = []; |
| let currentToken = cursor.getOneToken(); |
| |
| while (currentToken && isCommentToken(currentToken)) { |
| tokens.push(currentToken); |
| currentToken = cursor.getOneToken(); |
| } |
| |
| return tokens; |
| } |
| |
| //------------------------------------------------------------------------------ |
| // Exports |
| //------------------------------------------------------------------------------ |
| |
| /** |
| * The token store. |
| * |
| * This class provides methods to get tokens by locations as fast as possible. |
| * The methods are a part of public API, so we should be careful if it changes this class. |
| * |
| * People can get tokens in O(1) by the hash map which is mapping from the location of tokens/comments to tokens. |
| * Also people can get a mix of tokens and comments in O(log k), the k is the number of comments. |
| * Assuming that comments to be much fewer than tokens, this does not make hash map from token's locations to comments to reduce memory cost. |
| * This uses binary-searching instead for comments. |
| */ |
| module.exports = class TokenStore { |
| |
| /** |
| * Initializes this token store. |
| * @param {Token[]} tokens - The array of tokens. |
| * @param {Comment[]} comments - The array of comments. |
| */ |
| constructor(tokens, comments) { |
| this[TOKENS] = tokens; |
| this[COMMENTS] = comments; |
| this[INDEX_MAP] = createIndexMap(tokens, comments); |
| } |
| |
| //-------------------------------------------------------------------------- |
| // Gets single token. |
| //-------------------------------------------------------------------------- |
| |
| /** |
| * Gets the token starting at the specified index. |
| * @param {number} offset - Index of the start of the token's range. |
| * @param {Object} [options=0] - The option object. |
| * @param {boolean} [options.includeComments=false] - The flag to iterate comments as well. |
| * @returns {Token|null} The token starting at index, or null if no such token. |
| */ |
| getTokenByRangeStart(offset, options) { |
| const includeComments = options && options.includeComments; |
| const token = cursors.forward.createBaseCursor( |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| offset, |
| -1, |
| includeComments |
| ).getOneToken(); |
| |
| if (token && token.range[0] === offset) { |
| return token; |
| } |
| return null; |
| } |
| |
| /** |
| * Gets the first token of the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {number|Function|Object} [options=0] - The option object. If this is a number then it's `options.skip`. If this is a function then it's `options.filter`. |
| * @param {boolean} [options.includeComments=false] - The flag to iterate comments as well. |
| * @param {Function|null} [options.filter=null] - The predicate function to choose tokens. |
| * @param {number} [options.skip=0] - The count of tokens the cursor skips. |
| * @returns {Token|null} An object representing the token. |
| */ |
| getFirstToken(node, options) { |
| return createCursorWithSkip( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[0], |
| node.range[1], |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the last token of the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstToken() |
| * @returns {Token|null} An object representing the token. |
| */ |
| getLastToken(node, options) { |
| return createCursorWithSkip( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[0], |
| node.range[1], |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the token that precedes a given node or token. |
| * @param {ASTNode|Token|Comment} node - The AST node or token. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstToken() |
| * @returns {Token|null} An object representing the token. |
| */ |
| getTokenBefore(node, options) { |
| return createCursorWithSkip( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| -1, |
| node.range[0], |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the token that follows a given node or token. |
| * @param {ASTNode|Token|Comment} node - The AST node or token. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstToken() |
| * @returns {Token|null} An object representing the token. |
| */ |
| getTokenAfter(node, options) { |
| return createCursorWithSkip( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[1], |
| -1, |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the first token between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left - Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right - Node after the desired token range. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstToken() |
| * @returns {Token|null} An object representing the token. |
| */ |
| getFirstTokenBetween(left, right, options) { |
| return createCursorWithSkip( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| left.range[1], |
| right.range[0], |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the last token between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right Node after the desired token range. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstToken() |
| * @returns {Token|null} An object representing the token. |
| */ |
| getLastTokenBetween(left, right, options) { |
| return createCursorWithSkip( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| left.range[1], |
| right.range[0], |
| options |
| ).getOneToken(); |
| } |
| |
| /** |
| * Gets the token that precedes a given node or token in the token stream. |
| * This is defined for backward compatibility. Use `includeComments` option instead. |
| * TODO: We have a plan to remove this in a future major version. |
| * @param {ASTNode|Token|Comment} node The AST node or token. |
| * @param {number} [skip=0] A number of tokens to skip. |
| * @returns {Token|null} An object representing the token. |
| * @deprecated |
| */ |
| getTokenOrCommentBefore(node, skip) { |
| return this.getTokenBefore(node, { includeComments: true, skip }); |
| } |
| |
| /** |
| * Gets the token that follows a given node or token in the token stream. |
| * This is defined for backward compatibility. Use `includeComments` option instead. |
| * TODO: We have a plan to remove this in a future major version. |
| * @param {ASTNode|Token|Comment} node The AST node or token. |
| * @param {number} [skip=0] A number of tokens to skip. |
| * @returns {Token|null} An object representing the token. |
| * @deprecated |
| */ |
| getTokenOrCommentAfter(node, skip) { |
| return this.getTokenAfter(node, { includeComments: true, skip }); |
| } |
| |
| //-------------------------------------------------------------------------- |
| // Gets multiple tokens. |
| //-------------------------------------------------------------------------- |
| |
| /** |
| * Gets the first `count` tokens of the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {number|Function|Object} [options=0] - The option object. If this is a number then it's `options.count`. If this is a function then it's `options.filter`. |
| * @param {boolean} [options.includeComments=false] - The flag to iterate comments as well. |
| * @param {Function|null} [options.filter=null] - The predicate function to choose tokens. |
| * @param {number} [options.count=0] - The maximum count of tokens the cursor iterates. |
| * @returns {Token[]} Tokens. |
| */ |
| getFirstTokens(node, options) { |
| return createCursorWithCount( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[0], |
| node.range[1], |
| options |
| ).getAllTokens(); |
| } |
| |
| /** |
| * Gets the last `count` tokens of the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstTokens() |
| * @returns {Token[]} Tokens. |
| */ |
| getLastTokens(node, options) { |
| return createCursorWithCount( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[0], |
| node.range[1], |
| options |
| ).getAllTokens().reverse(); |
| } |
| |
| /** |
| * Gets the `count` tokens that precedes a given node or token. |
| * @param {ASTNode|Token|Comment} node - The AST node or token. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstTokens() |
| * @returns {Token[]} Tokens. |
| */ |
| getTokensBefore(node, options) { |
| return createCursorWithCount( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| -1, |
| node.range[0], |
| options |
| ).getAllTokens().reverse(); |
| } |
| |
| /** |
| * Gets the `count` tokens that follows a given node or token. |
| * @param {ASTNode|Token|Comment} node - The AST node or token. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstTokens() |
| * @returns {Token[]} Tokens. |
| */ |
| getTokensAfter(node, options) { |
| return createCursorWithCount( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[1], |
| -1, |
| options |
| ).getAllTokens(); |
| } |
| |
| /** |
| * Gets the first `count` tokens between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left - Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right - Node after the desired token range. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstTokens() |
| * @returns {Token[]} Tokens between left and right. |
| */ |
| getFirstTokensBetween(left, right, options) { |
| return createCursorWithCount( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| left.range[1], |
| right.range[0], |
| options |
| ).getAllTokens(); |
| } |
| |
| /** |
| * Gets the last `count` tokens between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right Node after the desired token range. |
| * @param {number|Function|Object} [options=0] - The option object. Same options as getFirstTokens() |
| * @returns {Token[]} Tokens between left and right. |
| */ |
| getLastTokensBetween(left, right, options) { |
| return createCursorWithCount( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| left.range[1], |
| right.range[0], |
| options |
| ).getAllTokens().reverse(); |
| } |
| |
| /** |
| * Gets all tokens that are related to the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {Function|Object} options The option object. If this is a function then it's `options.filter`. |
| * @param {boolean} [options.includeComments=false] - The flag to iterate comments as well. |
| * @param {Function|null} [options.filter=null] - The predicate function to choose tokens. |
| * @param {number} [options.count=0] - The maximum count of tokens the cursor iterates. |
| * @returns {Token[]} Array of objects representing tokens. |
| */ |
| /** |
| * Gets all tokens that are related to the given node. |
| * @param {ASTNode} node - The AST node. |
| * @param {int} [beforeCount=0] - The number of tokens before the node to retrieve. |
| * @param {int} [afterCount=0] - The number of tokens after the node to retrieve. |
| * @returns {Token[]} Array of objects representing tokens. |
| */ |
| getTokens(node, beforeCount, afterCount) { |
| return createCursorWithPadding( |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| node.range[0], |
| node.range[1], |
| beforeCount, |
| afterCount |
| ).getAllTokens(); |
| } |
| |
| /** |
| * Gets all of the tokens between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right Node after the desired token range. |
| * @param {Function|Object} options The option object. If this is a function then it's `options.filter`. |
| * @param {boolean} [options.includeComments=false] - The flag to iterate comments as well. |
| * @param {Function|null} [options.filter=null] - The predicate function to choose tokens. |
| * @param {number} [options.count=0] - The maximum count of tokens the cursor iterates. |
| * @returns {Token[]} Tokens between left and right. |
| */ |
| /** |
| * Gets all of the tokens between two non-overlapping nodes. |
| * @param {ASTNode|Token|Comment} left Node before the desired token range. |
| * @param {ASTNode|Token|Comment} right Node after the desired token range. |
| * @param {int} [padding=0] Number of extra tokens on either side of center. |
| * @returns {Token[]} Tokens between left and right. |
| */ |
| getTokensBetween(left, right, padding) { |
| return createCursorWithPadding( |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| left.range[1], |
| right.range[0], |
| padding, |
| padding |
| ).getAllTokens(); |
| } |
| |
| //-------------------------------------------------------------------------- |
| // Others. |
| //-------------------------------------------------------------------------- |
| |
| /** |
| * Checks whether any comments exist or not between the given 2 nodes. |
| * |
| * @param {ASTNode} left - The node to check. |
| * @param {ASTNode} right - The node to check. |
| * @returns {boolean} `true` if one or more comments exist. |
| */ |
| commentsExistBetween(left, right) { |
| const index = utils.search(this[COMMENTS], left.range[1]); |
| |
| return ( |
| index < this[COMMENTS].length && |
| this[COMMENTS][index].range[1] <= right.range[0] |
| ); |
| } |
| |
| /** |
| * Gets all comment tokens directly before the given node or token. |
| * @param {ASTNode|token} nodeOrToken The AST node or token to check for adjacent comment tokens. |
| * @returns {Array} An array of comments in occurrence order. |
| */ |
| getCommentsBefore(nodeOrToken) { |
| const cursor = createCursorWithCount( |
| cursors.backward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| -1, |
| nodeOrToken.range[0], |
| { includeComments: true } |
| ); |
| |
| return getAdjacentCommentTokensFromCursor(cursor).reverse(); |
| } |
| |
| /** |
| * Gets all comment tokens directly after the given node or token. |
| * @param {ASTNode|token} nodeOrToken The AST node or token to check for adjacent comment tokens. |
| * @returns {Array} An array of comments in occurrence order. |
| */ |
| getCommentsAfter(nodeOrToken) { |
| const cursor = createCursorWithCount( |
| cursors.forward, |
| this[TOKENS], |
| this[COMMENTS], |
| this[INDEX_MAP], |
| nodeOrToken.range[1], |
| -1, |
| { includeComments: true } |
| ); |
| |
| return getAdjacentCommentTokensFromCursor(cursor); |
| } |
| |
| /** |
| * Gets all comment tokens inside the given node. |
| * @param {ASTNode} node The AST node to get the comments for. |
| * @returns {Array} An array of comments in occurrence order. |
| */ |
| getCommentsInside(node) { |
| return this.getTokens(node, { |
| includeComments: true, |
| filter: isCommentToken |
| }); |
| } |
| }; |
