535c56ccf9b08b9d5b5dc5421d56c9f97812bcf087b3bbf7b24f0e3df49eabfe33723630c0fa4a25f56eeca8be016632f6b6f6b2e7f838166841b69a7a0b3d 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275
  1. /*---------------------------------------------------------------------------------------------
  2. * Copyright (c) Microsoft Corporation. All rights reserved.
  3. * Licensed under the MIT License. See License.txt in the project root for license information.
  4. *--------------------------------------------------------------------------------------------*/
  5. var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
  6. var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
  7. if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
  8. else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
  9. return c > 3 && r && Object.defineProperty(target, key, r), r;
  10. };
  11. var __param = (this && this.__param) || function (paramIndex, decorator) {
  12. return function (target, key) { decorator(target, key, paramIndex); }
  13. };
  14. import { TokenMetadata } from '../encodedTokenAttributes.js';
  15. import { IThemeService } from '../../../platform/theme/common/themeService.js';
  16. import { ILogService, LogLevel } from '../../../platform/log/common/log.js';
  17. import { SparseMultilineTokens } from '../tokens/sparseMultilineTokens.js';
  18. import { ILanguageService } from '../languages/language.js';
  19. let SemanticTokensProviderStyling = class SemanticTokensProviderStyling {
  20. constructor(_legend, _themeService, _languageService, _logService) {
  21. this._legend = _legend;
  22. this._themeService = _themeService;
  23. this._languageService = _languageService;
  24. this._logService = _logService;
  25. this._hasWarnedOverlappingTokens = false;
  26. this._hasWarnedInvalidLengthTokens = false;
  27. this._hasWarnedInvalidEditStart = false;
  28. this._hashTable = new HashTable();
  29. }
  30. getMetadata(tokenTypeIndex, tokenModifierSet, languageId) {
  31. const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
  32. const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, encodedLanguageId);
  33. let metadata;
  34. if (entry) {
  35. metadata = entry.metadata;
  36. if (this._logService.getLevel() === LogLevel.Trace) {
  37. this._logService.trace(`SemanticTokensProviderStyling [CACHED] ${tokenTypeIndex} / ${tokenModifierSet}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
  38. }
  39. }
  40. else {
  41. let tokenType = this._legend.tokenTypes[tokenTypeIndex];
  42. const tokenModifiers = [];
  43. if (tokenType) {
  44. let modifierSet = tokenModifierSet;
  45. for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
  46. if (modifierSet & 1) {
  47. tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
  48. }
  49. modifierSet = modifierSet >> 1;
  50. }
  51. if (modifierSet > 0 && this._logService.getLevel() === LogLevel.Trace) {
  52. this._logService.trace(`SemanticTokensProviderStyling: unknown token modifier index: ${tokenModifierSet.toString(2)} for legend: ${JSON.stringify(this._legend.tokenModifiers)}`);
  53. tokenModifiers.push('not-in-legend');
  54. }
  55. const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId);
  56. if (typeof tokenStyle === 'undefined') {
  57. metadata = 2147483647 /* SemanticTokensProviderStylingConstants.NO_STYLING */;
  58. }
  59. else {
  60. metadata = 0;
  61. if (typeof tokenStyle.italic !== 'undefined') {
  62. const italicBit = (tokenStyle.italic ? 1 /* FontStyle.Italic */ : 0) << 11 /* MetadataConsts.FONT_STYLE_OFFSET */;
  63. metadata |= italicBit | 1 /* MetadataConsts.SEMANTIC_USE_ITALIC */;
  64. }
  65. if (typeof tokenStyle.bold !== 'undefined') {
  66. const boldBit = (tokenStyle.bold ? 2 /* FontStyle.Bold */ : 0) << 11 /* MetadataConsts.FONT_STYLE_OFFSET */;
  67. metadata |= boldBit | 2 /* MetadataConsts.SEMANTIC_USE_BOLD */;
  68. }
  69. if (typeof tokenStyle.underline !== 'undefined') {
  70. const underlineBit = (tokenStyle.underline ? 4 /* FontStyle.Underline */ : 0) << 11 /* MetadataConsts.FONT_STYLE_OFFSET */;
  71. metadata |= underlineBit | 4 /* MetadataConsts.SEMANTIC_USE_UNDERLINE */;
  72. }
  73. if (typeof tokenStyle.strikethrough !== 'undefined') {
  74. const strikethroughBit = (tokenStyle.strikethrough ? 8 /* FontStyle.Strikethrough */ : 0) << 11 /* MetadataConsts.FONT_STYLE_OFFSET */;
  75. metadata |= strikethroughBit | 8 /* MetadataConsts.SEMANTIC_USE_STRIKETHROUGH */;
  76. }
  77. if (tokenStyle.foreground) {
  78. const foregroundBits = (tokenStyle.foreground) << 15 /* MetadataConsts.FOREGROUND_OFFSET */;
  79. metadata |= foregroundBits | 16 /* MetadataConsts.SEMANTIC_USE_FOREGROUND */;
  80. }
  81. if (metadata === 0) {
  82. // Nothing!
  83. metadata = 2147483647 /* SemanticTokensProviderStylingConstants.NO_STYLING */;
  84. }
  85. }
  86. }
  87. else {
  88. if (this._logService.getLevel() === LogLevel.Trace) {
  89. this._logService.trace(`SemanticTokensProviderStyling: unknown token type index: ${tokenTypeIndex} for legend: ${JSON.stringify(this._legend.tokenTypes)}`);
  90. }
  91. metadata = 2147483647 /* SemanticTokensProviderStylingConstants.NO_STYLING */;
  92. tokenType = 'not-in-legend';
  93. }
  94. this._hashTable.add(tokenTypeIndex, tokenModifierSet, encodedLanguageId, metadata);
  95. if (this._logService.getLevel() === LogLevel.Trace) {
  96. this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
  97. }
  98. }
  99. return metadata;
  100. }
  101. warnOverlappingSemanticTokens(lineNumber, startColumn) {
  102. if (!this._hasWarnedOverlappingTokens) {
  103. this._hasWarnedOverlappingTokens = true;
  104. console.warn(`Overlapping semantic tokens detected at lineNumber ${lineNumber}, column ${startColumn}`);
  105. }
  106. }
  107. warnInvalidLengthSemanticTokens(lineNumber, startColumn) {
  108. if (!this._hasWarnedInvalidLengthTokens) {
  109. this._hasWarnedInvalidLengthTokens = true;
  110. console.warn(`Semantic token with invalid length detected at lineNumber ${lineNumber}, column ${startColumn}`);
  111. }
  112. }
  113. warnInvalidEditStart(previousResultId, resultId, editIndex, editStart, maxExpectedStart) {
  114. if (!this._hasWarnedInvalidEditStart) {
  115. this._hasWarnedInvalidEditStart = true;
  116. console.warn(`Invalid semantic tokens edit detected (previousResultId: ${previousResultId}, resultId: ${resultId}) at edit #${editIndex}: The provided start offset ${editStart} is outside the previous data (length ${maxExpectedStart}).`);
  117. }
  118. }
  119. };
  120. SemanticTokensProviderStyling = __decorate([
  121. __param(1, IThemeService),
  122. __param(2, ILanguageService),
  123. __param(3, ILogService)
  124. ], SemanticTokensProviderStyling);
  125. export { SemanticTokensProviderStyling };
  126. export function toMultilineTokens2(tokens, styling, languageId) {
  127. const srcData = tokens.data;
  128. const tokenCount = (tokens.data.length / 5) | 0;
  129. const tokensPerArea = Math.max(Math.ceil(tokenCount / 1024 /* SemanticColoringConstants.DesiredMaxAreas */), 400 /* SemanticColoringConstants.DesiredTokensPerArea */);
  130. const result = [];
  131. let tokenIndex = 0;
  132. let lastLineNumber = 1;
  133. let lastStartCharacter = 0;
  134. while (tokenIndex < tokenCount) {
  135. const tokenStartIndex = tokenIndex;
  136. let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount);
  137. // Keep tokens on the same line in the same area...
  138. if (tokenEndIndex < tokenCount) {
  139. let smallTokenEndIndex = tokenEndIndex;
  140. while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) {
  141. smallTokenEndIndex--;
  142. }
  143. if (smallTokenEndIndex - 1 === tokenStartIndex) {
  144. // there are so many tokens on this line that our area would be empty, we must now go right
  145. let bigTokenEndIndex = tokenEndIndex;
  146. while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) {
  147. bigTokenEndIndex++;
  148. }
  149. tokenEndIndex = bigTokenEndIndex;
  150. }
  151. else {
  152. tokenEndIndex = smallTokenEndIndex;
  153. }
  154. }
  155. let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4);
  156. let destOffset = 0;
  157. let areaLine = 0;
  158. let prevLineNumber = 0;
  159. let prevEndCharacter = 0;
  160. while (tokenIndex < tokenEndIndex) {
  161. const srcOffset = 5 * tokenIndex;
  162. const deltaLine = srcData[srcOffset];
  163. const deltaCharacter = srcData[srcOffset + 1];
  164. // Casting both `lineNumber`, `startCharacter` and `endCharacter` here to uint32 using `|0`
  165. // to validate below with the actual values that will be inserted in the Uint32Array result
  166. const lineNumber = (lastLineNumber + deltaLine) | 0;
  167. const startCharacter = (deltaLine === 0 ? (lastStartCharacter + deltaCharacter) | 0 : deltaCharacter);
  168. const length = srcData[srcOffset + 2];
  169. const endCharacter = (startCharacter + length) | 0;
  170. const tokenTypeIndex = srcData[srcOffset + 3];
  171. const tokenModifierSet = srcData[srcOffset + 4];
  172. if (endCharacter <= startCharacter) {
  173. // this token is invalid (most likely a negative length casted to uint32)
  174. styling.warnInvalidLengthSemanticTokens(lineNumber, startCharacter + 1);
  175. }
  176. else if (prevLineNumber === lineNumber && prevEndCharacter > startCharacter) {
  177. // this token overlaps with the previous token
  178. styling.warnOverlappingSemanticTokens(lineNumber, startCharacter + 1);
  179. }
  180. else {
  181. const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
  182. if (metadata !== 2147483647 /* SemanticTokensProviderStylingConstants.NO_STYLING */) {
  183. if (areaLine === 0) {
  184. areaLine = lineNumber;
  185. }
  186. destData[destOffset] = lineNumber - areaLine;
  187. destData[destOffset + 1] = startCharacter;
  188. destData[destOffset + 2] = endCharacter;
  189. destData[destOffset + 3] = metadata;
  190. destOffset += 4;
  191. prevLineNumber = lineNumber;
  192. prevEndCharacter = endCharacter;
  193. }
  194. }
  195. lastLineNumber = lineNumber;
  196. lastStartCharacter = startCharacter;
  197. tokenIndex++;
  198. }
  199. if (destOffset !== destData.length) {
  200. destData = destData.subarray(0, destOffset);
  201. }
  202. const tokens = SparseMultilineTokens.create(areaLine, destData);
  203. result.push(tokens);
  204. }
  205. return result;
  206. }
  207. class HashTableEntry {
  208. constructor(tokenTypeIndex, tokenModifierSet, languageId, metadata) {
  209. this.tokenTypeIndex = tokenTypeIndex;
  210. this.tokenModifierSet = tokenModifierSet;
  211. this.languageId = languageId;
  212. this.metadata = metadata;
  213. this.next = null;
  214. }
  215. }
  216. class HashTable {
  217. constructor() {
  218. this._elementsCount = 0;
  219. this._currentLengthIndex = 0;
  220. this._currentLength = HashTable._SIZES[this._currentLengthIndex];
  221. this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
  222. this._elements = [];
  223. HashTable._nullOutEntries(this._elements, this._currentLength);
  224. }
  225. static _nullOutEntries(entries, length) {
  226. for (let i = 0; i < length; i++) {
  227. entries[i] = null;
  228. }
  229. }
  230. _hash2(n1, n2) {
  231. return (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
  232. }
  233. _hashFunc(tokenTypeIndex, tokenModifierSet, languageId) {
  234. return this._hash2(this._hash2(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
  235. }
  236. get(tokenTypeIndex, tokenModifierSet, languageId) {
  237. const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
  238. let p = this._elements[hash];
  239. while (p) {
  240. if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
  241. return p;
  242. }
  243. p = p.next;
  244. }
  245. return null;
  246. }
  247. add(tokenTypeIndex, tokenModifierSet, languageId, metadata) {
  248. this._elementsCount++;
  249. if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
  250. // expand!
  251. const oldElements = this._elements;
  252. this._currentLengthIndex++;
  253. this._currentLength = HashTable._SIZES[this._currentLengthIndex];
  254. this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0);
  255. this._elements = [];
  256. HashTable._nullOutEntries(this._elements, this._currentLength);
  257. for (const first of oldElements) {
  258. let p = first;
  259. while (p) {
  260. const oldNext = p.next;
  261. p.next = null;
  262. this._add(p);
  263. p = oldNext;
  264. }
  265. }
  266. }
  267. this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
  268. }
  269. _add(element) {
  270. const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
  271. element.next = this._elements[hash];
  272. this._elements[hash] = element;
  273. }
  274. }
  275. HashTable._SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143];