1 module dlangide.tools.d.dsyntax; 2 3 import dlangui.core.logger; 4 import dlangui.widgets.editors; 5 import dlangui.widgets.srcedit; 6 7 import ddc.lexer.textsource; 8 import ddc.lexer.exceptions; 9 import ddc.lexer.tokenizer; 10 11 class SimpleDSyntaxSupport : SyntaxSupport { 12 13 EditableContent _content; 14 SourceFile _file; 15 ArraySourceLines _lines; 16 Tokenizer _tokenizer; 17 this (string filename) { 18 _file = new SourceFile(filename); 19 _lines = new ArraySourceLines(); 20 _tokenizer = new Tokenizer(_lines); 21 _tokenizer.errorTolerant = true; 22 } 23 24 TokenPropString[] _props; 25 26 /// returns editable content 27 @property EditableContent content() { return _content; } 28 /// set editable content 29 @property SyntaxSupport content(EditableContent content) { 30 _content = content; 31 return this; 32 } 33 34 private enum BracketMatch { 35 CONTINUE, 36 FOUND, 37 ERROR 38 } 39 private static struct BracketStack { 40 dchar[] buf; 41 int pos; 42 bool reverse; 43 void init(bool reverse) { 44 this.reverse = reverse; 45 pos = 0; 46 } 47 void push(dchar ch) { 48 if (buf.length <= pos) 49 buf.length = pos + 16; 50 buf[pos++] = ch; 51 } 52 dchar pop() { 53 if (pos <= 0) 54 return 0; 55 return buf[--pos]; 56 } 57 BracketMatch process(dchar ch) { 58 if (reverse) { 59 if (isCloseBracket(ch)) { 60 push(ch); 61 return BracketMatch.CONTINUE; 62 } else { 63 if (pop() != pairedBracket(ch)) 64 return BracketMatch.ERROR; 65 if (pos == 0) 66 return BracketMatch.FOUND; 67 return BracketMatch.CONTINUE; 68 } 69 } else { 70 if (isOpenBracket(ch)) { 71 push(ch); 72 return BracketMatch.CONTINUE; 73 } else { 74 if (pop() != pairedBracket(ch)) 75 return BracketMatch.ERROR; 76 if (pos == 0) 77 return BracketMatch.FOUND; 78 return BracketMatch.CONTINUE; 79 } 80 } 81 } 82 } 83 BracketStack _bracketStack; 84 static bool isBracket(dchar ch) { 85 return pairedBracket(ch) != 0; 86 } 87 static dchar pairedBracket(dchar ch) { 88 switch (ch) { 89 case '(': 90 return ')'; 91 case ')': 92 return '('; 93 case '{': 94 return '}'; 95 case '}': 96 return '{'; 97 case '[': 98 return ']'; 99 case ']': 100 return '['; 101 default: 102 return 0; // not a bracket 103 } 104 } 105 static bool isOpenBracket(dchar ch) { 106 switch (ch) { 107 case '(': 108 case '{': 109 case '[': 110 return true; 111 default: 112 return false; 113 } 114 } 115 static bool isCloseBracket(dchar ch) { 116 switch (ch) { 117 case ')': 118 case '}': 119 case ']': 120 return true; 121 default: 122 return false; 123 } 124 } 125 126 protected dchar nextBracket(int dir, ref TextPosition p) { 127 for (;;) { 128 TextPosition oldpos = p; 129 p = dir < 0 ? _content.prevCharPos(p) : _content.nextCharPos(p); 130 if (p == oldpos) 131 return 0; 132 auto prop = _content.tokenProp(p); 133 if (tokenCategory(prop) == TokenCategory.Op) { 134 dchar ch = _content[p]; 135 if (isBracket(ch)) 136 return ch; 137 } 138 } 139 } 140 141 /// returns paired bracket {} () [] for char at position p, returns paired char position or p if not found or not bracket 142 override TextPosition findPairedBracket(TextPosition p) { 143 if (p.line < 0 || p.line >= content.length) 144 return p; 145 dstring s = content.line(p.line); 146 if (p.pos < 0 || p.pos >= s.length) 147 return p; 148 dchar ch = content[p]; 149 dchar paired = pairedBracket(ch); 150 if (!paired) 151 return p; 152 TextPosition startPos = p; 153 int dir = isOpenBracket(ch) ? 1 : -1; 154 _bracketStack.init(dir < 0); 155 _bracketStack.process(ch); 156 for (;;) { 157 ch = nextBracket(dir, p); 158 if (!ch) // no more brackets 159 return startPos; 160 auto match = _bracketStack.process(ch); 161 if (match == BracketMatch.FOUND) 162 return p; 163 if (match == BracketMatch.ERROR) 164 return startPos; 165 // continue 166 } 167 } 168 169 170 /// return true if toggle line comment is supported for file type 171 override @property bool supportsToggleLineComment() { 172 return true; 173 } 174 175 /// return true if can toggle line comments for specified text range 176 override bool canToggleLineComment(TextRange range) { 177 TextRange r = content.fullLinesRange(range); 178 if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end)) 179 return false; 180 return true; 181 } 182 183 protected bool isLineComment(dstring s) { 184 for (int i = 0; i < cast(int)s.length - 1; i++) { 185 if (s[i] == '/' && s[i + 1] == '/') 186 return true; 187 else if (s[i] != ' ' && s[i] != '\t') 188 return false; 189 } 190 return false; 191 } 192 193 protected dstring commentLine(dstring s, int commentX) { 194 dchar[] res; 195 int x = 0; 196 bool commented = false; 197 for (int i = 0; i < s.length; i++) { 198 dchar ch = s[i]; 199 if (ch == '\t') { 200 int newX = (x + _content.tabSize) / _content.tabSize * _content.tabSize; 201 if (!commented && newX >= commentX) { 202 commented = true; 203 if (newX != commentX) { 204 // replace tab with space 205 for (; x <= commentX; x++) 206 res ~= ' '; 207 } else { 208 res ~= ch; 209 x = newX; 210 } 211 res ~= "//"d; 212 x += 2; 213 } else { 214 res ~= ch; 215 x = newX; 216 } 217 } else { 218 if (!commented && x == commentX) { 219 commented = true; 220 res ~= "//"d; 221 res ~= ch; 222 x += 3; 223 } else { 224 res ~= ch; 225 x++; 226 } 227 } 228 } 229 if (!commented) { 230 for (; x < commentX; x++) 231 res ~= ' '; 232 res ~= "//"d; 233 } 234 return cast(dstring)res; 235 } 236 237 /// remove single line comment from beginning of line 238 protected dstring uncommentLine(dstring s) { 239 int p = -1; 240 for (int i = 0; i < cast(int)s.length - 1; i++) { 241 if (s[i] == '/' && s[i + 1] == '/') { 242 p = i; 243 break; 244 } 245 } 246 if (p < 0) 247 return s; 248 s = s[0..p] ~ s[p + 2 .. $]; 249 for (int i = 0; i < s.length; i++) { 250 if (s[i] != ' ' && s[i] != '\t') { 251 return s; 252 } 253 } 254 return null; 255 } 256 257 /// searches for neares token start before or equal to position 258 protected TextPosition tokenStart(TextPosition pos) { 259 TextPosition p = pos; 260 for (;;) { 261 TextPosition prevPos = content.prevCharPos(p); 262 if (p == prevPos) 263 return p; // begin of file 264 TokenProp prop = content.tokenProp(p); 265 TokenProp prevProp = content.tokenProp(prevPos); 266 if (prop && prop != prevProp) 267 return p; 268 p = prevPos; 269 } 270 } 271 272 static struct TokenWithRange { 273 Token token; 274 TextRange range; 275 @property string toString() { 276 return token.toString ~ range.toString; 277 } 278 } 279 protected TextPosition _lastTokenStart; 280 protected Token _lastToken; 281 protected bool initTokenizer(TextPosition startPos) { 282 const dstring[] lines = content.lines; 283 _lines.initialize(cast(dstring[])(lines[startPos.line .. $]), _file, startPos.line); 284 _tokenizer.initialize(_lines, startPos.pos); 285 _lastTokenStart = startPos; 286 _lastToken = null; 287 nextToken(); 288 return true; 289 } 290 291 protected TokenWithRange nextToken() { 292 TokenWithRange res; 293 if (_lastToken && _lastToken.type == TokenType.EOF) { 294 // end of file 295 res.range.start = _lastTokenStart; 296 res.range.end = content.endOfFile(); 297 res.token = null; 298 return res; 299 } 300 res.range.start = _lastTokenStart; 301 res.token = _lastToken; 302 _lastToken = _tokenizer.nextToken(); 303 if (_lastToken) 304 _lastToken = _lastToken.clone(); 305 _lastTokenStart = _lastToken ? TextPosition(_lastToken.line - 1, _lastToken.pos - 1) : content.endOfFile(); 306 res.range.end = _lastTokenStart; 307 return res; 308 } 309 310 protected TokenWithRange getPositionToken(TextPosition pos) { 311 //Log.d("getPositionToken for ", pos); 312 TextPosition start = tokenStart(pos); 313 //Log.d("token start found: ", start); 314 initTokenizer(start); 315 for (;;) { 316 TokenWithRange tokenRange = nextToken(); 317 //Log.d("read token: ", tokenRange); 318 if (!tokenRange.token) { 319 //Log.d("end of file"); 320 return tokenRange; 321 } 322 if (pos >= tokenRange.range.start && pos < tokenRange.range.end) { 323 //Log.d("found: ", pos, " in ", tokenRange); 324 return tokenRange; 325 } 326 } 327 } 328 329 protected TokenWithRange[] getRangeTokens(TextRange range) { 330 TokenWithRange[] res; 331 //Log.d("getPositionToken for ", pos); 332 TextPosition start = tokenStart(range.start); 333 //Log.d("token start found: ", start); 334 initTokenizer(start); 335 for (;;) { 336 TokenWithRange tokenRange = nextToken(); 337 //Log.d("read token: ", tokenRange); 338 if (!tokenRange.token) { 339 //Log.d("end of file"); 340 return res; 341 } 342 if (tokenRange.range.intersects(range)) { 343 //Log.d("found: ", pos, " in ", tokenRange); 344 res ~= tokenRange; 345 } 346 } 347 } 348 349 protected bool isInsideBlockComment(TextPosition pos) { 350 TokenWithRange tokenRange = getPositionToken(pos); 351 if (tokenRange.token && tokenRange.token.type == TokenType.COMMENT && tokenRange.token.isMultilineComment) 352 return pos > tokenRange.range.start && pos < tokenRange.range.end; 353 return false; 354 } 355 356 /// toggle line comments for specified text range 357 override void toggleLineComment(TextRange range, Object source) { 358 TextRange r = content.fullLinesRange(range); 359 if (isInsideBlockComment(r.start) || isInsideBlockComment(r.end)) 360 return; 361 int lineCount = r.end.line - r.start.line; 362 bool noEolAtEndOfRange = false; 363 if (lineCount == 0 || r.end.pos > 0) { 364 noEolAtEndOfRange = true; 365 lineCount++; 366 } 367 int minLeftX = -1; 368 bool hasComments = false; 369 bool hasNoComments = false; 370 bool hasNonEmpty = false; 371 dstring[] srctext; 372 dstring[] dsttext; 373 for (int i = 0; i < lineCount; i++) { 374 int lineIndex = r.start.line + i; 375 dstring s = content.line(lineIndex); 376 srctext ~= s; 377 TextLineMeasure m = content.measureLine(lineIndex); 378 if (!m.empty) { 379 if (minLeftX < 0 || minLeftX > m.firstNonSpaceX) 380 minLeftX = m.firstNonSpaceX; 381 hasNonEmpty = true; 382 if (isLineComment(s)) 383 hasComments = true; 384 else 385 hasNoComments = true; 386 } 387 } 388 if (minLeftX < 0) 389 minLeftX = 0; 390 if (hasNoComments || !hasComments) { 391 // comment 392 for (int i = 0; i < lineCount; i++) { 393 dsttext ~= commentLine(srctext[i], minLeftX); 394 } 395 if (!noEolAtEndOfRange) 396 dsttext ~= ""d; 397 EditOperation op = new EditOperation(EditAction.Replace, r, dsttext); 398 _content.performOperation(op, source); 399 } else { 400 // uncomment 401 for (int i = 0; i < lineCount; i++) { 402 dsttext ~= uncommentLine(srctext[i]); 403 } 404 if (!noEolAtEndOfRange) 405 dsttext ~= ""d; 406 EditOperation op = new EditOperation(EditAction.Replace, r, dsttext); 407 _content.performOperation(op, source); 408 } 409 } 410 411 /// return true if toggle block comment is supported for file type 412 override @property bool supportsToggleBlockComment() { 413 return true; 414 } 415 /// return true if can toggle block comments for specified text range 416 override bool canToggleBlockComment(TextRange range) { 417 TokenWithRange startToken = getPositionToken(range.start); 418 TokenWithRange endToken = getPositionToken(range.end); 419 //Log.d("canToggleBlockComment: startToken=", startToken, " endToken=", endToken); 420 if (startToken.token && endToken.token && startToken.range == endToken.range && startToken.token.isMultilineComment) { 421 //Log.d("canToggleBlockComment: can uncomment"); 422 return true; 423 } 424 if (range.empty) 425 return false; 426 TokenWithRange[] tokens = getRangeTokens(range); 427 foreach(ref t; tokens) { 428 if (t.token.type == TokenType.COMMENT) { 429 if (t.token.isMultilineComment) { 430 // disable until nested comments support is implemented 431 return false; 432 } else { 433 // single line comment 434 if (t.range.isInside(range.start) || t.range.isInside(range.end)) 435 return false; 436 } 437 } 438 } 439 return true; 440 } 441 /// toggle block comments for specified text range 442 override void toggleBlockComment(TextRange srcrange, Object source) { 443 TokenWithRange startToken = getPositionToken(srcrange.start); 444 TokenWithRange endToken = getPositionToken(srcrange.end); 445 if (startToken.token && endToken.token && startToken.range == endToken.range && startToken.token.isMultilineComment) { 446 TextRange range = startToken.range; 447 dstring[] dsttext; 448 for (int i = range.start.line; i <= range.end.line; i++) { 449 dstring s = content.line(i); 450 int charsRemoved = 0; 451 int minp = 0; 452 if (i == range.start.line) { 453 int maxp = content.lineLength(range.start.line); 454 if (i == range.end.line) 455 maxp = range.end.pos - 2; 456 charsRemoved = 2; 457 for (int j = range.start.pos + charsRemoved; j < maxp; j++) { 458 if (s[j] != s[j - 1]) 459 break; 460 charsRemoved++; 461 } 462 //Log.d("line before removing start of comment:", s); 463 s = s[range.start.pos + charsRemoved .. $]; 464 //Log.d("line after removing start of comment:", s); 465 charsRemoved += range.start.pos; 466 } 467 if (i == range.end.line) { 468 int endp = range.end.pos; 469 if (charsRemoved > 0) 470 endp -= charsRemoved; 471 int endRemoved = 2; 472 for (int j = endp - endRemoved; j >= 0; j--) { 473 if (s[j] != s[j + 1]) 474 break; 475 endRemoved++; 476 } 477 //Log.d("line before removing end of comment:", s); 478 s = s[0 .. endp - endRemoved]; 479 //Log.d("line after removing end of comment:", s); 480 } 481 dsttext ~= s; 482 } 483 EditOperation op = new EditOperation(EditAction.Replace, range, dsttext); 484 _content.performOperation(op, source); 485 return; 486 } else { 487 if (srcrange.empty) 488 return; 489 TokenWithRange[] tokens = getRangeTokens(srcrange); 490 foreach(ref t; tokens) { 491 if (t.token.type == TokenType.COMMENT) { 492 if (t.token.isMultilineComment) { 493 // disable until nested comments support is implemented 494 return; 495 } else { 496 // single line comment 497 if (t.range.isInside(srcrange.start) || t.range.isInside(srcrange.end)) 498 return; 499 } 500 } 501 } 502 dstring[] dsttext; 503 for (int i = srcrange.start.line; i <= srcrange.end.line; i++) { 504 dstring s = content.line(i); 505 int charsAdded = 0; 506 if (i == srcrange.start.line) { 507 int p = srcrange.start.pos; 508 if (p < s.length) { 509 s = s[p .. $]; 510 charsAdded = -p; 511 } else { 512 charsAdded = -(cast(int)s.length); 513 s = null; 514 } 515 s = "/*" ~ s; 516 charsAdded += 2; 517 } 518 if (i == srcrange.end.line) { 519 int p = srcrange.end.pos + charsAdded; 520 s = p > 0 ? s[0..p] : null; 521 s ~= "*/"; 522 } 523 dsttext ~= s; 524 } 525 EditOperation op = new EditOperation(EditAction.Replace, srcrange, dsttext); 526 _content.performOperation(op, source); 527 return; 528 } 529 530 } 531 532 /// categorize characters in content by token types 533 void updateHighlight(dstring[] lines, TokenPropString[] props, int changeStartLine, int changeEndLine) { 534 //Log.d("updateHighlight"); 535 long ms0 = currentTimeMillis(); 536 _props = props; 537 changeStartLine = 0; 538 changeEndLine = cast(int)lines.length; 539 _lines.initialize(lines[changeStartLine..$], _file, changeStartLine); 540 _tokenizer.initialize(_lines); 541 int tokenPos = 0; 542 int tokenLine = 0; 543 ubyte category = 0; 544 try { 545 for (;;) { 546 Token token = _tokenizer.nextToken(); 547 if (token is null) { 548 //Log.d("Null token returned"); 549 break; 550 } 551 uint newPos = token.pos - 1; 552 uint newLine = token.line - 1; 553 554 //Log.d("", tokenLine + 1, ":", tokenPos + 1, " \t", token.line, ":", token.pos, "\t", token.toString); 555 if (token.type == TokenType.EOF) { 556 //Log.d("EOF token"); 557 } 558 559 // fill with category 560 for (int i = tokenLine; i <= newLine && i < lines.length; i++) { 561 int start = i > tokenLine ? 0 : tokenPos; 562 int end = i < newLine ? cast(int)lines[i].length : newPos; 563 for (int j = start; j < end; j++) { 564 if (j < _props[i].length) { 565 _props[i][j] = category; 566 } 567 } 568 } 569 570 // handle token - convert to category 571 switch(token.type) { 572 case TokenType.COMMENT: 573 category = token.isDocumentationComment ? TokenCategory.Comment_Documentation : TokenCategory.Comment; 574 break; 575 case TokenType.KEYWORD: 576 category = TokenCategory.Keyword; 577 break; 578 case TokenType.IDENTIFIER: 579 category = TokenCategory.Identifier; 580 break; 581 case TokenType.STRING: 582 category = TokenCategory.String; 583 break; 584 case TokenType.CHARACTER: 585 category = TokenCategory.Character; 586 break; 587 case TokenType.INTEGER: 588 category = TokenCategory.Integer; 589 break; 590 case TokenType.FLOAT: 591 category = TokenCategory.Float; 592 break; 593 case TokenType.OP: 594 category = TokenCategory.Op; 595 break; 596 case TokenType.INVALID: 597 switch (token.invalidTokenType) { 598 case TokenType.IDENTIFIER: 599 category = TokenCategory.Error_InvalidIdentifier; 600 break; 601 case TokenType.STRING: 602 category = TokenCategory.Error_InvalidString; 603 break; 604 case TokenType.COMMENT: 605 category = TokenCategory.Error_InvalidComment; 606 break; 607 case TokenType.OP: 608 category = TokenCategory.Error_InvalidOp; 609 break; 610 case TokenType.FLOAT: 611 case TokenType.INTEGER: 612 category = TokenCategory.Error_InvalidNumber; 613 break; 614 default: 615 category = TokenCategory.Error; 616 break; 617 } 618 break; 619 default: 620 category = 0; 621 break; 622 } 623 tokenPos = newPos; 624 tokenLine= newLine; 625 626 if (token.type == TokenType.EOF) { 627 //Log.d("EOF token"); 628 break; 629 } 630 } 631 } catch (Exception e) { 632 Log.e("exception while trying to parse D source", e); 633 } 634 _lines.close(); 635 _props = null; 636 long elapsed = currentTimeMillis() - ms0; 637 if (elapsed > 20) 638 Log.d("updateHighlight took ", elapsed, "ms"); 639 } 640 641 642 /// returns true if smart indent is supported 643 override bool supportsSmartIndents() { 644 return true; 645 } 646 647 protected bool _opInProgress; 648 protected void applyNewLineSmartIndent(EditOperation op, Object source) { 649 int line = op.newRange.end.line; 650 if (line == 0) 651 return; // not for first line 652 int prevLine = line - 1; 653 dstring lineText = _content.line(line); 654 TextLineMeasure lineMeasurement = _content.measureLine(line); 655 TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine); 656 while (prevLineMeasurement.empty && prevLine > 0) { 657 prevLine--; 658 prevLineMeasurement = _content.measureLine(prevLine); 659 } 660 if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX < prevLineMeasurement.firstNonSpaceX) { 661 dstring prevLineText = _content.line(prevLine); 662 TokenPropString prevLineTokenProps = _content.lineTokenProps(prevLine); 663 dchar lastOpChar = 0; 664 for (int j = prevLineMeasurement.lastNonSpace; j >= 0; j--) { 665 auto cat = j < prevLineTokenProps.length ? tokenCategory(prevLineTokenProps[j]) : 0; 666 if (cat == TokenCategory.Op) { 667 lastOpChar = prevLineText[j]; 668 break; 669 } else if (cat != TokenCategory.Comment && cat != TokenCategory.WhiteSpace) { 670 break; 671 } 672 } 673 int spacex = prevLineMeasurement.firstNonSpaceX; 674 if (lastOpChar == '{') 675 spacex = _content.nextTab(spacex); 676 dstring txt = _content.fillSpace(spacex); 677 EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace : 0)), [txt]); 678 _opInProgress = true; 679 _content.performOperation(op2, source); 680 _opInProgress = false; 681 } 682 } 683 684 protected void applyClosingCurlySmartIndent(EditOperation op, Object source) { 685 int line = op.newRange.end.line; 686 TextPosition p2 = findPairedBracket(op.newRange.start); 687 if (p2 == op.newRange.start || p2.line > op.newRange.start.line) 688 return; 689 int prevLine = p2.line; 690 TextLineMeasure lineMeasurement = _content.measureLine(line); 691 TextLineMeasure prevLineMeasurement = _content.measureLine(prevLine); 692 if (lineMeasurement.firstNonSpace != op.newRange.start.pos) 693 return; // not in beginning of line 694 if (lineMeasurement.firstNonSpaceX >= 0 && lineMeasurement.firstNonSpaceX != prevLineMeasurement.firstNonSpaceX) { 695 dstring prevLineText = _content.line(prevLine); 696 TokenPropString prevLineTokenProps = _content.lineTokenProps(prevLine); 697 int spacex = prevLineMeasurement.firstNonSpaceX; 698 if (spacex != lineMeasurement.firstNonSpaceX) { 699 dstring txt = _content.fillSpace(spacex); 700 txt = txt ~ "}"; 701 EditOperation op2 = new EditOperation(EditAction.Replace, TextRange(TextPosition(line, 0), TextPosition(line, lineMeasurement.firstNonSpace >= 0 ? lineMeasurement.firstNonSpace + 1 : 0)), [txt]); 702 _opInProgress = true; 703 _content.performOperation(op2, source); 704 _opInProgress = false; 705 } 706 } 707 } 708 709 /// apply smart indent, if supported 710 override void applySmartIndent(EditOperation op, Object source) { 711 if (_opInProgress) 712 return; 713 if (op.isInsertNewLine) { 714 // Enter key pressed - new line inserted or splitted 715 applyNewLineSmartIndent(op, source); 716 } else if (op.singleChar == '}') { 717 // } entered - probably need unindent 718 applyClosingCurlySmartIndent(op, source); 719 } else if (op.singleChar == '{') { 720 // { entered - probably need auto closing } 721 } 722 } 723 724 } 725