/platform/platform-api/src/com/intellij/lexer/LookAheadLexer.java

https://bitbucket.org/nbargnesi/idea · Java · 177 lines · 126 code · 33 blank · 18 comment · 4 complexity · d4bf1f15692fda47f0fe71315530d323 MD5 · raw file

  1. /*
  2. * Copyright 2000-2009 JetBrains s.r.o.
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. package com.intellij.lexer;
  17. import com.intellij.psi.tree.IElementType;
  18. import com.intellij.util.containers.ImmutableUserMap;
  19. import com.intellij.util.containers.Queue;
  20. /**
  21. * @author peter
  22. */
  23. public abstract class LookAheadLexer extends LexerBase{
  24. private int myLastOffset;
  25. private int myLastState;
  26. private final Lexer myBaseLexer;
  27. private int myTokenStart;
  28. private final Queue<IElementType> myTypeCache;
  29. private final Queue<Integer> myEndOffsetCache;
  30. public LookAheadLexer(final Lexer baseLexer, int capacity) {
  31. myBaseLexer = baseLexer;
  32. myTypeCache = new Queue<IElementType>(capacity);
  33. myEndOffsetCache = new Queue<Integer>(capacity);
  34. }
  35. public LookAheadLexer(final Lexer baseLexer) {
  36. this(baseLexer, 64);
  37. }
  38. protected void addToken(IElementType type) {
  39. addToken(myBaseLexer.getTokenEnd(), type);
  40. }
  41. protected void addToken(int endOffset, IElementType type) {
  42. myTypeCache.addLast(type);
  43. myEndOffsetCache.addLast(endOffset);
  44. }
  45. protected void lookAhead(Lexer baseLexer) {
  46. advanceLexer(baseLexer);
  47. }
  48. public void advance() {
  49. if (!myTypeCache.isEmpty()) {
  50. myTypeCache.pullFirst();
  51. myTokenStart = myEndOffsetCache.pullFirst();
  52. }
  53. if (myTypeCache.isEmpty()) {
  54. doLookAhead();
  55. }
  56. }
  57. private void doLookAhead() {
  58. myLastOffset = myTokenStart;
  59. myLastState = myBaseLexer.getState();
  60. lookAhead(myBaseLexer);
  61. assert !myTypeCache.isEmpty();
  62. }
  63. public CharSequence getBufferSequence() {
  64. return myBaseLexer.getBufferSequence();
  65. }
  66. public int getBufferEnd() {
  67. return myBaseLexer.getBufferEnd();
  68. }
  69. protected int getCacheSize() {
  70. return myTypeCache.size();
  71. }
  72. protected void resetCacheSize(int size) {
  73. while (myTypeCache.size() > size) {
  74. myTypeCache.removeLast();
  75. myEndOffsetCache.removeLast();
  76. }
  77. }
  78. public IElementType replaceCachedType(int index, IElementType token) {
  79. return myTypeCache.set(index, token);
  80. }
  81. public int getState() {
  82. int offset = myTokenStart - myLastOffset;
  83. return myLastState | (offset << 16);
  84. }
  85. public int getTokenEnd() {
  86. return myEndOffsetCache.peekFirst();
  87. }
  88. public int getTokenStart() {
  89. return myTokenStart;
  90. }
  91. public LookAheadLexerPosition getCurrentPosition() {
  92. return new LookAheadLexerPosition(this, ImmutableUserMap.EMPTY);
  93. }
  94. public final void restore(final LexerPosition _position) {
  95. restore((LookAheadLexerPosition) _position);
  96. }
  97. protected void restore(final LookAheadLexerPosition position) {
  98. start(myBaseLexer.getBufferSequence(), position.lastOffset, myBaseLexer.getBufferEnd(), position.lastState);
  99. for (int i = 0; i < position.advanceCount; i++) {
  100. advance();
  101. }
  102. }
  103. public IElementType getTokenType() {
  104. return myTypeCache.peekFirst();
  105. }
  106. @Override
  107. public void start(CharSequence buffer, int startOffset, int endOffset, int initialState) {
  108. myBaseLexer.start(buffer, startOffset, endOffset, initialState & 0xFFFF);
  109. myTokenStart = startOffset;
  110. myTypeCache.clear();
  111. myEndOffsetCache.clear();
  112. doLookAhead();
  113. }
  114. protected static class LookAheadLexerPosition implements LexerPosition {
  115. final int lastOffset;
  116. final int lastState;
  117. final int tokenStart;
  118. final int advanceCount;
  119. final ImmutableUserMap customMap;
  120. public LookAheadLexerPosition(final LookAheadLexer lookAheadLexer, final ImmutableUserMap map) {
  121. customMap = map;
  122. lastOffset = lookAheadLexer.myLastOffset;
  123. lastState = lookAheadLexer.myLastState;
  124. tokenStart = lookAheadLexer.myTokenStart;
  125. advanceCount = lookAheadLexer.myTypeCache.size() - 1;
  126. }
  127. public ImmutableUserMap getCustomMap() {
  128. return customMap;
  129. }
  130. public int getOffset() {
  131. return tokenStart;
  132. }
  133. public int getState() {
  134. return lastState;
  135. }
  136. }
  137. protected final void advanceLexer( Lexer lexer ) {
  138. advanceAs(lexer, lexer.getTokenType());
  139. }
  140. protected final void advanceAs(Lexer lexer, IElementType type) {
  141. addToken(type);
  142. lexer.advance();
  143. }
  144. }