adding in some better syntax highlighting

RappelBerryPi · May 5, 2020 · 4d5b352 · 4d5b352
1 parent 8aa1dda
commit 4d5b352
Show file tree

Hide file tree

Showing 8 changed files with 52 additions and 33 deletions.
diff --git a/META-INF/MANIFEST.MF b/META-INF/MANIFEST.MF
@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: RexxEditor Plug-in
 Bundle-SymbolicName: RexxEditor; singleton:=true
-Bundle-Version: 0.1.0
+Bundle-Version: 0.2.0
 Bundle-Activator: org.imperfectmommy.rexxeditor.Activator
 Bundle-Localization: plugin
 Require-Bundle: org.eclipse.ui,

diff --git a/bin/org/imperfectmommy/rexxeditor/scanner/RexxLine.class b/bin/org/imperfectmommy/rexxeditor/scanner/RexxLine.class
diff --git a/bin/org/imperfectmommy/rexxeditor/scanner/RexxLineScanner.class b/bin/org/imperfectmommy/rexxeditor/scanner/RexxLineScanner.class
diff --git a/bin/org/imperfectmommy/rexxeditor/scanner/rules/RexxKeywordRule.class b/bin/org/imperfectmommy/rexxeditor/scanner/rules/RexxKeywordRule.class
diff --git a/src/org/imperfectmommy/rexxeditor/scanner/RexxLine.java b/src/org/imperfectmommy/rexxeditor/scanner/RexxLine.java
@@ -11,18 +11,20 @@
 
 public class RexxLine {
     protected ArrayList<RexxToken> fTokenList = new ArrayList<>();
-    protected RexxTokenList tokenList;
-    protected int           fPosition  = 0;
-    IRexxTokenRule          rules[];
-
-    public RexxLine(RexxTokenList tokenList, RexxVariableMethodContainer varMetContainer) {
-        rules          = new IRexxTokenRule[5];
-        this.tokenList = tokenList;
-        rules[0]       = new KeywordRule(tokenList);
-        rules[1]       = new VariableDefinitionRule(tokenList, varMetContainer);
-        rules[2]       = new MethodRule(tokenList);
-        rules[3]       = new ClassDefinitionRule(tokenList, varMetContainer);
-        rules[4]       = new VariableRule(tokenList, varMetContainer);
+    protected RexxTokenList        tokenList;
+    protected IToken               fDefaultToken;
+    protected int                  fPosition  = 0;
+    IRexxTokenRule                 rules[];
+
+    public RexxLine(RexxTokenList tokenList, RexxVariableMethodContainer varMetContainer, IToken fDefaultToken) {
+        rules              = new IRexxTokenRule[5];
+        this.tokenList     = tokenList;
+        rules[0]           = new KeywordRule(tokenList);
+        rules[1]           = new VariableDefinitionRule(tokenList, varMetContainer);
+        rules[2]           = new MethodRule(tokenList);
+        rules[3]           = new ClassDefinitionRule(tokenList, varMetContainer);
+        rules[4]           = new VariableRule(tokenList, varMetContainer);
+        this.fDefaultToken = fDefaultToken;
     }
 
     public void changeToken(int position, RexxToken token) {
@@ -31,13 +33,15 @@ public void changeToken(int position, RexxToken token) {
     }
 
     public void add(RexxToken token) {
-        for (int i = 0; i < rules.length; i++) {
-            RexxToken token2;
-            token2 = rules[i].evaluate(token, this);
-            if (!token2.notExists()) {
-                fTokenList.add(fPosition, token2);
-                fPosition++;
-                return;
+        if (token == fDefaultToken) {
+            for (int i = 0; i < rules.length; i++) {
+                RexxToken token2;
+                token2 = rules[i].evaluate(token, this);
+                if (!token2.notExists()) {
+                    fTokenList.add(fPosition, token2);
+                    fPosition++;
+                    return;
+                }
             }
         }
 
@@ -108,5 +112,9 @@ public String toString() {
 
         return result;
     }
-
+
+    public void setDefaultReturnToken(IToken defaultReturnToken) {
+        this.fDefaultToken = defaultReturnToken;
+    }
+
 }
diff --git a/src/org/imperfectmommy/rexxeditor/scanner/RexxLineScanner.java b/src/org/imperfectmommy/rexxeditor/scanner/RexxLineScanner.java
@@ -39,6 +39,7 @@ public class RexxLineScanner extends RuleBasedScanner {
     ArrayList<RexxContentElement> variableList;
 
     RexxConfiguration configuration;
+    private int lastOffset;
 
     //TODO: Figure out why when a character is next to a special character, it turns gray
 
@@ -47,7 +48,8 @@ public RexxLineScanner(RexxConfiguration configuration) {
         this.configuration   = configuration;
         this.varMetContainer = new RexxVariableMethodContainer();
         this.tokenList       = new RexxTokenList();
-        this.fLine           = new RexxLine(this.tokenList, this.varMetContainer);
+        this.fLine           = new RexxLine(this.tokenList, this.varMetContainer, this.fDefaultReturnToken);
+        this.lastOffset      = -2;
         /*
         Activator.getActivator().getPreferenceStore().addPropertyChangeListener(new IPropertyChangeListener() {
             
@@ -115,18 +117,20 @@ private IToken scanLine() {
             for (IRule rule : this.fRules) {
                 token = rule.evaluate(this);
                 if (!token.isUndefined()) {
+                    lastOffset = fOffset;
                     return token;
                 }
             }
         }
-        if (this.read() == EOF) {
+        int  cInt = this.read();
+        if (cInt == EOF) {
             return Token.EOF;
         }
         this.unread();
-        char c = (char) this.read();
-        if (c == '\'' || c == '"') {
-            this.unread();
+        if (lastOffset == fOffset) {
+            this.read();
         }
+        lastOffset = fOffset;
         return this.fDefaultReturnToken;
     }
 
@@ -155,25 +159,31 @@ private void setRules() {
         IRule       whitespaceRule    = new WhitespaceRule(new RexxWhitespaceDetector());
         IRule       nestedCommentRule = new RexxNestedCommentRule(this.tokenList.rexxComment);
         IRule       eolCommentRule    = new RexxEndLineCommentRule(this.tokenList.rexxComment);
-        IRule       doubleQuoteRule   = new SingleLineRule("\"", "\"", this.tokenList.rexxTag);
-        IRule       singleQuoteRule   = new SingleLineRule("\"", "\"", this.tokenList.rexxTag);
+        IRule       doubleQuoteRule   = new SingleLineRule("\"", "\"", this.tokenList.rexxTag, (char)0, true);
+        IRule       singleQuoteRule   = new SingleLineRule("\'", "\'", this.tokenList.rexxTag, (char) 0, true);
         IRule       numberRule        = new NumberRule(this.tokenList.rexxComment);
         IRule       newLineRule       = new RexxNewLineRule(this.tokenList.newLine, this.tokenList.symbol);
         IRule       keywordRule       = new RexxKeywordRule(new RexxKeywordDetector(), this.tokenList);
         IRule       symbolRule        = new RexxSymbolRule(this.tokenList);
         IRule       generalRule       = new RexxGeneralWordRule(this.tokenList.word, new RexxKeywordDetector());
         List<IRule> rules             = new ArrayList<>();
-        rules.add(whitespaceRule);
-        rules.add(nestedCommentRule);
-        rules.add(eolCommentRule);
         rules.add(doubleQuoteRule);
         rules.add(singleQuoteRule);
         rules.add(numberRule);
+        rules.add(whitespaceRule);
+        rules.add(nestedCommentRule);
+        rules.add(eolCommentRule);
         rules.add(newLineRule);
         rules.add(keywordRule);
         rules.add(symbolRule);
         rules.add(generalRule);
         this.setRules(rules.toArray(new IRule[0]));
     }
-
+
+    @Override
+    public void setDefaultReturnToken(IToken defaultReturnToken) {
+        super.setDefaultReturnToken(defaultReturnToken);
+        this.fLine.setDefaultReturnToken(defaultReturnToken);
+    }
+
 }
diff --git a/src/org/imperfectmommy/rexxeditor/scanner/rules/RexxGeneralWordRule.java b/src/org/imperfectmommy/rexxeditor/scanner/rules/RexxGeneralWordRule.java
@@ -21,7 +21,7 @@ public IToken evaluate(ICharacterScanner scanner) {
             do {
                 c = (char) scanner.read();
             } while (c != ICharacterScanner.EOF && fDetector.isWordPart(c));
-            //include '.' in compoundStems
+            //only unread if it's not a '.' (for compound stems)
             if (c != '.') {
                 scanner.unread();
             }

diff --git a/src/org/imperfectmommy/rexxeditor/scanner/rules/RexxKeywordRule.java b/src/org/imperfectmommy/rexxeditor/scanner/rules/RexxKeywordRule.java
@@ -103,6 +103,7 @@ public IToken evaluate(ICharacterScanner scanner) {
 
                 if (fDefaultToken.isUndefined()) unreadBuffer(scanner);
 
+                scanner.unread();
                 return fDefaultToken;
             }
         }