Reduce tiny allocations & reduce trivia size

Author: WebFreak001

src/dparse/lexer.d

@@ -134,9 +134,9 @@ mixin template TokenTriviaFields()
      *
      * Contains: `comment`, `whitespace`, `specialTokenSequence`
      */
-    immutable(typeof(this))[] leadingTrivia;
+    immutable(TriviaToken)[] leadingTrivia;
     /// ditto
-    immutable(typeof(this))[] trailingTrivia;
+    immutable(TriviaToken)[] trailingTrivia;
 
     string memoizedLeadingComment = null;
     string memoizedTrailingComment = null;
@@ -171,11 +171,31 @@ mixin template TokenTriviaFields()
 
 // mixin in from dparse.lexer to make error messages more managable size as the
 // entire string is dumped when there is a type mismatch.
-private enum extraFields = "import dparse.lexer:TokenTriviaFields; mixin TokenTriviaFields;";
+private enum extraFields = "import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;";
+private enum extraFieldsBare = "
+    import dparse.lexer : Token;
+
+    this(Token token) pure nothrow @safe @nogc {
+        this(token.type, token.text, token.line, token.column, token.index);
+    }
+
+    int opCmp(size_t i) const pure nothrow @safe @nogc {
+        if (index < i) return -1;
+        if (index > i) return 1;
+        return 0;
+    }
+
+    int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc {
+        return opCmp(other.index);
+    }
+";
 
 /// The token type in the D lexer
 public alias Token = std.experimental.lexer.TokenStructure!(IdType, extraFields);
 
+/// Same as Token, but doesn't contain child TriviaTokens
+public alias TriviaToken = std.experimental.lexer.TokenStructure!(IdType, extraFieldsBare);
+
 /**
  * Configure whitespace handling
  */
@@ -472,9 +492,9 @@ if (is(Unqual!(ElementEncodingType!R) : ubyte) && isDynamicArray!R)
     config.whitespaceBehavior = WhitespaceBehavior.include;
     config.commentBehavior = CommentBehavior.noIntern;
 
-    auto leadingTriviaAppender = appender!(Token[])();
+    auto leadingTriviaAppender = appender!(TriviaToken[])();
     leadingTriviaAppender.reserve(128);
-    auto trailingTriviaAppender = appender!(Token[])();
+    auto trailingTriviaAppender = appender!(TriviaToken[])();
     trailingTriviaAppender.reserve(128);
 
     auto output = appender!(typeof(return))();
@@ -485,9 +505,9 @@ if (is(Unqual!(ElementEncodingType!R) : ubyte) && isDynamicArray!R)
     case tok!"whitespace":
     case tok!"comment":
         if (!output.data.empty && lexer.front.line == output.data[$ - 1].line)
-            trailingTriviaAppender.put(lexer.front);
+            trailingTriviaAppender.put(TriviaToken(lexer.front));
         else
-            leadingTriviaAppender.put(lexer.front);
+            leadingTriviaAppender.put(TriviaToken(lexer.front));
         lexer.popFront();
         break;
     case tok!"__EOF__":

src/dparse/parser.d

@@ -8651,7 +8651,11 @@ protected: final:
                 stderr.writefln("%s(%d:%d)[error]: %s", fileName, line, column, message);
         }
         else
-            ++suppressMessages[$ - 1];
+        {
+            import std.checkedint;
+
+            ++suppressMessages.back.checked;
+        }
         while (shouldAdvance && moreTokens())
         {
             if (currentIsOneOf(tok!";", tok!"}",
@@ -9160,7 +9164,7 @@ protected: final:
         return suppressMessages.empty ? 0 : suppressMessages.back();
     }
 
-    uint[] suppressMessages;
+    TypedStackBuffer!uint suppressMessages;
     size_t index;
     int _traceDepth;
     string comment;

src/dparse/stack_buffer.d

@@ -70,7 +70,7 @@ struct StackBuffer
         }
     }
 
-    void[] opSlice()
+    inout(void[]) opSlice() inout pure nothrow @nogc @safe return scope
     {
         return arr[0 .. _length];
     }
@@ -121,3 +121,41 @@ unittest
     B d = new D;
     sb.put(d);
 }
+
+package struct TypedStackBuffer(T)
+if (__traits(isPOD, T))
+{
+    void push(T value) @trusted
+    {
+        sb.put(value);
+    }
+
+    void opOpAssign(string op : "~")(T value)
+    {
+        push(value);
+    }
+
+    ref inout(T) back() inout pure nothrow @nogc @safe @property
+    {
+        return (cast(inout(T)[])sb.opSlice[$ - T.sizeof .. $])[0];
+    }
+
+    void popBack() pure nothrow @nogc @safe @property
+    {
+        assert(!empty);
+        sb._length -= T.sizeof;
+    }
+
+    bool empty() const pure nothrow @nogc @safe @property
+    {
+        return sb.length == 0;
+    }
+
+    uint length() const pure nothrow @nogc @safe @property
+    {
+        return sb.length / T.sizeof;
+    }
+
+private:
+    StackBuffer sb;
+}

src/dparse/trivia.d

@@ -516,7 +516,7 @@ unittest
 
 /// Extracts and combines ddoc comments from trivia comments.
 string extractDdocFromTrivia(Tokens)(Tokens tokens) pure nothrow @safe
-    if (isInputRange!Tokens && is(ElementType!Tokens : Token))
+    if (isInputRange!Tokens && (is(ElementType!Tokens : Token) || is(ElementType!Tokens : TriviaToken)))
 {
     bool hasDoc;
     auto ret = appender!string;