Upgraded streams.

compiler/dub.json

@@ -7,6 +7,6 @@
 	"license": "MIT",
 	"name": "compiler",
 	"dependencies": {
-		"streams": "~>3.4.3"
+		"streams": "~>3.5.0"
 	}
 }

compiler/dub.selections.json

@@ -1,6 +1,6 @@
 {
 	"fileVersion": 1,
 	"versions": {
-		"streams": "3.4.3"
+		"streams": "3.5.0"
 	}
 }

compiler/source/app.d

@@ -2,6 +2,7 @@ import std.stdio;
 import std.string;
 import nimpc;
 import streams;
+import streams.types.mapping;
 
 int main(string[] args) {
 	if (args.length < 2) {
@@ -12,7 +13,8 @@ int main(string[] args) {
 	writefln!"Compiling %s"(files);
 	foreach (filename; files) {
 		auto sIn = FileInputStream(toStringz(filename));
-		auto tokens = tokenize(sIn);
+		auto tokens = tokenize(mappingInputStreamFor!((ubyte b) => cast(char) b)(sIn));
+		writeln(tokens);
 	}
 	return 0;
 }

compiler/source/nimpc/lexer.d

@@ -4,6 +4,9 @@ import streams;
 import std.stdio;
 import std.regex;
 import std.array;
+import std.typecons;
+import std.uni;
+import std.string;
 
 enum TokenType {
     KEYWORD,
@@ -12,10 +15,12 @@ enum TokenType {
     LITERAL_INTEGER,
     LITERAL_FLOAT,
     LITERAL_BOOLEAN,
-    LITERAL_STRING
+    LITERAL_STRING,
+    COMMENT_INLINE,
+    SYMBOL
 }
 
-struct Token {
+immutable struct Token {
     TokenType type;
     string content;
     uint line;
@@ -33,6 +38,12 @@ class LexerException : Exception {
     }
 }
 
+class StreamEndException : Exception {
+    this() {
+        super("Stream ended.");
+    }
+}
+
 /**
  * Parses a list of tokens from an input stream of lines of code.
  * Params:
@@ -41,22 +52,51 @@ class LexerException : Exception {
  */
 Token[] tokenize(S)(S inputStream) if (isInputStream!(S, char)) {
     Appender!(Token[]) tokenApp;
+    bool streamEnded = false;
     uint line = 0;
     uint col = 0;
 
-    while (true) {
+    try {
-        char c = readChar(inputStream, line, col);
+        while (true) {
-        writeln(c);
+            // Trim whitespace from the start of each line.
+            char c = readChar(inputStream, line, col);
+            while (isWhite(c)) {
+                c = readChar(inputStream, line, col);
+            }
+            // Inline comment from here to newline.
+            if (c == '#') {
+                const commentStartCol = col;
+                c = readChar(inputStream, line, col);
+                char[] commentText;
+                while (c != '\n') {
+                    commentText ~= c;
+                    c = readChar(inputStream, line, col);
+                }
+                tokenApp ~= Token(
+                    TokenType.COMMENT_INLINE,
+                    strip(cast(string) commentText),
+                    line,
+                    commentStartCol
+                );
+            }
+        }
+    } catch (StreamEndException e) {
+        // This is expected!
     }
-
-
     return tokenApp[];
 }
 
-private char readChar(S)(S stream, uint line, uint col) if (isInputStream!(S, char)) {
+private char readChar(S)(S stream, ref uint line, ref uint col) if (isInputStream!(S, char)) {
-    auto result = readOne(stream);
+    char[1] buffer;
+    StreamResult result = stream.readFromStream(buffer);
     if (result.hasError) {
         throw new LexerException("Failed to read one more char from stream.", line, col);
     }
-    return result.element;
+    if (result.count == 0) throw new StreamEndException();
+    col++;
+    if (buffer[0] == '\n') {
+        line++;
+        col = 0;
+    }
+    return buffer[0];
 }

examples/variables.nimp

@@ -3,7 +3,7 @@
 
 define x as integer with value of 42.
 
-define b as boolean.
+define b as boolean. # another comment.
 assign true to b.
 
 define bString as string of b.