// SDLang-D
// Written in the D programming language.

module gfx.decl.sdlang.lexer;

import std.algorithm;
import std.array;
static import std.ascii;
import std.base64;
import std.bigint;
import std.conv;
import std.datetime;
import std.file;
import std.format;
import std.traits;
import std.typecons;
import std.uni;
import std.utf;
import std.variant;

import gfx.decl.sdlang.exception;
import gfx.decl.sdlang.symbol;
import gfx.decl.sdlang.token;
import gfx.decl.sdlang.util;

alias gfx.decl.sdlang.util.startsWith startsWith;

Token[] lexFile(string filename)
{
	auto source = cast(string)read(filename);
	return lexSource(source, filename);
}

Token[] lexSource(string source, string filename=null)
{
	auto lexer = scoped!Lexer(source, filename);

	// Can't use 'std.array.array(Range)' because 'lexer' is scoped
	// and therefore cannot have its reference copied.
	Appender!(Token[]) tokens;
	foreach(tok; lexer)
		tokens.put(tok);

	return tokens.data;
}

// Kind of a poor-man's yield, but fast.
// Only to be used inside Lexer.popFront (and Lexer.this).
private template accept(string symbolName)
{
	static assert(symbolName != "Value", "Value symbols must also take a value.");
	enum accept = acceptImpl!(symbolName, "null");
}
private template accept(string symbolName, string value)
{
	static assert(symbolName == "Value", "Only a Value symbol can take a value.");
	enum accept = acceptImpl!(symbolName, value);
}
private template accept(string symbolName, string value, string startLocation, string endLocation)
{
	static assert(symbolName == "Value", "Only a Value symbol can take a value.");
	enum accept = ("
		{
			_front = makeToken!"~symbolName.stringof~";
			_front.value = "~value~";
			_front.location = "~(startLocation==""? "tokenStart" : startLocation)~";
			_front.data = source[
				"~(startLocation==""? "tokenStart.index" : startLocation)~"
				..
				"~(endLocation==""? "location.index" : endLocation)~"
			];
			return;
		}
	").replace("\n", "");
}
private template acceptImpl(string symbolName, string value)
{
	enum acceptImpl = ("
		{
			_front = makeToken!"~symbolName.stringof~";
			_front.value = "~value~";
			return;
		}
	").replace("\n", "");
}

class Lexer
{
	string source;
	string filename;
	Location location; /// Location of current character in source

	private dchar  ch;         // Current character
	private dchar  nextCh;     // Lookahead character
	private size_t nextPos;    // Position of lookahead character (an index into source)
	private bool   hasNextCh;  // If false, then there's no more lookahead, just EOF
	private size_t posAfterLookahead; // Position after lookahead character (an index into source)

	private Location tokenStart;    // The starting location of the token being lexed

	// Length so far of the token being lexed, not including current char
	private size_t tokenLength;   // Length in UTF-8 code units
	private size_t tokenLength32; // Length in UTF-32 code units

	// Slight kludge:
	// If a numeric fragment is found after a Date (separated by arbitrary
	// whitespace), it could be the "hours" part of a DateTime, or it could
	// be a separate numeric literal that simply follows a plain Date. If the
	// latter, then the Date must be emitted, but numeric fragment that was
	// found after it needs to be saved for the the lexer's next iteration.
	//
	// It's a slight kludge, and could instead be implemented as a slightly
	// kludgey parser hack, but it's the only situation where SDLang's lexing
	// needs to lookahead more than one character, so this is good enough.
	private struct LookaheadTokenInfo
	{
		bool     exists          = false;
		string   numericFragment = "";
		bool     isNegative      = false;
		Location tokenStart;
	}
	private LookaheadTokenInfo lookaheadTokenInfo;

	this(string source=null, string filename=null)
	{
		this.filename = filename;
		this.source = source;

		_front = Token(symbol!"Error", Location());
		lookaheadTokenInfo = LookaheadTokenInfo.init;

		if( source.startsWith( ByteOrderMarks[BOM.UTF8] ) )
		{
			source = source[ ByteOrderMarks[BOM.UTF8].length .. $ ];
			this.source = source;
		}

		foreach(bom; ByteOrderMarks)
		if( source.startsWith(bom) )
			error(Location(filename,0,0,0), "SDL spec only supports UTF-8, not UTF-16 or UTF-32");

		if(source == "")
			mixin(accept!"EOF");

		// Prime everything
		hasNextCh = true;
		nextCh = source.decode(posAfterLookahead);
		advanceChar(ErrorOnEOF.Yes);
		location = Location(filename, 0, 0, 0);
		popFront();
	}

	@property bool empty()
	{
		return _front.symbol == symbol!"EOF";
	}

	Token _front;
	@property Token front()
	{
		return _front;
	}

	@property bool isEOF()
	{
		return location.index == source.length && !lookaheadTokenInfo.exists;
	}

	private void error(string msg)
	{
		error(location, msg);
	}

	//TODO: Take varargs and use output range sink.
	private void error(Location loc, string msg)
	{
		throw new ParseException(loc, "Error: "~msg);
	}

	private Token makeToken(string symbolName)()
	{
		auto tok = Token(symbol!symbolName, tokenStart);
		tok.data = tokenData;
		return tok;
	}

	private @property string tokenData()
	{
		return source[ tokenStart.index .. location.index ];
	}

	/// Check the lookahead character
	private bool lookahead(dchar ch)
	{
		return hasNextCh && nextCh == ch;
	}

	private bool lookahead(bool function(dchar) condition)
	{
		return hasNextCh && condition(nextCh);
	}

	private static bool isNewline(dchar ch)
	{
		return ch == '\n' || ch == '\r' || ch == lineSep || ch == paraSep;
	}

	/// Returns the length of the newline sequence, or zero if the current
	/// character is not a newline
	///
	/// Note that there are only single character sequences and the two
	/// character sequence `\r\n` as used on Windows.
	private size_t isAtNewline()
	{
		if(ch == '\n' || ch == lineSep || ch == paraSep) return 1;
		else if(ch == '\r') return lookahead('\n') ? 2 : 1;
		else return 0;
	}

	/// Is 'ch' a valid base 64 character?
	private bool isBase64(dchar ch)
	{
		if(ch >= 'A' && ch <= 'Z')
			return true;

		if(ch >= 'a' && ch <= 'z')
			return true;

		if(ch >= '0' && ch <= '9')
			return true;

		return ch == '+' || ch == '/' || ch == '=';
	}

	/// Is the current character one that's allowed
	/// immediately *after* an int/float literal?
	private bool isEndOfNumber()
	{
		if(isEOF)
			return true;

		return !isDigit(ch) && ch != ':' && ch != '_' && !isAlpha(ch);
	}

	/// Is current character the last one in an ident?
	private bool isEndOfIdentCached = false;
	private bool _isEndOfIdent;
	private bool isEndOfIdent()
	{
		if(!isEndOfIdentCached)
		{
			if(!hasNextCh)
				_isEndOfIdent = true;
			else
				_isEndOfIdent = !isIdentChar(nextCh);

			isEndOfIdentCached = true;
		}

		return _isEndOfIdent;
	}

	/// Is 'ch' a character that's allowed *somewhere* in an identifier?
	private bool isIdentChar(dchar ch)
	{
		if(isAlpha(ch))
			return true;

		else if(isNumber(ch))
			return true;

		else
			return
				ch == '-' ||
				ch == '_' ||
				ch == '.' ||
				ch == '$';
	}

	private bool isDigit(dchar ch)
	{
		return ch >= '0' && ch <= '9';
	}

	private enum KeywordResult
	{
		Accept,   // Keyword is matched
		Continue, // Keyword is not matched *yet*
		Failed,   // Keyword doesn't match
	}
	private KeywordResult checkKeyword(dstring keyword32)
	{
		// Still within length of keyword
		if(tokenLength32 < keyword32.length)
		{
			if(ch == keyword32[tokenLength32])
				return KeywordResult.Continue;
			else
				return KeywordResult.Failed;
		}

		// At position after keyword
		else if(tokenLength32 == keyword32.length)
		{
			if(isEOF || !isIdentChar(ch))
			{
				debug assert(tokenData == to!string(keyword32));
				return KeywordResult.Accept;
			}
			else
				return KeywordResult.Failed;
		}

		assert(0, "Fell off end of keyword to check");
	}

	enum ErrorOnEOF { No, Yes }

	/// Advance one code point.
	private void advanceChar(ErrorOnEOF errorOnEOF)
	{
		if(auto cnt = isAtNewline())
		{
			if (cnt == 1)
				location.line++;
			location.col = 0;
		}
		else
			location.col++;

		location.index = nextPos;

		nextPos = posAfterLookahead;
		ch      = nextCh;

		if(!hasNextCh)
		{
			if(errorOnEOF == ErrorOnEOF.Yes)
				error("Unexpected end of file");

			return;
		}

		tokenLength32++;
		tokenLength = location.index - tokenStart.index;

		if(nextPos == source.length)
		{
			nextCh = dchar.init;
			hasNextCh = false;
			return;
		}

		nextCh = source.decode(posAfterLookahead);
		isEndOfIdentCached = false;
	}

	/// Advances the specified amount of characters
	private void advanceChar(size_t count, ErrorOnEOF errorOnEOF)
	{
		while(count-- > 0)
			advanceChar(errorOnEOF);
	}

	void popFront()
	{
		// -- Main Lexer -------------

		eatWhite();

		if(isEOF)
			mixin(accept!"EOF");

		tokenStart    = location;
		tokenLength   = 0;
		tokenLength32 = 0;
		isEndOfIdentCached = false;

		if(lookaheadTokenInfo.exists)
		{
			tokenStart = lookaheadTokenInfo.tokenStart;

			auto prevLATokenInfo = lookaheadTokenInfo;
			lookaheadTokenInfo = LookaheadTokenInfo.init;
			lexNumeric(prevLATokenInfo);
			return;
		}

		if(ch == '=')
		{
			advanceChar(ErrorOnEOF.No);
			mixin(accept!"=");
		}

		else if(ch == '{')
		{
			advanceChar(ErrorOnEOF.No);
			mixin(accept!"{");
		}

		else if(ch == '}')
		{
			advanceChar(ErrorOnEOF.No);
			mixin(accept!"}");
		}

		else if(ch == ':')
		{
			advanceChar(ErrorOnEOF.No);
			mixin(accept!":");
		}

		else if(ch == ';')
		{
			advanceChar(ErrorOnEOF.No);
			mixin(accept!"EOL");
		}

		else if(auto cnt = isAtNewline())
		{
			advanceChar(cnt, ErrorOnEOF.No);
			mixin(accept!"EOL");
		}

		else if(isAlpha(ch) || ch == '_')
			lexIdentKeyword();

		else if(ch == '"')
			lexRegularString();

		else if(ch == '`')
			lexRawString();

		else if(ch == '\'')
			lexCharacter();

		else if(ch == '[')
			lexBinary();

		else if(ch == '-' || ch == '.' || isDigit(ch))
			lexNumeric();

		else
		{
			if(ch == ',')
				error("Unexpected comma: SDLang is not a comma-separated format.");
			else if(std.ascii.isPrintable(ch))
				error(text("Unexpected: ", ch));
			else
				error("Unexpected character code 0x%02X".format(ch));

			advanceChar(ErrorOnEOF.No);
		}
	}

	/// Lex Ident or Keyword
	private void lexIdentKeyword()
	{
		assert(isAlpha(ch) || ch == '_');

		// Keyword
		struct Key
		{
			dstring name;
			Value value;
			bool failed = false;
		}
		static Key[5] keywords;
		static keywordsInited = false;
		if(!keywordsInited)
		{
			// Value (as a std.variant-based type) can't be statically inited
			keywords[0] = Key("true",  Value(true ));
			keywords[1] = Key("false", Value(false));
			keywords[2] = Key("on",    Value(true ));
			keywords[3] = Key("off",   Value(false));
			keywords[4] = Key("null",  Value(null ));
			keywordsInited = true;
		}

		foreach(ref key; keywords)
			key.failed = false;

		auto numKeys = keywords.length;

		do
		{
			foreach(ref key; keywords)
			if(!key.failed)
			{
				final switch(checkKeyword(key.name))
				{
				case KeywordResult.Accept:
					mixin(accept!("Value", "key.value"));

				case KeywordResult.Continue:
					break;

				case KeywordResult.Failed:
					key.failed = true;
					numKeys--;
					break;
				}
			}

			if(numKeys == 0)
			{
				lexIdent();
				return;
			}

			advanceChar(ErrorOnEOF.No);

		} while(!isEOF);

		foreach(ref key; keywords)
		if(!key.failed)
		if(key.name.length == tokenLength32+1)
			mixin(accept!("Value", "key.value"));

		mixin(accept!"Ident");
	}

	/// Lex Ident
	private void lexIdent()
	{
		if(tokenLength == 0)
			assert(isAlpha(ch) || ch == '_');

		while(!isEOF && isIdentChar(ch))
			advanceChar(ErrorOnEOF.No);

		mixin(accept!"Ident");
	}

	/// Lex regular string
	private void lexRegularString()
	{
		assert(ch == '"');

		Appender!string buf;
		size_t spanStart = nextPos;

		// Doesn't include current character
		void updateBuf()
		{
			if(location.index == spanStart)
				return;

			buf.put( source[spanStart..location.index] );
		}

		advanceChar(ErrorOnEOF.Yes);
		while(ch != '"')
		{
			if(ch == '\\')
			{
				updateBuf();

				bool wasEscSequence = true;
				if(hasNextCh)
				{
					switch(nextCh)
					{
					case 'n':  buf.put('\n'); break;
					case 'r':  buf.put('\r'); break;
					case 't':  buf.put('\t'); break;
					case '"':  buf.put('\"'); break;
					case '\\': buf.put('\\'); break;
					default: wasEscSequence = false; break;
					}
				}

				if(wasEscSequence)
				{
					advanceChar(ErrorOnEOF.Yes);
					spanStart = nextPos;
				}
				else
				{
					eatWhite(false);
					spanStart = location.index;
				}
			}

			else if(isNewline(ch))
				error("Unescaped newlines are only allowed in raw strings, not regular strings.");

			advanceChar(ErrorOnEOF.Yes);
		}

		updateBuf();
		advanceChar(ErrorOnEOF.No); // Skip closing double-quote
		mixin(accept!("Value", "buf.data"));
	}

	/// Lex raw string
	private void lexRawString()
	{
		assert(ch == '`');

		do
			advanceChar(ErrorOnEOF.Yes);
		while(ch != '`');

		advanceChar(ErrorOnEOF.No); // Skip closing back-tick
		mixin(accept!("Value", "tokenData[1..$-1]"));
	}

	/// Lex character literal
	private void lexCharacter()
	{
		assert(ch == '\'');
		advanceChar(ErrorOnEOF.Yes); // Skip opening single-quote

		dchar value;
		if(ch == '\\')
		{
			advanceChar(ErrorOnEOF.Yes); // Skip escape backslash
			switch(ch)
			{
			case 'n':  value = '\n'; break;
			case 'r':  value = '\r'; break;
			case 't':  value = '\t'; break;
			case '\'': value = '\''; break;
			case '\\': value = '\\'; break;
			default: error("Invalid escape sequence.");
			}
		}
		else if(isNewline(ch))
			error("Newline not alowed in character literal.");
		else
			value = ch;
		advanceChar(ErrorOnEOF.Yes); // Skip the character itself

		if(ch == '\'')
			advanceChar(ErrorOnEOF.No); // Skip closing single-quote
		else
			error("Expected closing single-quote.");

		mixin(accept!("Value", "value"));
	}

	/// Lex base64 binary literal
	private void lexBinary()
	{
		assert(ch == '[');
		advanceChar(ErrorOnEOF.Yes);

		void eatBase64Whitespace()
		{
			while(!isEOF && isWhite(ch))
			{
				if(isNewline(ch))
					advanceChar(ErrorOnEOF.Yes);

				if(!isEOF && isWhite(ch))
					eatWhite();
			}
		}

		eatBase64Whitespace();

		// Iterates all valid base64 characters, ending at ']'.
		// Skips all whitespace. Throws on invalid chars.
		struct Base64InputRange
		{
			Lexer lexer;
			private bool isInited = false;
			private int numInputCharsMod4 = 0;

			@property bool empty()
			{
				if(lexer.ch == ']')
				{
					if(numInputCharsMod4 != 0)
						lexer.error("Length of Base64 encoding must be a multiple of 4. ("~to!string(numInputCharsMod4)~")");

					return true;
				}

				return false;
			}

			@property dchar front()
			{
				return lexer.ch;
			}

			void popFront()
			{
				auto lex = lexer;

				if(!isInited)
				{
					if(lexer.isBase64(lexer.ch))
					{
						numInputCharsMod4++;
						numInputCharsMod4 %= 4;
					}

					isInited = true;
				}

				lex.advanceChar(lex.ErrorOnEOF.Yes);

				eatBase64Whitespace();

				if(lex.isEOF)
					lex.error("Unexpected end of file.");

				if(lex.ch != ']')
				{
					if(!lex.isBase64(lex.ch))
						lex.error("Invalid character in base64 binary literal.");

					numInputCharsMod4++;
					numInputCharsMod4 %= 4;
				}
			}
		}

		// This is a slow ugly hack. It's necessary because Base64.decode
		// currently requires the source to have known length.
		//TODO: Remove this when DMD issue #9543 is fixed.
		dchar[] tmpBuf = array(Base64InputRange(this));

		Appender!(ubyte[]) outputBuf;
		// Ugly workaround for DMD issue #9102
		//TODO: Remove this when DMD #9102 is fixed
		struct OutputBuf
		{
			void put(ubyte ch)
			{
				outputBuf.put(ch);
			}
		}

		try
			//Base64.decode(Base64InputRange(this), OutputBuf());
			Base64.decode(tmpBuf, OutputBuf());

		catch(Base64Exception e)
			error("Invalid character in base64 binary literal.");

		advanceChar(ErrorOnEOF.No); // Skip ']'
		mixin(accept!("Value", "outputBuf.data"));
	}

	private BigInt toBigInt(bool isNegative, string absValue)
	{
		auto num = BigInt(absValue);
		assert(num >= 0);

		if(isNegative)
			num = -num;

		return num;
	}

	/// Lex [0-9]+, but without emitting a token.
	/// This is used by the other numeric parsing functions.
	private string lexNumericFragment()
	{
		if(!isDigit(ch))
			error("Expected a digit 0-9.");

		auto spanStart = location.index;

		do
		{
			advanceChar(ErrorOnEOF.No);
		} while(!isEOF && isDigit(ch));

		return source[spanStart..location.index];
	}

	/// Lex anything that starts with 0-9 or '-'. Ints, floats, dates, etc.
	private void lexNumeric(LookaheadTokenInfo laTokenInfo = LookaheadTokenInfo.init)
	{
		bool isNegative;
		string firstFragment;
		if(laTokenInfo.exists)
		{
			firstFragment = laTokenInfo.numericFragment;
			isNegative    = laTokenInfo.isNegative;
		}
		else
		{
			assert(ch == '-' || ch == '.' || isDigit(ch));

			// Check for negative
			isNegative = ch == '-';
			if(isNegative)
				advanceChar(ErrorOnEOF.Yes);

			// Some floating point with omitted leading zero?
			if(ch == '.')
			{
				lexFloatingPoint("");
				return;
			}

			firstFragment = lexNumericFragment();
		}

		// Long integer (64-bit signed)?
		if(ch == 'L' || ch == 'l')
		{
			advanceChar(ErrorOnEOF.No);

			// BigInt(long.min) is a workaround for DMD issue #9548
			auto num = toBigInt(isNegative, firstFragment);
			if(num < BigInt(long.min) || num > long.max)
				error(tokenStart, "Value doesn't fit in 64-bit signed long integer: "~to!string(num));

			mixin(accept!("Value", "num.toLong()"));
		}

		// Float (32-bit signed)?
		else if(ch == 'F' || ch == 'f')
		{
			auto value = to!float(tokenData);
			advanceChar(ErrorOnEOF.No);
			mixin(accept!("Value", "value"));
		}

		// Double float (64-bit signed) with suffix?
		else if((ch == 'D' || ch == 'd') && !lookahead(':')
		)
		{
			auto value = to!double(tokenData);
			advanceChar(ErrorOnEOF.No);
			mixin(accept!("Value", "value"));
		}

		// Decimal (128+ bits signed)?
		else if(
			(ch == 'B' || ch == 'b') &&
			(lookahead('D') || lookahead('d'))
		)
		{
			auto value = to!real(tokenData);
			advanceChar(ErrorOnEOF.No);
			advanceChar(ErrorOnEOF.No);
			mixin(accept!("Value", "value"));
		}

		// Some floating point?
		else if(ch == '.')
			lexFloatingPoint(firstFragment);

		// Some date?
		else if(ch == '/' && hasNextCh && isDigit(nextCh))
			lexDate(isNegative, firstFragment);

		// Some time span?
		else if(ch == ':' || ch == 'd')
			lexTimeSpan(isNegative, firstFragment);

		// Integer (32-bit signed)?
		else if(isEndOfNumber())
		{
			auto num = toBigInt(isNegative, firstFragment);
			if(num < int.min || num > int.max)
				error(tokenStart, "Value doesn't fit in 32-bit signed integer: "~to!string(num));

			mixin(accept!("Value", "num.toInt()"));
		}

		// Invalid suffix
		else
			error("Invalid integer suffix.");
	}

	/// Lex any floating-point literal (after the initial numeric fragment was lexed)
	private void lexFloatingPoint(string firstPart)
	{
		assert(ch == '.');
		advanceChar(ErrorOnEOF.No);

		auto secondPart = lexNumericFragment();

		try
		{
			// Double float (64-bit signed) with suffix?
			if(ch == 'D' || ch == 'd')
			{
				auto value = to!double(tokenData);
				advanceChar(ErrorOnEOF.No);
				mixin(accept!("Value", "value"));
			}

			// Float (32-bit signed)?
			else if(ch == 'F' || ch == 'f')
			{
				auto value = to!float(tokenData);
				advanceChar(ErrorOnEOF.No);
				mixin(accept!("Value", "value"));
			}

			// Decimal (128+ bits signed)?
			else if(ch == 'B' || ch == 'b')
			{
				auto value = to!real(tokenData);
				advanceChar(ErrorOnEOF.Yes);

				if(!isEOF && (ch == 'D' || ch == 'd'))
				{
					advanceChar(ErrorOnEOF.No);
					if(isEndOfNumber())
						mixin(accept!("Value", "value"));
				}

				error("Invalid floating point suffix.");
			}

			// Double float (64-bit signed) without suffix?
			else if(isEOF || !isIdentChar(ch))
			{
				auto value = to!double(tokenData);
				mixin(accept!("Value", "value"));
			}

			// Invalid suffix
			else
				error("Invalid floating point suffix.");
		}
		catch(ConvException e)
			error("Invalid floating point literal.");
	}

	private Date makeDate(bool isNegative, string yearStr, string monthStr, string dayStr)
	{
		BigInt biTmp;

		biTmp = BigInt(yearStr);
		if(isNegative)
			biTmp = -biTmp;
		if(biTmp < int.min || biTmp > int.max)
			error(tokenStart, "Date's year is out of range. (Must fit within a 32-bit signed int.)");
		auto year = biTmp.toInt();

		biTmp = BigInt(monthStr);
		if(biTmp < 1 || biTmp > 12)
			error(tokenStart, "Date's month is out of range.");
		auto month = biTmp.toInt();

		biTmp = BigInt(dayStr);
		if(biTmp < 1 || biTmp > 31)
			error(tokenStart, "Date's month is out of range.");
		auto day = biTmp.toInt();

		return Date(year, month, day);
	}

	private DateTimeFrac makeDateTimeFrac(
		bool isNegative, Date date, string hourStr, string minuteStr,
		string secondStr, string millisecondStr
	)
	{
		BigInt biTmp;

		biTmp = BigInt(hourStr);
		if(biTmp < int.min || biTmp > int.max)
			error(tokenStart, "Datetime's hour is out of range.");
		auto numHours = biTmp.toInt();

		biTmp = BigInt(minuteStr);
		if(biTmp < 0 || biTmp > int.max)
			error(tokenStart, "Datetime's minute is out of range.");
		auto numMinutes = biTmp.toInt();

		int numSeconds = 0;
		if(secondStr != "")
		{
			biTmp = BigInt(secondStr);
			if(biTmp < 0 || biTmp > int.max)
				error(tokenStart, "Datetime's second is out of range.");
			numSeconds = biTmp.toInt();
		}

		int millisecond = 0;
		if(millisecondStr != "")
		{
			biTmp = BigInt(millisecondStr);
			if(biTmp < 0 || biTmp > int.max)
				error(tokenStart, "Datetime's millisecond is out of range.");
			millisecond = biTmp.toInt();

			if(millisecondStr.length == 1)
				millisecond *= 100;
			else if(millisecondStr.length == 2)
				millisecond *= 10;
		}

		Duration fracSecs = millisecond.msecs;

		auto offset = hours(numHours) + minutes(numMinutes) + seconds(numSeconds);

		if(isNegative)
		{
			offset   = -offset;
			fracSecs = -fracSecs;
		}

		return DateTimeFrac(DateTime(date) + offset, fracSecs);
	}

	private Duration makeDuration(
		bool isNegative, string dayStr,
		string hourStr, string minuteStr, string secondStr,
		string millisecondStr
	)
	{
		BigInt biTmp;

		long day = 0;
		if(dayStr != "")
		{
			biTmp = BigInt(dayStr);
			if(biTmp < long.min || biTmp > long.max)
				error(tokenStart, "Time span's day is out of range.");
			day = biTmp.toLong();
		}

		biTmp = BigInt(hourStr);
		if(biTmp < long.min || biTmp > long.max)
			error(tokenStart, "Time span's hour is out of range.");
		auto hour = biTmp.toLong();

		biTmp = BigInt(minuteStr);
		if(biTmp < long.min || biTmp > long.max)
			error(tokenStart, "Time span's minute is out of range.");
		auto minute = biTmp.toLong();

		biTmp = BigInt(secondStr);
		if(biTmp < long.min || biTmp > long.max)
			error(tokenStart, "Time span's second is out of range.");
		auto second = biTmp.toLong();

		long millisecond = 0;
		if(millisecondStr != "")
		{
			biTmp = BigInt(millisecondStr);
			if(biTmp < long.min || biTmp > long.max)
				error(tokenStart, "Time span's millisecond is out of range.");
			millisecond = biTmp.toLong();

			if(millisecondStr.length == 1)
				millisecond *= 100;
			else if(millisecondStr.length == 2)
				millisecond *= 10;
		}

		auto duration =
			dur!"days"   (day)    +
			dur!"hours"  (hour)   +
			dur!"minutes"(minute) +
			dur!"seconds"(second) +
			dur!"msecs"  (millisecond);

		if(isNegative)
			duration = -duration;

		return duration;
	}

	// This has to reproduce some weird corner case behaviors from the
	// original Java version of SDL. So some of this may seem weird.
	private Nullable!Duration getTimeZoneOffset(string str)
	{
		if(str.length < 2)
			return Nullable!Duration(); // Unknown timezone

		if(str[0] != '+' && str[0] != '-')
			return Nullable!Duration(); // Unknown timezone

		auto isNegative = str[0] == '-';

		string numHoursStr;
		string numMinutesStr;
		if(str[1] == ':')
		{
			numMinutesStr = str[1..$];
			numHoursStr = "";
		}
		else
		{
			numMinutesStr = str.find(':');
			numHoursStr = str[1 .. $-numMinutesStr.length];
		}

		long numHours = 0;
		long numMinutes = 0;
		bool isUnknown = false;
		try
		{
			switch(numHoursStr.length)
			{
			case 0:
				if(numMinutesStr.length == 3)
				{
					numHours   = 0;
					numMinutes = to!long(numMinutesStr[1..$]);
				}
				else
					isUnknown = true;
				break;

			case 1:
			case 2:
				if(numMinutesStr.length == 0)
				{
					numHours   = to!long(numHoursStr);
					numMinutes = 0;
				}
				else if(numMinutesStr.length == 3)
				{
					numHours   = to!long(numHoursStr);
					numMinutes = to!long(numMinutesStr[1..$]);
				}
				else
					isUnknown = true;
				break;

			default:
				if(numMinutesStr.length == 0)
				{
					// Yes, this is correct
					numHours   = 0;
					numMinutes = to!long(numHoursStr[1..$]);
				}
				else
					isUnknown = true;
				break;
			}
		}
		catch(ConvException e)
			isUnknown = true;

		if(isUnknown)
			return Nullable!Duration(); // Unknown timezone

		auto timeZoneOffset = hours(numHours) + minutes(numMinutes);
		if(isNegative)
			timeZoneOffset = -timeZoneOffset;

		// Timezone valid
		return Nullable!Duration(timeZoneOffset);
	}

	/// Lex date or datetime (after the initial numeric fragment was lexed)
	private void lexDate(bool isDateNegative, string yearStr)
	{
		assert(ch == '/');

		// Lex months
		advanceChar(ErrorOnEOF.Yes); // Skip '/'
		auto monthStr = lexNumericFragment();

		// Lex days
		if(ch != '/')
			error("Invalid date format: Missing days.");
		advanceChar(ErrorOnEOF.Yes); // Skip '/'
		auto dayStr = lexNumericFragment();

		auto date = makeDate(isDateNegative, yearStr, monthStr, dayStr);

		if(!isEndOfNumber() && ch != '/')
			error("Dates cannot have suffixes.");

		// Date?
		if(isEOF)
			mixin(accept!("Value", "date"));

		auto endOfDate = location;

		while(
			!isEOF &&
			( ch == '\\' || ch == '/' || (isWhite(ch) && !isNewline(ch)) )
		)
		{
			if(ch == '\\' && hasNextCh && isNewline(nextCh))
			{
				advanceChar(ErrorOnEOF.Yes);
				if(isAtNewline())
					advanceChar(ErrorOnEOF.Yes);
				advanceChar(ErrorOnEOF.No);
			}

			eatWhite();
		}

		// Date?
		if(isEOF || (!isDigit(ch) && ch != '-'))
			mixin(accept!("Value", "date", "", "endOfDate.index"));

		auto startOfTime = location;

		// Is time negative?
		bool isTimeNegative = ch == '-';
		if(isTimeNegative)
			advanceChar(ErrorOnEOF.Yes);

		// Lex hours
		auto hourStr = ch == '.'? "" : lexNumericFragment();

		// Lex minutes
		if(ch != ':')
		{
			// No minutes found. Therefore we had a plain Date followed
			// by a numeric literal, not a DateTime.
			lookaheadTokenInfo.exists          = true;
			lookaheadTokenInfo.numericFragment = hourStr;
			lookaheadTokenInfo.isNegative      = isTimeNegative;
			lookaheadTokenInfo.tokenStart      = startOfTime;
			mixin(accept!("Value", "date", "", "endOfDate.index"));
		}
		advanceChar(ErrorOnEOF.Yes); // Skip ':'
		auto minuteStr = lexNumericFragment();

		// Lex seconds, if exists
		string secondStr;
		if(ch == ':')
		{
			advanceChar(ErrorOnEOF.Yes); // Skip ':'
			secondStr = lexNumericFragment();
		}

		// Lex milliseconds, if exists
		string millisecondStr;
		if(ch == '.')
		{
			advanceChar(ErrorOnEOF.Yes); // Skip '.'
			millisecondStr = lexNumericFragment();
		}

		auto dateTimeFrac = makeDateTimeFrac(isTimeNegative, date, hourStr, minuteStr, secondStr, millisecondStr);

		// Lex zone, if exists
		if(ch == '-')
		{
			advanceChar(ErrorOnEOF.Yes); // Skip '-'
			auto timezoneStart = location;

			if(!isAlpha(ch))
				error("Invalid timezone format.");

			while(!isEOF && !isWhite(ch))
				advanceChar(ErrorOnEOF.No);

			auto timezoneStr = source[timezoneStart.index..location.index];
			if(timezoneStr.startsWith("GMT"))
			{
				auto isoPart = timezoneStr["GMT".length..$];
				auto offset = getTimeZoneOffset(isoPart);

				if(offset.isNull())
				{
					// Unknown time zone
					mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
				}
				else
				{
					auto timezone = new immutable SimpleTimeZone(offset.get());
					mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
				}
			}

			try
			{
				version(Windows)
					auto timezone = WindowsTimeZone.getTimeZone(timezoneStr);
				else version(Posix)
					auto timezone = PosixTimeZone.getTimeZone(timezoneStr);
				else
					static assert(0);

				if(timezone)
					mixin(accept!("Value", "SysTime(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezone)"));
			}
			catch(TimeException e)
			{
				// Time zone not found. So just move along to "Unknown time zone" below.
			}

			// Unknown time zone
			mixin(accept!("Value", "DateTimeFracUnknownZone(dateTimeFrac.dateTime, dateTimeFrac.fracSecs, timezoneStr)"));
		}

		if(!isEndOfNumber())
			error("Date-Times cannot have suffixes.");

		mixin(accept!("Value", "dateTimeFrac"));
	}

	/// Lex time span (after the initial numeric fragment was lexed)
	private void lexTimeSpan(bool isNegative, string firstPart)
	{
		assert(ch == ':' || ch == 'd');

		string dayStr = "";
		string hourStr;

		// Lexed days?
		bool hasDays = ch == 'd';
		if(hasDays)
		{
			dayStr = firstPart;
			advanceChar(ErrorOnEOF.Yes); // Skip 'd'

			// Lex hours
			if(ch != ':')
				error("Invalid time span format: Missing hours.");
			advanceChar(ErrorOnEOF.Yes); // Skip ':'
			hourStr = lexNumericFragment();
		}
		else
			hourStr = firstPart;

		// Lex minutes
		if(ch != ':')
			error("Invalid time span format: Missing minutes.");
		advanceChar(ErrorOnEOF.Yes); // Skip ':'
		auto minuteStr = lexNumericFragment();

		// Lex seconds
		if(ch != ':')
			error("Invalid time span format: Missing seconds.");
		advanceChar(ErrorOnEOF.Yes); // Skip ':'
		auto secondStr = lexNumericFragment();

		// Lex milliseconds, if exists
		string millisecondStr = "";
		if(ch == '.')
		{
			advanceChar(ErrorOnEOF.Yes); // Skip '.'
			millisecondStr = lexNumericFragment();
		}

		if(!isEndOfNumber())
			error("Time spans cannot have suffixes.");

		auto duration = makeDuration(isNegative, dayStr, hourStr, minuteStr, secondStr, millisecondStr);
		mixin(accept!("Value", "duration"));
	}

	/// Advances past whitespace and comments
	private void eatWhite(bool allowComments=true)
	{
		// -- Comment/Whitepace Lexer -------------

		enum State
		{
			normal,
			lineComment,  // Got "#" or "//" or "--", Eating everything until newline
			blockComment, // Got "/*", Eating everything until "*/"
		}

		if(isEOF)
			return;

		Location commentStart;
		State state = State.normal;
		bool consumeNewlines = false;
		bool hasConsumedNewline = false;
		while(true)
		{
			final switch(state)
			{
			case State.normal:

				if(ch == '\\')
				{
					commentStart = location;
					consumeNewlines = true;
					hasConsumedNewline = false;
				}

				else if(ch == '#')
				{
					if(!allowComments)
						return;

					commentStart = location;
					state = State.lineComment;
					continue;
				}

				else if(ch == '/' || ch == '-')
				{
					commentStart = location;
					if(lookahead(ch))
					{
						if(!allowComments)
							return;

						advanceChar(ErrorOnEOF.No);
						state = State.lineComment;
						continue;
					}
					else if(ch == '/' && lookahead('*'))
					{
						if(!allowComments)
							return;

						advanceChar(ErrorOnEOF.No);
						state = State.blockComment;
						continue;
					}
					else
						return; // Done
				}
				else if(isAtNewline())
				{
					if(consumeNewlines)
						hasConsumedNewline = true;
					else
						return; // Done
				}
				else if(!isWhite(ch))
				{
					if(consumeNewlines)
					{
						if(hasConsumedNewline)
							return; // Done
						else
							error("Only whitespace can come between a line-continuation backslash and the following newline.");
					}
					else
						return; // Done
				}

				break;

			case State.lineComment:
				if(lookahead(&isNewline))
					state = State.normal;
				break;

			case State.blockComment:
				if(ch == '*' && lookahead('/'))
				{
					advanceChar(ErrorOnEOF.No);
					state = State.normal;
				}
				break;
			}

			advanceChar(ErrorOnEOF.No);
			if(isEOF)
			{
				// Reached EOF

				if(consumeNewlines && !hasConsumedNewline)
					error("Missing newline after line-continuation backslash.");

				else if(state == State.blockComment)
					error(commentStart, "Unterminated block comment.");

				else
					return; // Done, reached EOF
			}
		}
	}
}

version(unittest)
{
	import std.stdio;

	version(Have_unit_threaded) import unit_threaded;
	else                        { enum DontTest; }

	private auto loc  = Location("filename", 0, 0, 0);
	private auto loc2 = Location("a", 1, 1, 1);

	@("lexer: EOL")
	unittest
	{
		assert([Token(symbol!"EOL",loc)             ] == [Token(symbol!"EOL",loc)              ] );
		assert([Token(symbol!"EOL",loc,Value(7),"A")] == [Token(symbol!"EOL",loc2,Value(7),"B")] );
	}

	private int numErrors = 0;
	@DontTest
	private void testLex(string source, Token[] expected, bool test_locations = false, string file=__FILE__, size_t line=__LINE__)
	{
		Token[] actual;
		try
			actual = lexSource(source, "filename");
		catch(ParseException e)
		{
			numErrors++;
			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
			stderr.writeln("    Expected:");
			stderr.writeln("        ", expected);
			stderr.writeln("    Actual: ParseException thrown:");
			stderr.writeln("        ", e.msg);
			return;
		}

		bool is_same = actual == expected;
		if (is_same && test_locations) {
			is_same = actual.map!(t => t.location).equal(expected.map!(t => t.location));
		}

		if(!is_same)
		{
			numErrors++;
			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
			stderr.writeln("    Expected:");
			stderr.writeln("        ", expected);
			stderr.writeln("    Actual:");
			stderr.writeln("        ", actual);

			if(expected.length > 1 || actual.length > 1)
			{
				stderr.writeln("    expected.length: ", expected.length);
				stderr.writeln("    actual.length:   ", actual.length);

				if(actual.length == expected.length)
				foreach(i; 0..actual.length)
				if(actual[i] != expected[i])
				{
					stderr.writeln("    Unequal at index #", i, ":");
					stderr.writeln("        Expected:");
					stderr.writeln("            ", expected[i]);
					stderr.writeln("        Actual:");
					stderr.writeln("            ", actual[i]);
				}
			}
		}
	}

	private void testLexThrows(string file=__FILE__, size_t line=__LINE__)(string source)
	{
		bool hadException = false;
		Token[] actual;
		try
			actual = lexSource(source, "filename");
		catch(ParseException e)
			hadException = true;

		if(!hadException)
		{
			numErrors++;
			stderr.writeln(file, "(", line, "): testLex failed on: ", source);
			stderr.writeln("    Expected ParseException");
			stderr.writeln("    Actual:");
			stderr.writeln("        ", actual);
		}
	}
}

@("sdlang lexer")
unittest
{
	testLex("",        []);
	testLex(" ",       []);
	testLex("\\\n",    []);
	testLex("/*foo*/", []);
	testLex("/* multiline \n comment */", []);
	testLex("/* * */", []);
	testLexThrows("/* ");

	testLex(":",  [ Token(symbol!":",  loc) ]);
	testLex("=",  [ Token(symbol!"=",  loc) ]);
	testLex("{",  [ Token(symbol!"{",  loc) ]);
	testLex("}",  [ Token(symbol!"}",  loc) ]);
	testLex(";",  [ Token(symbol!"EOL",loc) ]);
	testLex("\n", [ Token(symbol!"EOL",loc) ]);

	testLex("foo",     [ Token(symbol!"Ident",loc,Value(null),"foo")     ]);
	testLex("_foo",    [ Token(symbol!"Ident",loc,Value(null),"_foo")    ]);
	testLex("foo.bar", [ Token(symbol!"Ident",loc,Value(null),"foo.bar") ]);
	testLex("foo-bar", [ Token(symbol!"Ident",loc,Value(null),"foo-bar") ]);
	testLex("foo.",    [ Token(symbol!"Ident",loc,Value(null),"foo.")    ]);
	testLex("foo-",    [ Token(symbol!"Ident",loc,Value(null),"foo-")    ]);
	testLexThrows(".foo");

	testLex("foo bar", [
		Token(symbol!"Ident",loc,Value(null),"foo"),
		Token(symbol!"Ident",loc,Value(null),"bar"),
	]);
	testLex("foo \\  \n  \n  bar", [
		Token(symbol!"Ident",loc,Value(null),"foo"),
		Token(symbol!"Ident",loc,Value(null),"bar"),
	]);
	testLex("foo \\  \n \\ \n  bar", [
		Token(symbol!"Ident",loc,Value(null),"foo"),
		Token(symbol!"Ident",loc,Value(null),"bar"),
	]);
	testLexThrows("foo \\ ");
	testLexThrows("foo \\ bar");
	testLexThrows("foo \\  \n  \\ ");
	testLexThrows("foo \\  \n  \\ bar");

	testLex("foo : = { } ; \n bar \n", [
		Token(symbol!"Ident",loc,Value(null),"foo"),
		Token(symbol!":",loc),
		Token(symbol!"=",loc),
		Token(symbol!"{",loc),
		Token(symbol!"}",loc),
		Token(symbol!"EOL",loc),
		Token(symbol!"EOL",loc),
		Token(symbol!"Ident",loc,Value(null),"bar"),
		Token(symbol!"EOL",loc),
	]);

	testLexThrows("<");
	testLexThrows("*");
	testLexThrows(`\`);

	// Integers
	testLex(  "7", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
	testLex( "-7", [ Token(symbol!"Value",loc,Value(cast( int)-7)) ]);
	testLex( "7L", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
	testLex( "7l", [ Token(symbol!"Value",loc,Value(cast(long) 7)) ]);
	testLex("-7L", [ Token(symbol!"Value",loc,Value(cast(long)-7)) ]);
	testLex(  "0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);
	testLex( "-0", [ Token(symbol!"Value",loc,Value(cast( int) 0)) ]);

	testLex("7/**/", [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);
	testLex("7#",    [ Token(symbol!"Value",loc,Value(cast( int) 7)) ]);

	testLex("7 A", [
		Token(symbol!"Value",loc,Value(cast(int)7)),
		Token(symbol!"Ident",loc,Value(      null),"A"),
	]);
	testLexThrows("7A");
	testLexThrows("-A");
	testLexThrows(`-""`);

	testLex("7;", [
		Token(symbol!"Value",loc,Value(cast(int)7)),
		Token(symbol!"EOL",loc),
	]);

	// Floats
	testLex("1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
	testLex("1.2f" , [ Token(symbol!"Value",loc,Value(cast( float)1.2)) ]);
	testLex("1.2"  , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
	testLex("1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
	testLex("1.2d" , [ Token(symbol!"Value",loc,Value(cast(double)1.2)) ]);
	testLex("1.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
	testLex("1.2bd", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
	testLex("1.2Bd", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);
	testLex("1.2bD", [ Token(symbol!"Value",loc,Value(cast(  real)1.2)) ]);

	testLex(".2F" , [ Token(symbol!"Value",loc,Value(cast( float)0.2)) ]);
	testLex(".2"  , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
	testLex(".2D" , [ Token(symbol!"Value",loc,Value(cast(double)0.2)) ]);
	testLex(".2BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.2)) ]);

	testLex("-1.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-1.2)) ]);
	testLex("-1.2"  , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
	testLex("-1.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-1.2)) ]);
	testLex("-1.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)-1.2)) ]);

	testLex("-.2F" , [ Token(symbol!"Value",loc,Value(cast( float)-0.2)) ]);
	testLex("-.2"  , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
	testLex("-.2D" , [ Token(symbol!"Value",loc,Value(cast(double)-0.2)) ]);
	testLex("-.2BD", [ Token(symbol!"Value",loc,Value(cast(  real)-0.2)) ]);

	testLex( "0.0"  , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
	testLex( "0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
	testLex( "0.0BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
	testLex("-0.0"  , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
	testLex("-0.0F" , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
	testLex("-0.0BD", [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
	testLex( "7F"   , [ Token(symbol!"Value",loc,Value(cast( float)7.0)) ]);
	testLex( "7D"   , [ Token(symbol!"Value",loc,Value(cast(double)7.0)) ]);
	testLex( "7BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)7.0)) ]);
	testLex( "0F"   , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
	testLex( "0D"   , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
	testLex( "0BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);
	testLex("-0F"   , [ Token(symbol!"Value",loc,Value(cast( float)0.0)) ]);
	testLex("-0D"   , [ Token(symbol!"Value",loc,Value(cast(double)0.0)) ]);
	testLex("-0BD"  , [ Token(symbol!"Value",loc,Value(cast(  real)0.0)) ]);

	testLex("1.2 F", [
		Token(symbol!"Value",loc,Value(cast(double)1.2)),
		Token(symbol!"Ident",loc,Value(           null),"F"),
	]);
	testLexThrows("1.2A");
	testLexThrows("1.2B");
	testLexThrows("1.2BDF");

	testLex("1.2;", [
		Token(symbol!"Value",loc,Value(cast(double)1.2)),
		Token(symbol!"EOL",loc),
	]);

	testLex("1.2F;", [
		Token(symbol!"Value",loc,Value(cast(float)1.2)),
		Token(symbol!"EOL",loc),
	]);

	testLex("1.2BD;", [
		Token(symbol!"Value",loc,Value(cast(real)1.2)),
		Token(symbol!"EOL",loc),
	]);

	// Booleans and null
	testLex("true",   [ Token(symbol!"Value",loc,Value( true)) ]);
	testLex("false",  [ Token(symbol!"Value",loc,Value(false)) ]);
	testLex("on",     [ Token(symbol!"Value",loc,Value( true)) ]);
	testLex("off",    [ Token(symbol!"Value",loc,Value(false)) ]);
	testLex("null",   [ Token(symbol!"Value",loc,Value( null)) ]);

	testLex("TRUE",   [ Token(symbol!"Ident",loc,Value(null),"TRUE")  ]);
	testLex("true ",  [ Token(symbol!"Value",loc,Value(true)) ]);
	testLex("true  ", [ Token(symbol!"Value",loc,Value(true)) ]);
	testLex("tru",    [ Token(symbol!"Ident",loc,Value(null),"tru")   ]);
	testLex("truX",   [ Token(symbol!"Ident",loc,Value(null),"truX")  ]);
	testLex("trueX",  [ Token(symbol!"Ident",loc,Value(null),"trueX") ]);

	// Raw Backtick Strings
	testLex("`hello world`",      [ Token(symbol!"Value",loc,Value(`hello world`   )) ]);
	testLex("` hello world `",    [ Token(symbol!"Value",loc,Value(` hello world ` )) ]);
	testLex("`hello \\t world`",  [ Token(symbol!"Value",loc,Value(`hello \t world`)) ]);
	testLex("`hello \\n world`",  [ Token(symbol!"Value",loc,Value(`hello \n world`)) ]);
	testLex("`hello \n world`",   [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
	testLex("`hello \r\n world`", [ Token(symbol!"Value",loc,Value("hello \r\n world")) ]);
	testLex("`hello \"world\"`",  [ Token(symbol!"Value",loc,Value(`hello "world"` )) ]);

	testLexThrows("`foo");
	testLexThrows("`");

	// Double-Quote Strings
	testLex(`"hello world"`,            [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
	testLex(`" hello world "`,          [ Token(symbol!"Value",loc,Value(" hello world " )) ]);
	testLex(`"hello \t world"`,         [ Token(symbol!"Value",loc,Value("hello \t world")) ]);
	testLex(`"hello \n world"`,         [ Token(symbol!"Value",loc,Value("hello \n world")) ]);
	testLex("\"hello \\\n world\"",     [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
	testLex("\"hello \\  \n world\"",   [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
	testLex("\"hello \\  \n\n world\"", [ Token(symbol!"Value",loc,Value("hello world"   )) ]);
	testLex(`"\"hello world\""`,        [ Token(symbol!"Value",loc,Value(`"hello world"` )) ]);
	testLex(`""`,                       [ Token(symbol!"Value",loc,Value(""              )) ]); // issue #34

	testLexThrows("\"hello \n world\"");
	testLexThrows(`"foo`);
	testLexThrows(`"`);

	// Characters
	testLex("'a'",   [ Token(symbol!"Value",loc,Value(cast(dchar) 'a')) ]);
	testLex("'\\n'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\n')) ]);
	testLex("'\\t'", [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
	testLex("'\t'",  [ Token(symbol!"Value",loc,Value(cast(dchar)'\t')) ]);
	testLex("'\\''", [ Token(symbol!"Value",loc,Value(cast(dchar)'\'')) ]);
	testLex(`'\\'`,  [ Token(symbol!"Value",loc,Value(cast(dchar)'\\')) ]);

	testLexThrows("'a");
	testLexThrows("'aa'");
	testLexThrows("''");
	testLexThrows("'\\\n'");
	testLexThrows("'\n'");
	testLexThrows(`'\`);
	testLexThrows(`'\'`);
	testLexThrows("'");

	// Unicode
	testLex("日本語",         [ Token(symbol!"Ident",loc,Value(null), "日本語") ]);
	testLex("`おはよう、日本。`", [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
	testLex(`"おはよう、日本。"`, [ Token(symbol!"Value",loc,Value(`おはよう、日本。`)) ]);
	testLex("'月'",           [ Token(symbol!"Value",loc,Value("月"d.dup[0]))   ]);

	// Base64 Binary
	testLex("[aGVsbG8gd29ybGQ=]",              [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
	testLex("[ aGVsbG8gd29ybGQ= ]",            [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);
	testLex("[\n aGVsbG8g \n \n d29ybGQ= \n]", [ Token(symbol!"Value",loc,Value(cast(ubyte[])"hello world".dup))]);

	testLexThrows("[aGVsbG8gd29ybGQ]"); // Ie: Not multiple of 4
	testLexThrows("[ aGVsbG8gd29ybGQ ]");

	// Date
	testLex( "1999/12/5", [ Token(symbol!"Value",loc,Value(Date( 1999, 12, 5))) ]);
	testLex( "2013/2/22", [ Token(symbol!"Value",loc,Value(Date( 2013, 2, 22))) ]);
	testLex("-2013/2/22", [ Token(symbol!"Value",loc,Value(Date(-2013, 2, 22))) ]);

	testLexThrows("7/");
	testLexThrows("2013/2/22a");
	testLexThrows("2013/2/22f");

	testLex("1999/12/5\n", [
		Token(symbol!"Value",loc,Value(Date(1999, 12, 5))),
		Token(symbol!"EOL",loc),
	]);

	// DateTime, no timezone
	testLex( "2013/2/22 07:53",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22 \t 07:53",     [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22/*foo*/07:53",  [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22 /*foo*/ \\\n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22 /*foo*/ \\\n\n  \n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22 /*foo*/ \\\n\\\n  \\\n  /*bar*/ 07:53", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22/*foo*/\\\n/*bar*/07:53",      [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0)))) ]);
	testLex("-2013/2/22 07:53",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 7, 53,  0)))) ]);
	testLex( "2013/2/22 -07:53",       [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53)))) ]);
	testLex("-2013/2/22 -07:53",       [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53)))) ]);
	testLex( "2013/2/22 07:53:34",     [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34)))) ]);
	testLex( "2013/2/22 07:53:34.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs))) ]);
	testLex( "2013/2/22 07:53:34.12",  [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 120.msecs))) ]);
	testLex( "2013/2/22 07:53:34.1",   [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53, 34), 100.msecs))) ]);
	testLex( "2013/2/22 07:53.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs))) ]);

	testLex( "2013/2/22 34:65",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0)))) ]);
	testLex( "2013/2/22 34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds(77), 123.msecs))) ]);
	testLex( "2013/2/22 34:65.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) + hours(34) + minutes(65) + seconds( 0), 123.msecs))) ]);

	testLex( "2013/2/22 -34:65",        [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0)))) ]);
	testLex( "2013/2/22 -34:65:77.123", [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds(77), -123.msecs))) ]);
	testLex( "2013/2/22 -34:65.123",    [ Token(symbol!"Value",loc,Value(DateTimeFrac(DateTime( 2013, 2, 22, 0, 0, 0) - hours(34) - minutes(65) - seconds( 0), -123.msecs))) ]);

	testLexThrows("2013/2/22 07:53a");
	testLexThrows("2013/2/22 07:53f");
	testLexThrows("2013/2/22 07:53:34.123a");
	testLexThrows("2013/2/22 07:53:34.123f");
	testLexThrows("2013/2/22a 07:53");

	testLex(`2013/2/22 "foo"`, [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value("foo")),
	]);

	testLex("2013/2/22 07", [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value(cast(int)7)),
	]);

	testLex("2013/2/22 1.2F", [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value(cast(float)1.2)),
	]);

	testLex("2013/2/22 .2F", [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value(cast(float)0.2)),
	]);

	testLex("2013/2/22 -1.2F", [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value(cast(float)-1.2)),
	]);

	testLex("2013/2/22 -.2F", [
		Token(symbol!"Value",loc,Value(Date(2013, 2, 22))),
		Token(symbol!"Value",loc,Value(cast(float)-0.2)),
	]);

	// DateTime, with known timezone
	testLex( "2013/2/22 07:53-GMT+00:00",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex("-2013/2/22 07:53-GMT+00:00",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex( "2013/2/22 -07:53-GMT+00:00",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex("-2013/2/22 -07:53-GMT+00:00",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex( "2013/2/22 07:53-GMT+02:10",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
	testLex( "2013/2/22 07:53-GMT-05:30",        [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
	testLex( "2013/2/22 07:53:34-GMT+00:00",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex( "2013/2/22 07:53:34-GMT+02:10",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
	testLex( "2013/2/22 07:53:34-GMT-05:30",     [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
	testLex( "2013/2/22 07:53:34.123-GMT+00:00", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex( "2013/2/22 07:53:34.123-GMT+02:10", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
	testLex( "2013/2/22 07:53:34.123-GMT-05:30", [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);
	testLex( "2013/2/22 07:53.123-GMT+00:00",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone( hours(0)            )))) ]);
	testLex( "2013/2/22 07:53.123-GMT+02:10",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone( hours(2)+minutes(10))))) ]);
	testLex( "2013/2/22 07:53.123-GMT-05:30",    [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);

	testLex( "2013/2/22 -34:65-GMT-05:30",       [ Token(symbol!"Value",loc,Value(SysTime(DateTime( 2013, 2, 22, 0,  0,  0) - hours(34) - minutes(65) - seconds( 0), new immutable SimpleTimeZone(-hours(5)-minutes(30))))) ]);

	// DateTime, with Java SDLang's occasionally weird interpretation of some
	// "not quite ISO" variations of the "GMT with offset" timezone strings.
	Token testTokenSimpleTimeZone(Duration d)
	{
		auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
		auto tz = new immutable SimpleTimeZone(d);
		return Token( symbol!"Value", loc, Value(SysTime(dateTime,tz)) );
	}
	Token testTokenUnknownTimeZone(string tzName)
	{
		auto dateTime = DateTime(2013, 2, 22, 7, 53, 0);
		auto frac = 0.msecs;
		return Token( symbol!"Value", loc, Value(DateTimeFracUnknownZone(dateTime,frac,tzName)) );
	}
	testLex("2013/2/22 07:53-GMT+",          [ testTokenUnknownTimeZone("GMT+")     ]);
	testLex("2013/2/22 07:53-GMT+:",         [ testTokenUnknownTimeZone("GMT+:")    ]);
	testLex("2013/2/22 07:53-GMT+:3",        [ testTokenUnknownTimeZone("GMT+:3")   ]);
	testLex("2013/2/22 07:53-GMT+:03",       [ testTokenSimpleTimeZone(minutes(3))  ]);
	testLex("2013/2/22 07:53-GMT+:003",      [ testTokenUnknownTimeZone("GMT+:003") ]);

	testLex("2013/2/22 07:53-GMT+4",         [ testTokenSimpleTimeZone(hours(4))            ]);
	testLex("2013/2/22 07:53-GMT+4:",        [ testTokenUnknownTimeZone("GMT+4:")           ]);
	testLex("2013/2/22 07:53-GMT+4:3",       [ testTokenUnknownTimeZone("GMT+4:3")          ]);
	testLex("2013/2/22 07:53-GMT+4:03",      [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
	testLex("2013/2/22 07:53-GMT+4:003",     [ testTokenUnknownTimeZone("GMT+4:003")        ]);

	testLex("2013/2/22 07:53-GMT+04",        [ testTokenSimpleTimeZone(hours(4))            ]);
	testLex("2013/2/22 07:53-GMT+04:",       [ testTokenUnknownTimeZone("GMT+04:")          ]);
	testLex("2013/2/22 07:53-GMT+04:3",      [ testTokenUnknownTimeZone("GMT+04:3")         ]);
	testLex("2013/2/22 07:53-GMT+04:03",     [ testTokenSimpleTimeZone(hours(4)+minutes(3)) ]);
	testLex("2013/2/22 07:53-GMT+04:03abc",  [ testTokenUnknownTimeZone("GMT+04:03abc")     ]);
	testLex("2013/2/22 07:53-GMT+04:003",    [ testTokenUnknownTimeZone("GMT+04:003")       ]);

	testLex("2013/2/22 07:53-GMT+004",       [ testTokenSimpleTimeZone(minutes(4))     ]);
	testLex("2013/2/22 07:53-GMT+004:",      [ testTokenUnknownTimeZone("GMT+004:")    ]);
	testLex("2013/2/22 07:53-GMT+004:3",     [ testTokenUnknownTimeZone("GMT+004:3")   ]);
	testLex("2013/2/22 07:53-GMT+004:03",    [ testTokenUnknownTimeZone("GMT+004:03")  ]);
	testLex("2013/2/22 07:53-GMT+004:003",   [ testTokenUnknownTimeZone("GMT+004:003") ]);

	testLex("2013/2/22 07:53-GMT+0004",      [ testTokenSimpleTimeZone(minutes(4))      ]);
	testLex("2013/2/22 07:53-GMT+0004:",     [ testTokenUnknownTimeZone("GMT+0004:")    ]);
	testLex("2013/2/22 07:53-GMT+0004:3",    [ testTokenUnknownTimeZone("GMT+0004:3")   ]);
	testLex("2013/2/22 07:53-GMT+0004:03",   [ testTokenUnknownTimeZone("GMT+0004:03")  ]);
	testLex("2013/2/22 07:53-GMT+0004:003",  [ testTokenUnknownTimeZone("GMT+0004:003") ]);

	testLex("2013/2/22 07:53-GMT+00004",     [ testTokenSimpleTimeZone(minutes(4))       ]);
	testLex("2013/2/22 07:53-GMT+00004:",    [ testTokenUnknownTimeZone("GMT+00004:")    ]);
	testLex("2013/2/22 07:53-GMT+00004:3",   [ testTokenUnknownTimeZone("GMT+00004:3")   ]);
	testLex("2013/2/22 07:53-GMT+00004:03",  [ testTokenUnknownTimeZone("GMT+00004:03")  ]);
	testLex("2013/2/22 07:53-GMT+00004:003", [ testTokenUnknownTimeZone("GMT+00004:003") ]);

	// DateTime, with unknown timezone
	testLex( "2013/2/22 07:53-Bogus/Foo",        [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53,  0),   0.msecs, "Bogus/Foo")), "2013/2/22 07:53-Bogus/Foo") ]);
	testLex("-2013/2/22 07:53-Bogus/Foo",        [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 7, 53,  0),   0.msecs, "Bogus/Foo"))) ]);
	testLex( "2013/2/22 -07:53-Bogus/Foo",       [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
	testLex("-2013/2/22 -07:53-Bogus/Foo",       [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime(-2013, 2, 22, 0,  0,  0) - hours(7) - minutes(53), 0.msecs, "Bogus/Foo"))) ]);
	testLex( "2013/2/22 07:53:34-Bogus/Foo",     [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34),   0.msecs, "Bogus/Foo"))) ]);
	testLex( "2013/2/22 07:53:34.123-Bogus/Foo", [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53, 34), 123.msecs, "Bogus/Foo"))) ]);
	testLex( "2013/2/22 07:53.123-Bogus/Foo",    [ Token(symbol!"Value",loc,Value(DateTimeFracUnknownZone(DateTime( 2013, 2, 22, 7, 53,  0), 123.msecs, "Bogus/Foo"))) ]);

	// Time Span
	testLex( "12:14:42",         [ Token(symbol!"Value",loc,Value( days( 0)+hours(12)+minutes(14)+seconds(42)+msecs(  0))) ]);
	testLex("-12:14:42",         [ Token(symbol!"Value",loc,Value(-days( 0)-hours(12)-minutes(14)-seconds(42)-msecs(  0))) ]);
	testLex( "00:09:12",         [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 9)+seconds(12)+msecs(  0))) ]);
	testLex( "00:00:01.023",     [ Token(symbol!"Value",loc,Value( days( 0)+hours( 0)+minutes( 0)+seconds( 1)+msecs( 23))) ]);
	testLex( "23d:05:21:23.532", [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(532))) ]);
	testLex( "23d:05:21:23.53",  [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(530))) ]);
	testLex( "23d:05:21:23.5",   [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(500))) ]);
	testLex("-23d:05:21:23.532", [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(532))) ]);
	testLex("-23d:05:21:23.5",   [ Token(symbol!"Value",loc,Value(-days(23)-hours( 5)-minutes(21)-seconds(23)-msecs(500))) ]);
	testLex( "23d:05:21:23",     [ Token(symbol!"Value",loc,Value( days(23)+hours( 5)+minutes(21)+seconds(23)+msecs(  0))) ]);

	testLexThrows("12:14:42a");
	testLexThrows("23d:05:21:23.532a");
	testLexThrows("23d:05:21:23.532f");

	// Combination
	testLex("foo. 7", [
		Token(symbol!"Ident",loc,Value(      null),"foo."),
		Token(symbol!"Value",loc,Value(cast(int)7))
	]);

	testLex(`
		namespace:person "foo" "bar" 1 23L name.first="ひとみ" name.last="Smith" {
			namespace:age 37; namespace:favorite_color "blue" // comment
			somedate 2013/2/22  07:53 -- comment

			inventory /* comment */ {
				socks
			}
		}
	`,
	[
		Token(symbol!"EOL",loc,Value(null),"\n"),

		Token(symbol!"Ident", loc, Value(         null ), "namespace"),
		Token(symbol!":",     loc, Value(         null ), ":"),
		Token(symbol!"Ident", loc, Value(         null ), "person"),
		Token(symbol!"Value", loc, Value(        "foo" ), `"foo"`),
		Token(symbol!"Value", loc, Value(        "bar" ), `"bar"`),
		Token(symbol!"Value", loc, Value( cast( int) 1 ), "1"),
		Token(symbol!"Value", loc, Value( cast(long)23 ), "23L"),
		Token(symbol!"Ident", loc, Value(         null ), "name.first"),
		Token(symbol!"=",     loc, Value(         null ), "="),
		Token(symbol!"Value", loc, Value(       "ひとみ" ), `"ひとみ"`),
		Token(symbol!"Ident", loc, Value(         null ), "name.last"),
		Token(symbol!"=",     loc, Value(         null ), "="),
		Token(symbol!"Value", loc, Value(      "Smith" ), `"Smith"`),
		Token(symbol!"{",     loc, Value(         null ), "{"),
		Token(symbol!"EOL",   loc, Value(         null ), "\n"),

		Token(symbol!"Ident", loc, Value(        null ), "namespace"),
		Token(symbol!":",     loc, Value(        null ), ":"),
		Token(symbol!"Ident", loc, Value(        null ), "age"),
		Token(symbol!"Value", loc, Value( cast(int)37 ), "37"),
		Token(symbol!"EOL",   loc, Value(        null ), ";"),
		Token(symbol!"Ident", loc, Value(        null ), "namespace"),
		Token(symbol!":",     loc, Value(        null ), ":"),
		Token(symbol!"Ident", loc, Value(        null ), "favorite_color"),
		Token(symbol!"Value", loc, Value(      "blue" ), `"blue"`),
		Token(symbol!"EOL",   loc, Value(        null ), "\n"),

		Token(symbol!"Ident", loc, Value( null ), "somedate"),
		Token(symbol!"Value", loc, Value( DateTimeFrac(DateTime(2013, 2, 22, 7, 53, 0)) ), "2013/2/22  07:53"),
		Token(symbol!"EOL",   loc, Value( null ), "\n"),
		Token(symbol!"EOL",   loc, Value( null ), "\n"),

		Token(symbol!"Ident", loc, Value(null), "inventory"),
		Token(symbol!"{",     loc, Value(null), "{"),
		Token(symbol!"EOL",   loc, Value(null), "\n"),

		Token(symbol!"Ident", loc, Value(null), "socks"),
		Token(symbol!"EOL",   loc, Value(null), "\n"),

		Token(symbol!"}",     loc, Value(null), "}"),
		Token(symbol!"EOL",   loc, Value(null), "\n"),

		Token(symbol!"}",     loc, Value(null), "}"),
		Token(symbol!"EOL",   loc, Value(null), "\n"),
	]);

	if(numErrors > 0)
		stderr.writeln(numErrors, " failed test(s)");
}

@("lexer: Regression test issue #8")
unittest
{
	testLex(`"\n \n"`, [ Token(symbol!"Value",loc,Value("\n \n"),`"\n \n"`) ]);
	testLex(`"\t\t"`, [ Token(symbol!"Value",loc,Value("\t\t"),`"\t\t"`) ]);
	testLex(`"\n\n"`, [ Token(symbol!"Value",loc,Value("\n\n"),`"\n\n"`) ]);
}

@("lexer: Regression test issue #11")
unittest
{
	void test(string input)
	{
		testLex(
			input,
			[
				Token(symbol!"EOL", loc, Value(null), "\n"),
				Token(symbol!"Ident",loc,Value(null), "a")
			]
		);
	}

	test("//X\na");
	test("//\na");
	test("--\na");
	test("#\na");
}

@("ast: Regression test issue #28")
unittest
{
	enum offset = 1; // workaround for an of-by-one error for line numbers
	testLex("test", [
		Token(symbol!"Ident", Location("filename", 0, 0, 0), Value(null), "test")
	], true);
	testLex("\ntest", [
		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\n"),
		Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
	], true);
	testLex("\rtest", [
		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
		Token(symbol!"Ident", Location("filename", 1, 0, 1), Value(null), "test")
	], true);
	testLex("\r\ntest", [
		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
		Token(symbol!"Ident", Location("filename", 1, 0, 2), Value(null), "test")
	], true);
	testLex("\r\n\ntest", [
		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r\n"),
		Token(symbol!"EOL", Location("filename", 1, 0, 2), Value(null), "\n"),
		Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
	], true);
	testLex("\r\r\ntest", [
		Token(symbol!"EOL", Location("filename", 0, 0, 0), Value(null), "\r"),
		Token(symbol!"EOL", Location("filename", 1, 0, 1), Value(null), "\r\n"),
		Token(symbol!"Ident", Location("filename", 2, 0, 3), Value(null), "test")
	], true);
}