shithub: rgbds

Info  •  Files  •  Log  •  Branches

Implement infrastructure around new lexer

The lexer itself is very much incomplete, but this is intended to be a
safe point to revert to should further implementation go south.

--- a/Makefile

+++ b/Makefile

@@ -56,7 +56,6 @@

 	src/asm/asmy.o \

 	src/asm/charmap.o \

 	src/asm/fstack.o \

-	src/asm/globlex.o \

 	src/asm/lexer.o \

 	src/asm/macro.o \

 	src/asm/main.o \

@@ -73,7 +72,7 @@

 	src/hashmap.o \

 	src/linkdefs.o

-src/asm/globlex.o src/asm/lexer.o src/asm/constexpr.o: src/asm/asmy.h

+src/asm/lexer.o: src/asm/asmy.h

 rgblink_obj := \

 	src/link/assign.o \

--- a/include/asm/asm.h

+++ b/include/asm/asm.h

@@ -24,11 +24,8 @@

 #define MAXMACROARGS	99999

 #define MAXINCPATHS	128

-extern int32_t nLineNo;

 extern uint32_t nTotalLines;

 extern uint32_t nIFDepth;

-extern bool skipElif;

-extern char tzCurrentFileName[_MAX_PATH + 1];

 extern struct Section *pCurrentSection;

 extern bool oDontExpandStrings;

--- a/include/asm/fstack.h

+++ b/include/asm/fstack.h

@@ -24,7 +24,7 @@

 struct MacroArgs;

 struct sContext {

-	YY_BUFFER_STATE FlexHandle;

+	struct LexerState *lexerState;

 	struct Symbol const *pMacro;

 	struct sContext *next;

 	char tzFileName[_MAX_PATH + 1];

@@ -32,7 +32,6 @@

 	uint32_t uniqueID;

 	int32_t nLine;

 	uint32_t nStatus;

-	FILE *pFile;

 	char *pREPTBlock;

 	uint32_t nREPTBlockCount;

 	uint32_t nREPTBlockSize;

@@ -46,11 +45,17 @@

 void fstk_Init(char *s);

 void fstk_Dump(void);

 void fstk_DumpToStr(char *buf, size_t len);

-void fstk_DumpStringExpansions(void);

 void fstk_AddIncludePath(char *s);

 void fstk_RunMacro(char *s, struct MacroArgs *args);

 void fstk_RunRept(uint32_t count, int32_t nReptLineNo);

-FILE *fstk_FindFile(char const *fname, char **incPathUsed);

+/**

+ * @param path The user-provided file name

+ * @param fullPath The address of a pointer, which will be made to point at the full path

+ *                 The pointer's value must be a valid argument to `realloc`, including NULL

+ * @param size Current size of the buffer, or 0 if the pointer is NULL

+ * @return True if the file was found, false if no path worked

+ */

+bool fstk_FindFile(char const *path, char **fullPath, size_t *size);

 int32_t fstk_GetLine(void);

 #endif /* RGBDS_ASM_FSTACK_H */

--- a/include/asm/lexer.h

+++ b/include/asm/lexer.h

@@ -9,78 +9,44 @@

 #ifndef RGBDS_ASM_LEXER_H

 #define RGBDS_ASM_LEXER_H

-#include <stdint.h>

-#include <stdio.h>

-

-#define LEXHASHSIZE	(1 << 11)

 #define MAXSTRLEN	255

-struct sLexInitString {

-	char *tzName;

-	uint32_t nToken;

-};

+struct LexerState;

+extern struct LexerState *lexerState;

+extern struct LexerState *lexerStateEOL;

-struct sLexFloat {

-	uint32_t (*Callback)(char *s, uint32_t size);

-	uint32_t nToken;

-};

+static inline struct LexerState *lexer_GetState(void)

+{

+	return lexerState;

+}

-struct yy_buffer_state {

-	/* Actual starting address */

-	char *pBufferRealStart;

-	/* Address where the data is initially written after a safety margin */

-	char *pBufferStart;

-	char *pBuffer;

-	size_t nBufferSize;

-	uint32_t oAtLineStart;

-};

+static inline void lexer_SetState(struct LexerState *state)

+{

+	lexerState = state;

+}

-enum eLexerState {

-	LEX_STATE_NORMAL,

-	LEX_STATE_MACROARGS

-};

+static inline void lexer_SetStateAtEOL(struct LexerState *state)

+{

+	lexerStateEOL = state;

+}

-struct sStringExpansionPos {

-	char *tzName;

-	char *pBuffer;

-	char *pBufferPos;

-	struct sStringExpansionPos *pParent;

+struct LexerState *lexer_OpenFile(char const *path);

+struct LexerState *lexer_OpenFileView(void);

+void lexer_DeleteState(struct LexerState *state);

+

+enum LexerMode {

+	LEXER_NORMAL,

+	LEXER_RAW

 };

-#define INITIAL		0

-#define macroarg	3

+void lexer_SetMode(enum LexerMode mode);

+void lexer_ToggleStringExpansion(bool enable);

-typedef struct yy_buffer_state *YY_BUFFER_STATE;

-

-void setup_lexer(void);

-

-void yy_set_state(enum eLexerState i);

-YY_BUFFER_STATE yy_create_buffer(FILE *f);

-YY_BUFFER_STATE yy_scan_bytes(char const *mem, uint32_t size);

-void yy_delete_buffer(YY_BUFFER_STATE buf);

-void yy_switch_to_buffer(YY_BUFFER_STATE buf);

-uint32_t lex_FloatAlloc(const struct sLexFloat *tok);

-void lex_FloatAddRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_FloatDeleteRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_FloatAddFirstRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_FloatDeleteFirstRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_FloatAddSecondRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_FloatDeleteSecondRange(uint32_t id, uint16_t start, uint16_t end);

-void lex_Init(void);

-void lex_AddStrings(const struct sLexInitString *lex);

-void lex_SetBuffer(char *buffer, uint32_t len);

-void lex_BeginStringExpansion(const char *tzName);

-int yywrap(void);

+char const *lexer_GetFileName(void);

+unsigned int lexer_GetLineNo(void);

+void lexer_DumpStringExpansions(void);

 int yylex(void);

-void yyunput(char c);

-void yyunputstr(const char *s);

-void yyskipbytes(uint32_t count);

-void yyunputbytes(uint32_t count);

-

-extern YY_BUFFER_STATE pCurrentBuffer;

-extern struct sStringExpansionPos *pCurrentStringExpansion;

-

-void upperstring(char *s);

-void lowerstring(char *s);

+void lexer_SkipToBlockEnd(int blockStartToken, int blockEndToken, int endToken,

+			  char **capture, size_t *size, char const *name);

 #endif /* RGBDS_ASM_LEXER_H */

--- a/include/asm/main.h

+++ b/include/asm/main.h

@@ -43,6 +43,10 @@

 void opt_Pop(void);

 void opt_Parse(char *s);

+void upperstring(char *s);

+void lowerstring(char *s);

+

+/* TODO: are these really needed? */

 #define YY_FATAL_ERROR fatalerror

 #ifdef YYLMAX

--- a/src/CMakeLists.txt

+++ b/src/CMakeLists.txt

@@ -31,7 +31,6 @@

     "${BISON_ASMy_OUTPUT_SOURCE}"

     "asm/charmap.c"

     "asm/fstack.c"

-    "asm/globlex.c"

     "asm/lexer.c"

     "asm/macro.c"

     "asm/main.c"

--- a/src/asm/asmy.y

+++ b/src/asm/asmy.y

@@ -39,6 +39,7 @@

 char *tzNewMacro;

 uint32_t ulNewMacroSize;

 int32_t nPCOffset;

+bool skipElifs; /* If this is set, ELIFs cannot be executed anymore */

 size_t symvaluetostring(char *dest, size_t maxLength, char *symName,

 			const char *mode)

@@ -111,278 +112,6 @@

 	return r;

 }

-static uint32_t isWhiteSpace(char s)

-{

-	return (s == ' ') || (s == '\t') || (s == '\0') || (s == '\n');

-}

-

-static uint32_t isRept(char *s)

-{

-	return (strncasecmp(s, "REPT", 4) == 0)

-		&& isWhiteSpace(*(s - 1)) && isWhiteSpace(s[4]);

-}

-

-static uint32_t isEndr(char *s)

-{

-	return (strncasecmp(s, "ENDR", 4) == 0)

-		&& isWhiteSpace(*(s - 1)) && isWhiteSpace(s[4]);

-}

-

-static void copyrept(void)

-{

-	int32_t level = 1, len, instring = 0;

-	char *src = pCurrentBuffer->pBuffer;

-	char *bufferEnd = pCurrentBuffer->pBufferStart

-			+ pCurrentBuffer->nBufferSize;

-

-	while (src < bufferEnd && level) {

-		if (instring == 0) {

-			if (isRept(src)) {

-				level++;

-				src += 4;

-			} else if (isEndr(src)) {

-				level--;

-				src += 4;

-			} else {

-				if (*src == '\"')

-					instring = 1;

-				src++;

-			}

-		} else {

-			if (*src == '\\') {

-				src += 2;

-			} else if (*src == '\"') {

-				src++;

-				instring = 0;

-			} else {

-				src++;

-			}

-		}

-	}

-

-	if (level != 0)

-		fatalerror("Unterminated REPT block\n");

-

-	len = src - pCurrentBuffer->pBuffer - 4;

-

-	src = pCurrentBuffer->pBuffer;

-	ulNewMacroSize = len;

-

-	tzNewMacro = malloc(ulNewMacroSize + 1);

-

-	if (tzNewMacro == NULL)

-		fatalerror("Not enough memory for REPT block.\n");

-

-	uint32_t i;

-

-	tzNewMacro[ulNewMacroSize] = 0;

-	for (i = 0; i < ulNewMacroSize; i++) {

-		tzNewMacro[i] = src[i];

-		if (src[i] == '\n')

-			nLineNo++;

-	}

-

-	yyskipbytes(ulNewMacroSize + 4);

-

-}

-

-static uint32_t isMacro(char *s)

-{

-	return (strncasecmp(s, "MACRO", 4) == 0)

-		&& isWhiteSpace(*(s - 1)) && isWhiteSpace(s[5]);

-}

-

-static uint32_t isEndm(char *s)

-{

-	return (strncasecmp(s, "ENDM", 4) == 0)

-		&& isWhiteSpace(*(s - 1)) && isWhiteSpace(s[4]);

-}

-

-static void copymacro(void)

-{

-	int32_t level = 1, len, instring = 0;

-	char *src = pCurrentBuffer->pBuffer;

-	char *bufferEnd = pCurrentBuffer->pBufferStart

-			+ pCurrentBuffer->nBufferSize;

-

-	while (src < bufferEnd && level) {

-		if (instring == 0) {

-			if (isMacro(src)) {

-				level++;

-				src += 4;

-			} else if (isEndm(src)) {

-				level--;

-				src += 4;

-			} else {

-				if(*src == '\"')

-					instring = 1;

-				src++;

-			}

-		} else {

-			if (*src == '\\') {

-				src += 2;

-			} else if (*src == '\"') {

-				src++;

-				instring = 0;

-			} else {

-				src++;

-			}

-		}

-	}

-

-	if (level != 0)

-		fatalerror("Unterminated MACRO definition.\n");

-

-	len = src - pCurrentBuffer->pBuffer - 4;

-

-	src = pCurrentBuffer->pBuffer;

-	ulNewMacroSize = len;

-

-	tzNewMacro = (char *)malloc(ulNewMacroSize + 1);

-	if (tzNewMacro == NULL)

-		fatalerror("Not enough memory for MACRO definition.\n");

-

-	uint32_t i;

-

-	tzNewMacro[ulNewMacroSize] = 0;

-	for (i = 0; i < ulNewMacroSize; i++) {

-		tzNewMacro[i] = src[i];

-		if (src[i] == '\n')

-			nLineNo++;

-	}

-

-	yyskipbytes(ulNewMacroSize + 4);

-}

-

-static bool endsIf(char c)

-{

-	return isWhiteSpace(c) || c == '(' || c == '{';

-}

-

-static uint32_t isIf(char *s)

-{

-	return (strncasecmp(s, "IF", 2) == 0)

-		&& isWhiteSpace(s[-1]) && endsIf(s[2]);

-}

-

-static uint32_t isElif(char *s)

-{

-	return (strncasecmp(s, "ELIF", 4) == 0)

-		&& isWhiteSpace(s[-1]) && endsIf(s[4]);

-}

-

-static uint32_t isElse(char *s)

-{

-	return (strncasecmp(s, "ELSE", 4) == 0)

-		&& isWhiteSpace(s[-1]) && isWhiteSpace(s[4]);

-}

-

-static uint32_t isEndc(char *s)

-{

-	return (strncasecmp(s, "ENDC", 4) == 0)

-		&& isWhiteSpace(s[-1]) && isWhiteSpace(s[4]);

-}

-

-static void if_skip_to_else(void)

-{

-	int32_t level = 1;

-	bool inString = false;

-	char *src = pCurrentBuffer->pBuffer;

-

-	while (*src && level) {

-		if (*src == '\n')

-			nLineNo++;

-

-		if (!inString) {

-			if (isIf(src)) {

-				level++;

-				src += 2;

-

-			} else if (level == 1 && isElif(src)) {

-				level--;

-				skipElif = false;

-

-			} else if (level == 1 && isElse(src)) {

-				level--;

-				src += 4;

-

-			} else if (isEndc(src)) {

-				level--;

-				if (level != 0)

-					src += 4;

-

-			} else {

-				if (*src == '\"')

-					inString = true;

-				src++;

-			}

-		} else {

-			if (*src == '\"') {

-				inString = false;

-			} else if (*src == '\\') {

-				/* Escaped quotes don't end the string */

-				if (*++src != '\"')

-					src--;

-			}

-			src++;

-		}

-	}

-

-	if (level != 0)

-		fatalerror("Unterminated IF construct\n");

-

-	int32_t len = src - pCurrentBuffer->pBuffer;

-

-	yyskipbytes(len);

-	yyunput('\n');

-	nLineNo--;

-}

-

-static void if_skip_to_endc(void)

-{

-	int32_t level = 1;

-	bool inString = false;

-	char *src = pCurrentBuffer->pBuffer;

-

-	while (*src && level) {

-		if (*src == '\n')

-			nLineNo++;

-

-		if (!inString) {

-			if (isIf(src)) {

-				level++;

-				src += 2;

-			} else if (isEndc(src)) {

-				level--;

-				if (level != 0)

-					src += 4;

-			} else {

-				if (*src == '\"')

-					inString = true;

-				src++;

-			}

-		} else {

-			if (*src == '\"') {

-				inString = false;

-			} else if (*src == '\\') {

-				/* Escaped quotes don't end the string */

-				if (*++src != '\"')

-					src--;

-			}

-			src++;

-		}

-	}

-

-	if (level != 0)

-		fatalerror("Unterminated IF construct\n");

-

-	int32_t len = src - pCurrentBuffer->pBuffer;

-

-	yyskipbytes(len);

-	yyunput('\n');

-	nLineNo--;

-}

-

 static size_t strlenUTF8(const char *s)

 {

 	size_t len = 0;

@@ -660,7 +389,6 @@

 			nListCountEmpty = 0;

 			nPCOffset = 0;

 		} line '\n' {

-			nLineNo++;

 			nTotalLines++;

 		}

 ;

@@ -699,9 +427,9 @@

 ;

 macro		: T_ID {

-			yy_set_state(LEX_STATE_MACROARGS);

+			lexer_SetMode(LEXER_RAW);

 		} macroargs {

-			yy_set_state(LEX_STATE_NORMAL);

+			lexer_SetMode(LEXER_NORMAL);

 			fstk_RunMacro($1, $3);

 		}

 ;

@@ -786,9 +514,9 @@

 ;

 opt		: T_POP_OPT {

-			yy_set_state(LEX_STATE_MACROARGS);

+			lexer_SetMode(LEXER_RAW);

 		} opt_list {

-			yy_set_state(LEX_STATE_NORMAL);

+			lexer_SetMode(LEXER_NORMAL);

 		}

 ;

@@ -875,15 +603,21 @@

 ;

 rept		: T_POP_REPT uconst {

-			uint32_t nDefinitionLineNo = nLineNo;

-			copyrept();

+			uint32_t nDefinitionLineNo = lexer_GetLineNo();

+			char *body;

+			size_t size;

+			lexer_SkipToBlockEnd(T_POP_REPT, T_POP_ENDR, T_POP_ENDR,

+					     &body, &size, "REPT block");

 			fstk_RunRept($2, nDefinitionLineNo);

 		}

 ;

 macrodef	: T_LABEL ':' T_POP_MACRO {

-			int32_t nDefinitionLineNo = nLineNo;

-			copymacro();

+			int32_t nDefinitionLineNo = lexer_GetLineNo();

+			char *body;

+			size_t size;

+			lexer_SkipToBlockEnd(T_POP_MACRO, T_POP_ENDM, T_POP_ENDM,

+					     &body, &size, "macro definition");

 			sym_AddMacro($1, nDefinitionLineNo);

 		}

 ;

@@ -956,9 +690,9 @@

 ;

 purge		: T_POP_PURGE {

-			oDontExpandStrings = true;

+			lexer_ToggleStringExpansion(false);

 		} purge_list {

-			oDontExpandStrings = false;

+			lexer_ToggleStringExpansion(true);

 		}

 ;

@@ -1054,8 +788,14 @@

 if		: T_POP_IF const {

 			nIFDepth++;

-			if (!$2)

-				if_skip_to_else();

+			if (!$2) {

+				/* The function is hardcoded to also stop on T_POP_ELSE and ENDC */

+				lexer_SkipToBlockEnd(T_POP_IF, T_POP_ENDC, T_POP_ELIF,

+						     NULL, NULL, "if block");

+				skipElifs = false;

+			} else {

+				skipElifs = true;

+			}

 		}

 ;

@@ -1063,7 +803,7 @@

 			if (nIFDepth <= 0)

 				fatalerror("Found ELIF outside an IF construct\n");

-			if (skipElif) {

+			if (skipElifs) {

 				/*

 				 * Executed when ELIF is reached at the end of

 				 * an IF or ELIF block for which the condition

@@ -1071,7 +811,8 @@

 				 *

 				 * Continue parsing at ENDC keyword

 				 */

-				if_skip_to_endc();

+				lexer_SkipToBlockEnd(T_POP_IF, T_POP_ENDC, T_POP_ENDC,

+						     NULL, NULL, "elif block");

 			} else {

 				/*

 				 * Executed when ELIF is skipped to because the

@@ -1078,7 +819,6 @@

 				 * condition of the previous IF or ELIF block

 				 * was false.

 				 */

-				skipElif = true;

 				if (!$2) {

 					/*

@@ -1085,7 +825,10 @@

 					 * Continue parsing after ELSE, or at

 					 * ELIF or ENDC keyword.

 					 */

-					if_skip_to_else();

+					lexer_SkipToBlockEnd(T_POP_IF, T_POP_ENDC, T_POP_ELIF,

+							     NULL, NULL, "elif block");

+				} else {

+					skipElifs = true;

 				}

 			}

 		}

@@ -1096,7 +839,8 @@

 				fatalerror("Found ELSE outside an IF construct\n");

 			/* Continue parsing at ENDC keyword */

-			if_skip_to_endc();

+			lexer_SkipToBlockEnd(T_POP_IF, T_POP_ENDC, T_POP_ENDC,

+					     NULL, NULL, "else block");

 		}

 ;

@@ -1267,13 +1011,13 @@

 		}

 		| T_OP_BANK '(' string ')'	{ rpn_BankSection(&$$, $3); }

 		| T_OP_DEF {

-			oDontExpandStrings = true;

+			lexer_ToggleStringExpansion(false);

 		} '(' scoped_id ')' {

 			struct Symbol const *sym = sym_FindSymbol($4);

 			rpn_Number(&$$, !!sym);

-			oDontExpandStrings = false;

+			lexer_ToggleStringExpansion(true);

 		}

 		| T_OP_ROUND '(' const ')' {

 			rpn_Number(&$$, math_Round($3));

--- a/src/asm/fstack.c

+++ b/src/asm/fstack.c

@@ -36,10 +36,7 @@

 static unsigned int nFileStackDepth;

 unsigned int nMaxRecursionDepth;

 static struct Symbol const *pCurrentMacro;

-static YY_BUFFER_STATE CurrentFlexHandle;

-static FILE *pCurrentFile;

 static uint32_t nCurrentStatus;

-char tzCurrentFileName[_MAX_PATH + 1];

 static char IncludePaths[MAXINCPATHS][_MAX_PATH + 1];

 static int32_t NextIncPath;

 static uint32_t nMacroCount;

@@ -81,10 +78,8 @@

 	if (*ppFileStack == NULL)

 		fatalerror("No memory for context\n");

-	(*ppFileStack)->FlexHandle = CurrentFlexHandle;

 	(*ppFileStack)->next = NULL;

-	strcpy((char *)(*ppFileStack)->tzFileName, (char *)tzCurrentFileName);

-	(*ppFileStack)->nLine = nLineNo;

+	(*ppFileStack)->nLine = lexer_GetLineNo();

 	switch ((*ppFileStack)->nStatus = nCurrentStatus) {

 	case STAT_isMacroArg:

@@ -93,7 +88,6 @@

 		(*ppFileStack)->pMacro = pCurrentMacro;

 		break;

 	case STAT_isInclude:

-		(*ppFileStack)->pFile = pCurrentFile;

 		break;

 	case STAT_isREPTBlock:

 		(*ppFileStack)->macroArgs = macro_GetCurrentArgs();

@@ -107,8 +101,6 @@

 		fatalerror("%s: Internal error.\n", __func__);

 	}

 	(*ppFileStack)->uniqueID = macro_GetUniqueID();

-

-	nLineNo = 0;

 }

 static int32_t popcontext(void)

@@ -122,20 +114,15 @@

 			int nNbCharsWritten;

 			int nNbCharsLeft;

-			yy_delete_buffer(CurrentFlexHandle);

-			CurrentFlexHandle =

-				yy_scan_bytes(pCurrentREPTBlock,

-					      nCurrentREPTBlockSize);

-			yy_switch_to_buffer(CurrentFlexHandle);

 			macro_SetUniqueID(nMacroCount++);

 			/* Increment REPT count in file path */

 			pREPTIterationWritePtr =

-				strrchr(tzCurrentFileName, '~') + 1;

+				strrchr(lexer_GetFileName(), '~') + 1;

 			nREPTIterationNo =

 				strtoul(pREPTIterationWritePtr, NULL, 10);

-			nNbCharsLeft = sizeof(tzCurrentFileName)

-				- (pREPTIterationWritePtr - tzCurrentFileName);

+			nNbCharsLeft = sizeof(lexer_GetFileName())

+				- (pREPTIterationWritePtr - lexer_GetFileName());

 			nNbCharsWritten = snprintf(pREPTIterationWritePtr,

 						   nNbCharsLeft, "%lu",

 						   nREPTIterationNo + 1);

@@ -150,7 +137,6 @@

 				fatalerror("Cannot write REPT count to file path\n");

 			}

-			nLineNo = nCurrentREPTBodyFirstLine;

 			return 0;

 		}

 	}

@@ -165,20 +151,9 @@

 		pLastFile = *ppLastFile;

 	}

-	yy_delete_buffer(CurrentFlexHandle);

-	nLineNo = nCurrentStatus == STAT_isREPTBlock ? nCurrentREPTBodyLastLine

-						     : pLastFile->nLine;

+	lexer_DeleteState(lexer_GetState());

+	lexer_SetState(pLastFile->lexerState);

-	if (nCurrentStatus == STAT_isInclude)

-		fclose(pCurrentFile);

-

-	if (nCurrentStatus == STAT_isMacro

-	 || nCurrentStatus == STAT_isREPTBlock)

-		nLineNo++;

-

-	CurrentFlexHandle = pLastFile->FlexHandle;

-	strcpy((char *)tzCurrentFileName, (char *)pLastFile->tzFileName);

-

 	switch (pLastFile->nStatus) {

 		struct MacroArgs *args;

@@ -193,7 +168,6 @@

 		pCurrentMacro = pLastFile->pMacro;

 		break;

 	case STAT_isInclude:

-		pCurrentFile = pLastFile->pFile;

 		break;

 	case STAT_isREPTBlock:

 		args = macro_GetCurrentArgs();

@@ -218,7 +192,6 @@

 	free(*ppLastFile);

 	*ppLastFile = NULL;

-	yy_switch_to_buffer(CurrentFlexHandle);

 	return 0;

 }

@@ -229,11 +202,11 @@

 	switch (nCurrentStatus) {

 	case STAT_isInclude:

 		/* This is the normal mode, also used when including a file. */

-		return nLineNo;

+		return lexer_GetLineNo();

 	case STAT_isMacro:

 		break; /* Peek top file of the stack */

 	case STAT_isMacroArg:

-		return nLineNo; /* ??? */

+		return lexer_GetLineNo(); /* ??? */

 	case STAT_isREPTBlock:

 		break; /* Peek top file of the stack */

 	default:

@@ -277,7 +250,7 @@

 		pLastFile = pLastFile->next;

 	}

-	fprintf(stderr, "%s(%" PRId32 ")", tzCurrentFileName, nLineNo);

+	fprintf(stderr, "%s(%" PRId32 ")", lexer_GetFileName(), lexer_GetLineNo());

 }

 void fstk_DumpToStr(char *buf, size_t buflen)

@@ -299,7 +272,7 @@

 	}

 	retcode = snprintf(&buf[buflen - len], len, "%s(%" PRId32 ")",

-			   tzCurrentFileName, nLineNo);

+			   lexer_GetFileName(), lexer_GetLineNo());

 	if (retcode < 0)

 		fatalerror("Failed to dump file stack to string: %s\n", strerror(errno));

 	else if (retcode >= len)

@@ -312,20 +285,6 @@

 }

 /*

- * Dump the string expansion stack to stderr

- */

-void fstk_DumpStringExpansions(void)

-{

-	const struct sStringExpansionPos *pExpansion = pCurrentStringExpansion;

-

-	while (pExpansion) {

-		fprintf(stderr, "while expanding symbol \"%s\"\n",

-			pExpansion->tzName);

-		pExpansion = pExpansion->pParent;

-	}

-}

-

-/*

  * Extra includepath stuff

  */

 void fstk_AddIncludePath(char *s)

@@ -351,63 +310,58 @@

 	}

 }

-static FILE *getFile(char const *pathname)

+static bool isPathValid(char const *pathname)

 {

 	struct stat statbuf;

 	if (stat(pathname, &statbuf) != 0)

-		return NULL;

+		return false;

 	/* Reject directories */

-	if (S_ISDIR(statbuf.st_mode))

-		return NULL;

-

-	return fopen(pathname, "rb");

+	return !S_ISDIR(statbuf.st_mode);

 }

-FILE *fstk_FindFile(char const *fname, char **incPathUsed)

+bool fstk_FindFile(char const *path, char **fullPath, size_t *size)

 {

-	if (fname == NULL)

-		return NULL;

-

-	char path[_MAX_PATH];

-	FILE *f = getFile(fname);

-

-	if (f) {

-		printdep(fname);

-		return f;

+	if (!*size) {

+		*size = 64; /* This is arbitrary, really */

+		*fullPath = realloc(*fullPath, *size);

+		if (!*fullPath)

+			error("realloc error during include path search: %s\n",

+			      strerror(errno));

 	}

-	for (size_t i = 0; i < NextIncPath; ++i) {

-		/*

-		 * The function snprintf() does not write more than `size` bytes

-		 * (including the terminating null byte ('\0')).  If the output

-		 * was truncated due to this limit, the return value is the

-		 * number of characters (excluding the terminating null byte)

-		 * which would have been written to the final string if enough

-		 * space had been available. Thus, a return value of `size` or

-		 * more means that the output was truncated.

-		 */

-		int fullpathlen = snprintf(path, sizeof(path), "%s%s",

-					   IncludePaths[i], fname);

+	if (*fullPath) {

+		for (size_t i = 0; i <= NextIncPath; ++i) {

+			char *incPath = i ? IncludePaths[i - 1] : "";

+			int len = snprintf(*fullPath, *size, "%s%s", incPath, path);

-		if (fullpathlen >= (int)sizeof(path))

-			continue;

+			/* Oh how I wish `asnprintf` was standard... */

+			if (len >= *size) { /* `len` doesn't include the terminator, `size` does */

+				*size = len + 1;

+				*fullPath = realloc(*fullPath, *size);

+				if (!*fullPath) {

+					error("realloc error during include path search: %s\n",

+					      strerror(errno));

+					break;

+				}

+				len = sprintf(*fullPath, "%s%s", incPath, path);

+			}

-		f = getFile(path);

-		if (f) {

-			printdep(path);

-

-			if (incPathUsed)

-				*incPathUsed = IncludePaths[i];

-			return f;

+			if (len < 0) {

+				error("snprintf error during include path search: %s\n",

+				      strerror(errno));

+			} else if (isPathValid(*fullPath)) {

+				printdep(*fullPath);

+				return true;

+			}

 		}

 	}

 	errno = ENOENT;

 	if (oGeneratedMissingIncludes)

-		printdep(fname);

-	return NULL;

+		printdep(path);

+	return false;

 }

 /*

@@ -415,33 +369,31 @@

  */

 void fstk_RunInclude(char *tzFileName)

 {

-	char *incPathUsed = "";

-	FILE *f = fstk_FindFile(tzFileName, &incPathUsed);

+	char *fullPath = NULL;

+	size_t size = 0;

-	if (f == NULL) {

-		if (oGeneratedMissingIncludes) {

+	if (!fstk_FindFile(tzFileName, &fullPath, &size)) {

+		if (oGeneratedMissingIncludes)

 			oFailedOnMissingInclude = true;

-			return;

-		}

-		error("Unable to open included file '%s': %s\n", tzFileName, strerror(errno));

+		else

+			error("Unable to open included file '%s': %s\n",

+			      tzFileName, strerror(errno));

+		free(fullPath);

 		return;

 	}

 	pushcontext();

-	nLineNo = 1;

 	nCurrentStatus = STAT_isInclude;

-	snprintf(tzCurrentFileName, sizeof(tzCurrentFileName), "%s%s",

-		 incPathUsed, tzFileName);

 	if (verbose)

-		printf("Assembling %s\n", tzCurrentFileName);

-	pCurrentFile = f;

-	CurrentFlexHandle = yy_create_buffer(pCurrentFile);

-	yy_switch_to_buffer(CurrentFlexHandle);

+		printf("Assembling %s\n", fullPath);

-	/* Dirty hack to give the INCLUDE directive a linefeed */

+	struct LexerState *state = lexer_OpenFile(fullPath);

-	yyunput('\n');

-	nLineNo--;

+	if (!state)

+		/* If lexer had an error, it already reported it */

+		fatalerror("Failed to open file for INCLUDE\n"); /* TODO: make this non-fatal? */

+	lexer_SetStateAtEOL(state);

+	free(fullPath);

 }

 /*

@@ -450,7 +402,6 @@

 void fstk_RunMacro(char *s, struct MacroArgs *args)

 {

 	struct Symbol const *sym = sym_FindSymbol(s);

-	int nPrintedChars;

 	if (sym == NULL) {

 		error("Macro \"%s\" not defined\n", s);

@@ -464,21 +415,10 @@

 	pushcontext();

 	macro_SetUniqueID(nMacroCount++);

 	/* Minus 1 because there is a newline at the beginning of the buffer */

-	nLineNo = sym->fileLine - 1;

 	macro_UseNewArgs(args);

 	nCurrentStatus = STAT_isMacro;

-	nPrintedChars = snprintf(tzCurrentFileName, _MAX_PATH + 1,

-				 "%s::%s", sym->fileName, s);

-	if (nPrintedChars > _MAX_PATH) {

-		popcontext();

-		fatalerror("File name + macro name is too large to fit into buffer\n");

-	}

 	pCurrentMacro = sym;

-	/* TODO: why is `strlen` being used when there's a macro size field? */

-	CurrentFlexHandle = yy_scan_bytes(pCurrentMacro->macro,

-					  strlen(pCurrentMacro->macro));

-	yy_switch_to_buffer(CurrentFlexHandle);

 }

 /*

@@ -487,11 +427,6 @@

 void fstk_RunRept(uint32_t count, int32_t nReptLineNo)

 {

 	if (count) {

-		static const char *tzReptStr = "::REPT~1";

-

-		/* For error printing to make sense, fake nLineNo */

-		nCurrentREPTBodyLastLine = nLineNo;

-		nLineNo = nReptLineNo;

 		pushcontext();

 		macro_SetUniqueID(nMacroCount++);

 		nCurrentREPTBlockCount = count;

@@ -499,15 +434,6 @@

 		nCurrentREPTBlockSize = ulNewMacroSize;

 		pCurrentREPTBlock = tzNewMacro;

 		nCurrentREPTBodyFirstLine = nReptLineNo + 1;

-		nLineNo = nReptLineNo;

-

-		if (strlen(tzCurrentFileName) + strlen(tzReptStr) > _MAX_PATH)

-			fatalerror("Cannot append \"%s\" to file path\n", tzReptStr);

-		strcat(tzCurrentFileName, tzReptStr);

-

-		CurrentFlexHandle =

-			yy_scan_bytes(pCurrentREPTBlock, nCurrentREPTBlockSize);

-		yy_switch_to_buffer(CurrentFlexHandle);

 	}

 }

@@ -526,7 +452,6 @@

 	// minus 2 to account for trailing "\"\0"

 	// minus 1 to avoid a buffer overflow in extreme cases

 	while (*c && fileNameIndex < sizeof(tzSymFileName) - 2 - 1) {

-

 		if (*c == '"') {

 			tzSymFileName[fileNameIndex++] = '\\';

 		}

@@ -541,19 +466,8 @@

 	sym_AddString("__FILE__", tzSymFileName);

 	pFileStack = NULL;

-	if (strcmp(pFileName, "-") == 0) {

-		pCurrentFile = stdin;

-	} else {

-		pCurrentFile = fopen(pFileName, "rb");

-		if (pCurrentFile == NULL)

-			fatalerror("Unable to open file '%s': %s\n", pFileName, strerror(errno));

-	}

 	nFileStackDepth = 0;

 	nMacroCount = 0;

 	nCurrentStatus = STAT_isInclude;

-	snprintf(tzCurrentFileName, _MAX_PATH + 1, "%s", pFileName);

-	CurrentFlexHandle = yy_create_buffer(pCurrentFile);

-	yy_switch_to_buffer(CurrentFlexHandle);

-	nLineNo = 1;

 }

--- a/src/asm/globlex.c

+++ b/src/asm/globlex.c

@@ -287,10 +287,10 @@

 			/* Feed the symbol's contents into the buffer */

 			yyunputstr(s = sym_GetStringValue(sym));

-			/* Lines inserted this way shall not increase nLineNo */

+			/* Lines inserted this way shall not increase lexer_GetLineNo() */

 			while (*s) {

 				if (*s++ == '\n')

-					nLineNo--;

+					lexer_GetLineNo()--;

 			}

 			return 0;

 		}

--- a/src/asm/lexer.c

+++ b/src/asm/lexer.c

@@ -1,1054 +1,364 @@

 /*

  * This file is part of RGBDS.

  *

- * Copyright (c) 1997-2019, Carsten Sorensen and RGBDS contributors.

+ * Copyright (c) 2020, Eldred Habert and RGBDS contributors.

  *

  * SPDX-License-Identifier: MIT

  */

+#include <sys/mman.h>

+#include <sys/stat.h>

 #include <assert.h>

-#include <ctype.h>

-#include <inttypes.h>

-#include <stdio.h>

+#include <errno.h>

+#include <fcntl.h>

+#include <limits.h>

+#include <stdbool.h>

 #include <stdint.h>

+#include <stdio.h>

 #include <stdlib.h>

 #include <string.h>

+#include <unistd.h>

-#include "asm/asm.h"

-#include "asm/fstack.h"

 #include "asm/lexer.h"

-#include "asm/macro.h"

-#include "asm/main.h"

 #include "asm/rpn.h"

-#include "asm/section.h"

+#include "asm/symbol.h" /* For MAXSYMLEN in asmy.h */

 #include "asm/warning.h"

+/* Include this last so it gets all type & constant definitions */

+#include "asmy.h" /* For token definitions, generated from asmy.y */

-#include "extern/err.h"

+#define LEXER_BUF_SIZE 42 /* TODO: determine a sane value for this */

+/* This caps the size of buffer reads, and according to POSIX, passing more than SSIZE_MAX is UB */

+static_assert(LEXER_BUF_SIZE <= SSIZE_MAX);

-#include "asmy.h"

-#include "platform.h" // strncasecmp, strdup

+struct LexerState {

+	char const *path;

-struct sLexString {

-	char *tzName;

-	uint32_t nToken;

-	uint32_t nNameLength;

-	struct sLexString *next;

+	/* mmap()-dependent IO state */

+	bool isMmapped;

+	union {

+		struct { /* If mmap()ed */

+			char *ptr;

+			off_t size;

+			off_t offset;

+		};

+		struct { /* Otherwise */

+			int fd;

+			size_t index; /* Read index into the buffer */

+			size_t nbChars; /* Number of chars in front of the buffer */

+			char buf[LEXER_BUF_SIZE]; /* Circular buffer */

+		};

+	};

+

+	/* Common state */

+	enum LexerMode mode;

+	bool atLineStart;

+	unsigned int lineNo;

+	bool capturing; /* Whether the text being lexed should be captured */

+	size_t captureSize; /* Amount of text captured */

+	char *captureBuf; /* Buffer to send the captured text to if non-NULL */

+	size_t captureCapacity; /* Size of the buffer above */

+	bool expandStrings;

 };

-#define pLexBufferRealStart	(pCurrentBuffer->pBufferRealStart)

-#define pLexBuffer		(pCurrentBuffer->pBuffer)

-#define AtLineStart		(pCurrentBuffer->oAtLineStart)

+struct LexerState *lexerState = NULL;

+struct LexerState *lexerStateEOL = NULL;

-#define SAFETYMARGIN		1024

-

-#define BOM_SIZE 3

-

-struct sLexFloat tLexFloat[32];

-struct sLexString *tLexHash[LEXHASHSIZE];

-YY_BUFFER_STATE pCurrentBuffer;

-uint32_t nLexMaxLength; // max length of all keywords and operators

-

-uint32_t tFloatingSecondChar[256];

-uint32_t tFloatingFirstChar[256];

-uint32_t tFloatingChars[256];

-uint32_t nFloating;

-enum eLexerState lexerstate = LEX_STATE_NORMAL;

-

-struct sStringExpansionPos *pCurrentStringExpansion;

-static unsigned int nNbStringExpansions;

-

-/* UTF-8 byte order mark */

-static const unsigned char bom[BOM_SIZE] = { 0xEF, 0xBB, 0xBF };

-

-void upperstring(char *s)

+struct LexerState *lexer_OpenFile(char const *path)

 {

-	while (*s) {

-		*s = toupper(*s);

-		s++;

-	}

-}

+	bool isStdin = !strcmp(path, "-");

+	struct LexerState *state = malloc(sizeof(*state));

-void lowerstring(char *s)

-{

-	while (*s) {

-		*s = tolower(*s);

-		s++;

+	/* Give stdin a nicer file name */

+	if (isStdin)

+		path = "<stdin>";

+	if (!state) {

+		error("Failed to open file \"%s\": %s\n", path, strerror(errno));

+		return NULL;

 	}

-}

+	state->path = path;

-void yyskipbytes(uint32_t count)

-{

-	pLexBuffer += count;

-}

+	state->fd = isStdin ? STDIN_FILENO : open(path, O_RDONLY);

+	state->isMmapped = false; /* By default, assume it won't be mmap()ed */

+	off_t size = lseek(state->fd, 0, SEEK_END);

-void yyunputbytes(uint32_t count)

-{

-	pLexBuffer -= count;

-}

+	if (size != 1) {

+		/* The file is a regular file, so use `mmap` for better performance */

-void yyunput(char c)

-{

-	if (pLexBuffer <= pLexBufferRealStart)

-		fatalerror("Buffer safety margin exceeded\n");

+		/*

+		 * Important: do NOT assign to `state->ptr` directly, to avoid a cast that may

+		 * alter an eventual `MAP_FAILED` value. It would also invalidate `state->fd`,

+		 * being on the other side of the union.

+		 */

+		void *pa = mmap(NULL, size, PROT_READ, MAP_PRIVATE, state->fd, 0);

-	*(--pLexBuffer) = c;

-}

+		if (pa == MAP_FAILED && errno == ENOTSUP)

+			/*

+			 * The implementation may not support MAP_PRIVATE; try again with MAP_SHARED

+			 * instead, offering, I believe, weaker guarantees about external

+			 * modifications to the file while reading it. That's still better than not

+			 * opening it at all, though.

+			 */

+			pa = mmap(NULL, size, PROT_READ, MAP_SHARED, state->fd, 0);

-void yyunputstr(const char *s)

-{

-	int32_t len;

+		if (pa == MAP_FAILED) {

+			/* If mmap()ing failed, try again using another method (below) */

+			state->isMmapped = false;

+		} else {

+			/* IMPORTANT: the `union` mandates this is accessed before other members! */

+			close(state->fd);

-	len = strlen(s);

+			state->isMmapped = true;

+			state->ptr = pa;

+			state->size = size;

+		}

+	}

+	if (!state->isMmapped) {

+		/* Sometimes mmap() fails or isn't available, so have a fallback */

+		lseek(state->fd, 0, SEEK_SET);

+		state->index = 0;

+		state->nbChars = 0;

+	}

-	/*

-	 * It would be undefined behavior to subtract `len` from pLexBuffer and

-	 * potentially have it point outside of pLexBufferRealStart's buffer,

-	 * this is why the check is done this way.

-	 * Refer to https://github.com/rednex/rgbds/pull/411#discussion_r319779797

-	 */

-	if (pLexBuffer - pLexBufferRealStart < len)

-		fatalerror("Buffer safety margin exceeded\n");

-

-	pLexBuffer -= len;

-

-	memcpy(pLexBuffer, s, len);

+	state->mode = LEXER_NORMAL;

+	state->atLineStart = true;

+	state->lineNo = 0;

+	state->capturing = false;

+	state->captureBuf = NULL;

+	return state;

 }

-/*

- * Marks that a new string expansion with name `tzName` ends here

- * Enforces recursion depth

- */

-void lex_BeginStringExpansion(const char *tzName)

+struct LexerState *lexer_OpenFileView(void)

 {

-	if (++nNbStringExpansions > nMaxRecursionDepth)

-		fatalerror("Recursion limit (%u) exceeded\n", nMaxRecursionDepth);

-

-	struct sStringExpansionPos *pNewStringExpansion =

-		malloc(sizeof(*pNewStringExpansion));

-	char *tzNewExpansionName = strdup(tzName);

-

-	if (!pNewStringExpansion || !tzNewExpansionName)

-		fatalerror("Could not allocate memory to expand '%s'\n", tzName);

-

-	pNewStringExpansion->tzName = tzNewExpansionName;

-	pNewStringExpansion->pBuffer = pLexBufferRealStart;

-	pNewStringExpansion->pBufferPos = pLexBuffer;

-	pNewStringExpansion->pParent = pCurrentStringExpansion;

-

-	pCurrentStringExpansion = pNewStringExpansion;

+	return NULL;

 }

-void yy_switch_to_buffer(YY_BUFFER_STATE buf)

+void lexer_DeleteState(struct LexerState *state)

 {

-	pCurrentBuffer = buf;

+	if (state->isMmapped)

+		munmap(state->ptr, state->size);

+	else

+		close(state->fd);

+	free(state);

 }

-void yy_set_state(enum eLexerState i)

+void lexer_SetMode(enum LexerMode mode)

 {

-	lexerstate = i;

+	lexerState->mode = mode;

 }

-void yy_delete_buffer(YY_BUFFER_STATE buf)

+void lexer_ToggleStringExpansion(bool enable)

 {

-	free(buf->pBufferStart - SAFETYMARGIN);

-	free(buf);

+	lexerState->expandStrings = enable;

 }

-/*

- * Maintains the following invariants:

- * 1. nBufferSize < capacity

- * 2. The buffer is terminated with 0

- * 3. nBufferSize is the size without the terminator

- */

-static void yy_buffer_append(YY_BUFFER_STATE buf, size_t capacity, char c)

-{

-	assert(buf->pBufferStart[buf->nBufferSize] == 0);

-	assert(buf->nBufferSize + 1 < capacity);

+/* Functions for the actual lexer to obtain characters */

-	buf->pBufferStart[buf->nBufferSize++] = c;

-	buf->pBufferStart[buf->nBufferSize] = 0;

-}

-

-static void yy_buffer_append_newlines(YY_BUFFER_STATE buf, size_t capacity)

+static void reallocCaptureBuf(void)

 {

-	/* Add newline if file doesn't end with one */

-	if (buf->nBufferSize == 0

-	 || buf->pBufferStart[buf->nBufferSize - 1] != '\n')

-		yy_buffer_append(buf, capacity, '\n');

-

-	/* Add newline if \ will eat the last newline */

-	if (buf->nBufferSize >= 2) {

-		size_t pos = buf->nBufferSize - 2;

-

-		/* Skip spaces and tabs */

-		while (pos > 0 && (buf->pBufferStart[pos] == ' '

-				|| buf->pBufferStart[pos] == '\t'))

-			pos--;

-

-		if (buf->pBufferStart[pos] == '\\')

-			yy_buffer_append(buf, capacity, '\n');

-	}

+	lexerState->captureCapacity *= 2;

+	lexerState->captureBuf = realloc(lexerState->captureBuf, lexerState->captureCapacity);

+	if (!lexerState->captureBuf)

+		fatalerror("realloc error while resizing capture buffer: %s\n", strerror(errno));

 }

-YY_BUFFER_STATE yy_scan_bytes(char const *mem, uint32_t size)

+/* If at any point we need more than 255 characters of lookahead, something went VERY wrong. */

+static int peek(uint8_t distance)

 {

-	YY_BUFFER_STATE pBuffer = malloc(sizeof(struct yy_buffer_state));

-

-	if (pBuffer == NULL)

-		fatalerror("%s: Out of memory!\n", __func__);

-

-	size_t capacity = size + 3; /* space for 2 newlines and terminator */

-

-	pBuffer->pBufferRealStart = malloc(capacity + SAFETYMARGIN);

-

-	if (pBuffer->pBufferRealStart == NULL)

-		fatalerror("%s: Out of memory for buffer!\n", __func__);

-

-	pBuffer->pBufferStart = pBuffer->pBufferRealStart + SAFETYMARGIN;

-	pBuffer->pBuffer = pBuffer->pBufferRealStart + SAFETYMARGIN;

-	memcpy(pBuffer->pBuffer, mem, size);

-	pBuffer->pBuffer[size] = 0;

-	pBuffer->nBufferSize = size;

-	yy_buffer_append_newlines(pBuffer, capacity);

-	pBuffer->oAtLineStart = 1;

-

-	return pBuffer;

-}

-

-YY_BUFFER_STATE yy_create_buffer(FILE *f)

-{

-	YY_BUFFER_STATE pBuffer = malloc(sizeof(struct yy_buffer_state));

-

-	if (pBuffer == NULL)

-		fatalerror("%s: Out of memory!\n", __func__);

-

-	size_t size = 0, capacity = -1;

-	char *buf = NULL;

-

-	/*

-	 * Check if we can get the file size without implementation-defined

-	 * behavior:

-	 *

-	 * From ftell(3p):

-	 * [On error], ftell() and ftello() shall return −1, and set errno to

-	 * indicate the error.

-	 *

-	 * The ftell() and ftello() functions shall fail if: [...]

-	 * ESPIPE The file descriptor underlying stream is associated with a

-	 * pipe, FIFO, or socket.

-	 *

-	 * From fseek(3p):

-	 * The behavior of fseek() on devices which are incapable of seeking

-	 * is implementation-defined.

-	 */

-	if (ftell(f) != -1) {

-		fseek(f, 0, SEEK_END);

-		capacity = ftell(f);

-		rewind(f);

+	if (lexerState->isMmapped) {

+		if (lexerState->offset + distance >= lexerState->size)

+			return EOF;

+		return lexerState->ptr[lexerState->offset + distance];

 	}

-	// If ftell errored or the block above wasn't executed

-	if (capacity == -1)

-		capacity = 4096;

-	// Handle 0-byte files gracefully

-	else if (capacity == 0)

-		capacity = 1;

+	if (lexerState->nbChars <= distance) {

+		/* Buffer isn't full enough, read some chars in */

-	do {

-		if (buf == NULL || size >= capacity) {

-			if (buf)

-				capacity *= 2;

-			/* Give extra room for 2 newlines and terminator */

-			buf = realloc(buf, capacity + SAFETYMARGIN + 3);

+		/* Compute the index we'll start writing to */

+		size_t writeIndex = (lexerState->index + lexerState->nbChars) % LEXER_BUF_SIZE;

+		size_t target = LEXER_BUF_SIZE - lexerState->nbChars; /* Aim: making the buf full */

+		ssize_t nbCharsRead = 0;

-			if (buf == NULL)

-				fatalerror("%s: Out of memory for buffer!\n",

-					   __func__);

-		}

+#define readChars(size) do { \

+	nbCharsRead = read(lexerState->fd, &lexerState->buf[writeIndex], (size)); \

+	if (nbCharsRead == -1) \

+		fatalerror("Error while reading \"%s\": %s\n", lexerState->path, errno); \

+	writeIndex += nbCharsRead; \

+	if (writeIndex == LEXER_BUF_SIZE) \

+		writeIndex = 0; \

+	lexerState->nbChars += nbCharsRead; /* Count all those chars in */ \

+	target -= nbCharsRead; \

+} while (0)

-		char *bufpos = buf + SAFETYMARGIN + size;

-		size_t read_count = fread(bufpos, 1, capacity - size, f);

-

-		if (read_count == 0 && !feof(f))

-			fatalerror("%s: fread error\n", __func__);

-

-		size += read_count;

-	} while (!feof(f));

-

-	pBuffer->pBufferRealStart = buf;

-	pBuffer->pBufferStart = buf + SAFETYMARGIN;

-	pBuffer->pBuffer = buf + SAFETYMARGIN;

-	pBuffer->pBuffer[size] = 0;

-	pBuffer->nBufferSize = size;

-

-	/* This is added here to make the buffer scaling above easy to express,

-	 * while taking the newline space into account

-	 * for the yy_buffer_append_newlines() call below.

-	 */

-	capacity += 3;

-

-	/* Skip UTF-8 byte order mark. */

-	if (pBuffer->nBufferSize >= BOM_SIZE

-	 && !memcmp(pBuffer->pBuffer, bom, BOM_SIZE))

-		pBuffer->pBuffer += BOM_SIZE;

-

-	/* Convert all line endings to LF and spaces */

-

-	char *mem = pBuffer->pBuffer;

-	int32_t lineCount = 0;

-

-	while (*mem) {

-		if ((mem[0] == '\\') && (mem[1] == '\"' || mem[1] == '\\')) {

-			mem += 2;

-		} else {

-			/* LF CR and CR LF */

-			if (((mem[0] == '\n') && (mem[1] == '\r'))

-			 || ((mem[0] == '\r') && (mem[1] == '\n'))) {

-				*mem++ = ' ';

-				*mem++ = '\n';

-				lineCount++;

-			/* LF and CR */

-			} else if ((mem[0] == '\n') || (mem[0] == '\r')) {

-				*mem++ = '\n';

-				lineCount++;

-			} else {

-				mem++;

-			}

+		/* If the range to fill passes over the buffer wrapping point, we need two reads */

+		if (writeIndex + target > LEXER_BUF_SIZE) {

+			readChars(LEXER_BUF_SIZE - writeIndex);

+			/* If the read was incomplete, don't perform a second read */

+			if (nbCharsRead < LEXER_BUF_SIZE - writeIndex)

+				target = 0;

 		}

-	}

+		if (target != 0)

+			readChars(target);

-	if (mem != pBuffer->pBuffer + size) {

-		nLineNo = lineCount + 1;

-		fatalerror("Found null character\n");

-	}

+#undef readChars

-	/* Remove comments */

-

-	mem = pBuffer->pBuffer;

-	bool instring = false;

-

-	while (*mem) {

-		if (*mem == '\"')

-			instring = !instring;

-

-		if ((mem[0] == '\\') && (mem[1] == '\"' || mem[1] == '\\')) {

-			mem += 2;

-		} else if (instring) {

-			mem++;

-		} else {

-			/* Comments that start with ; anywhere in a line */

-			if (*mem == ';') {

-				while (!((*mem == '\n') || (*mem == '\0')))

-					*mem++ = ' ';

-			/* Comments that start with * at the start of a line */

-			} else if ((mem[0] == '\n') && (mem[1] == '*')) {

-				warning(WARNING_OBSOLETE,

-					"'*' is deprecated for comments, please use ';' instead\n");

-				mem++;

-				while (!((*mem == '\n') || (*mem == '\0')))

-					*mem++ = ' ';

-			} else {

-				mem++;

-			}

-		}

+		/* If there aren't enough chars even after refilling, give up */

+		if (lexerState->nbChars <= distance)

+			return EOF;

 	}

-

-	yy_buffer_append_newlines(pBuffer, capacity);

-	pBuffer->oAtLineStart = 1;

-	return pBuffer;

+	return lexerState->buf[(lexerState->index + distance) % LEXER_BUF_SIZE];

 }

-uint32_t lex_FloatAlloc(const struct sLexFloat *token)

+static void shiftChars(uint8_t distance)

 {

-	tLexFloat[nFloating] = *token;

-

-	return (1 << (nFloating++));

-}

-

-/*

- * Make sure that only non-zero ASCII characters are used. Also, check if the

- * start is greater than the end of the range.

- */

-bool lex_CheckCharacterRange(uint16_t start, uint16_t end)

-{

-	if (start > end || start < 1 || end > 127) {

-		error("Invalid character range (start: %" PRIu16 ", end: %" PRIu16 ")\n",

-			start, end);

-		return false;

-	}

-	return true;

-}

-

-void lex_FloatDeleteRange(uint32_t id, uint16_t start, uint16_t end)

-{

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingChars[start] &= ~id;

-			start++;

+	if (lexerState->capturing) {

+		if (lexerState->captureBuf) {

+			if (lexerState->captureSize + distance >= lexerState->captureCapacity)

+				reallocCaptureBuf();

+			/* TODO: improve this? */

+			for (uint8_t i = 0; i < distance; i++)

+				lexerState->captureBuf[lexerState->captureSize++] = peek(i);

+		} else {

+			lexerState->captureSize += distance;

 		}

 	}

-}

-void lex_FloatAddRange(uint32_t id, uint16_t start, uint16_t end)

-{

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingChars[start] |= id;

-			start++;

-		}

+	if (lexerState->isMmapped) {

+		lexerState->offset += distance;

+	} else {

+		lexerState->nbChars -= distance;

+		lexerState->index += distance;

+		/* Wrap around if necessary */

+		if (lexerState->index >= LEXER_BUF_SIZE)

+			lexerState->index %= LEXER_BUF_SIZE;

 	}

 }

-void lex_FloatDeleteFirstRange(uint32_t id, uint16_t start, uint16_t end)

+static int nextChar(void)

 {

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingFirstChar[start] &= ~id;

-			start++;

-		}

-	}

-}

+	int c = peek(0);

-void lex_FloatAddFirstRange(uint32_t id, uint16_t start, uint16_t end)

-{

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingFirstChar[start] |= id;

-			start++;

-		}

-	}

+	/* If not at EOF, advance read position */

+	if (c != EOF)

+		shiftChars(1);

+	return c;

 }

-void lex_FloatDeleteSecondRange(uint32_t id, uint16_t start, uint16_t end)

-{

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingSecondChar[start] &= ~id;

-			start++;

-		}

-	}

-}

+/* "Services" provided by the lexer to the rest of the program */

-void lex_FloatAddSecondRange(uint32_t id, uint16_t start, uint16_t end)

+char const *lexer_GetFileName(void)

 {

-	if (lex_CheckCharacterRange(start, end)) {

-		while (start <= end) {

-			tFloatingSecondChar[start] |= id;

-			start++;

-		}

-	}

+	return lexerState->path;

 }

-static struct sLexFloat *lexgetfloat(uint32_t nFloatMask)

+unsigned int lexer_GetLineNo(void)

 {

-	if (nFloatMask == 0)

-		fatalerror("Internal error in %s\n", __func__);

-

-	int32_t i = 0;

-

-	while ((nFloatMask & 1) == 0) {

-		nFloatMask >>= 1;

-		i++;

-	}

-

-	return &tLexFloat[i];

+	return lexerState->lineNo;

 }

-static uint32_t lexcalchash(char *s)

+void lexer_DumpStringExpansions(void)

 {

-	uint32_t hash = 0;

-

-	while (*s)

-		hash = (hash * 283) ^ toupper(*s++);

-

-	return hash % LEXHASHSIZE;

+	/* TODO */

 }

-void lex_Init(void)

+static int yylex_NORMAL(void)

 {

-	uint32_t i;

+	for (;;) {

+		int c = nextChar();

-	for (i = 0; i < LEXHASHSIZE; i++)

-		tLexHash[i] = NULL;

+		switch (c) {

+		case '\n':

+			if (lexerStateEOL) {

+				lexer_SetState(lexerStateEOL);

+				lexerStateEOL = NULL;

+			}

+			return '\n';

-	for (i = 0; i < 256; i++) {

-		tFloatingFirstChar[i] = 0;

-		tFloatingSecondChar[i] = 0;

-		tFloatingChars[i] = 0;

-	}

+		/* Ignore whitespace */

+		case ' ':

+		case '\t':

+			break;

-	nLexMaxLength = 0;

-	nFloating = 0;

+		case EOF:

+			/* Captures end at their buffer's boundary no matter what */

+			if (!lexerState->capturing) {

+				/* TODO: use `yywrap()` */

+			}

+			return 0;

-	pCurrentStringExpansion = NULL;

-	nNbStringExpansions = 0;

-}

-

-void lex_AddStrings(const struct sLexInitString *lex)

-{

-	while (lex->tzName) {

-		struct sLexString **ppHash;

-		uint32_t hash = lexcalchash(lex->tzName);

-

-		ppHash = &tLexHash[hash];

-		while (*ppHash)

-			ppHash = &((*ppHash)->next);

-

-		*ppHash = malloc(sizeof(struct sLexString));

-		if (*ppHash == NULL)

-			fatalerror("Out of memory!\n");

-

-		(*ppHash)->tzName = (char *)strdup(lex->tzName);

-		if ((*ppHash)->tzName == NULL)

-			fatalerror("Out of memory!\n");

-

-		(*ppHash)->nNameLength = strlen(lex->tzName);

-		(*ppHash)->nToken = lex->nToken;

-		(*ppHash)->next = NULL;

-

-		upperstring((*ppHash)->tzName);

-

-		if ((*ppHash)->nNameLength > nLexMaxLength)

-			nLexMaxLength = (*ppHash)->nNameLength;

-

-		lex++;

-	}

-}

-

-/*

- * Gets the "float" mask and "float" length.

- * "Float" refers to the token type of a token that is not a keyword.

- * The character classes floatingFirstChar, floatingSecondChar, and

- * floatingChars are defined separately for each token type.

- * It uses bit masks to match against a set of simple regular expressions

- * of the form /[floatingFirstChar]([floatingSecondChar][floatingChars]*)?/.

- * The token types with the longest match from the current position in the

- * buffer will have their bits set in the float mask.

- */

-void yylex_GetFloatMaskAndFloatLen(uint32_t *pnFloatMask, uint32_t *pnFloatLen)

-{

-	/*

-	 * Note that '\0' should always have a bit mask of 0 in the "floating"

-	 * tables, so it doesn't need to be checked for separately.

-	 */

-

-	char *s = pLexBuffer;

-	uint32_t nOldFloatMask = 0;

-	uint32_t nFloatMask = tFloatingFirstChar[(uint8_t)*s];

-

-	if (nFloatMask != 0) {

-		s++;

-		nOldFloatMask = nFloatMask;

-		nFloatMask &= tFloatingSecondChar[(uint8_t)*s];

-

-		while (nFloatMask != 0) {

-			s++;

-			nOldFloatMask = nFloatMask;

-			nFloatMask &= tFloatingChars[(uint8_t)*s];

+		default:

+			error("Unknown character '%c'\n");

 		}

 	}

-

-	*pnFloatMask = nOldFloatMask;

-	*pnFloatLen = (uint32_t)(s - pLexBuffer);

 }

-/*

- * Gets the longest keyword/operator from the current position in the buffer.

- */

-struct sLexString *yylex_GetLongestFixed(void)

+static int yylex_RAW(void)

 {

-	struct sLexString *pLongestFixed = NULL;

-	char *s = pLexBuffer;

-	uint32_t hash = 0;

-	uint32_t length = 0;

-

-	while (length < nLexMaxLength && *s) {

-		hash = (hash * 283) ^ toupper(*s);

-		s++;

-		length++;

-

-		struct sLexString *lex = tLexHash[hash % LEXHASHSIZE];

-

-		while (lex) {

-			if (lex->nNameLength == length

-			 && strncasecmp(pLexBuffer, lex->tzName, length) == 0) {

-				pLongestFixed = lex;

-				break;

-			}

-			lex = lex->next;

-		}

-	}

-

-	return pLongestFixed;

+	fatalerror("LEXER_RAW not yet implemented\n");

 }

-size_t CopyMacroArg(char *dest, size_t maxLength, char c)

+int yylex(void)

 {

-	size_t i;

-	char const *s;

+	if (lexerState->atLineStart)

+		lexerState->lineNo++;

-	if (c == '@')

-		s = macro_GetUniqueIDStr();

-	else if (c >= '1' && c <= '9')

-		s = macro_GetArg(c - '0');

-	else

-		return 0;

+	static int (* const lexerModeFuncs[])(void) = {

+		[LEXER_NORMAL] = yylex_NORMAL,

+		[LEXER_RAW]    = yylex_RAW,

+	};

+	int token = lexerModeFuncs[lexerState->mode]();

-	if (s == NULL)

-		fatalerror("Macro argument '\\%c' not defined\n", c);

+	if (token == '\n')

+		lexerState->atLineStart = true;

+	else if (lexerState->atLineStart)

+		lexerState->atLineStart = false;

-	// TODO: `strncpy`, nay?

-	for (i = 0; s[i] != 0; i++) {

-		if (i >= maxLength)

-			fatalerror("Macro argument too long to fit buffer\n");

-

-		dest[i] = s[i];

-	}

-

-	return i;

+	return token;

 }

-static inline void yylex_StringWriteChar(char *s, size_t index, char c)

+void lexer_SkipToBlockEnd(int blockStartToken, int blockEndToken, int endToken,

+			  char **capture, size_t *size, char const *name)

 {

-	if (index >= MAXSTRLEN)

-		fatalerror("String too long\n");

+	lexerState->capturing = true;

+	lexerState->captureSize = 0;

+	unsigned int level = 0;

+	char *captureStart;

-	s[index] = c;

-}

-

-static inline void yylex_SymbolWriteChar(char *s, size_t index, char c)

-{

-	if (index >= MAXSYMLEN)

-		fatalerror("Symbol too long\n");

-

-	s[index] = c;

-}

-

-/*

- * Trims white space at the end of a string.

- * The index parameter is the index of the 0 at the end of the string.

- */

-void yylex_TrimEnd(char *s, size_t index)

-{

-	int32_t i = (int32_t)index - 1;

-

-	while ((i >= 0) && (s[i] == ' ' || s[i] == '\t')) {

-		s[i] = 0;

-		i--;

-	}

-}

-

-size_t yylex_ReadBracketedSymbol(char *dest, size_t index)

-{

-	char sym[MAXSYMLEN + 1];

-	char ch;

-	size_t i = 0;

-	size_t length, maxLength;

-	const char *mode = NULL;

-

-	for (ch = *pLexBuffer;

-	     ch != '}' && ch != '"' && ch != '\n';

-		 ch = *(++pLexBuffer)) {

-		if (ch == '\\') {

-			ch = *(++pLexBuffer);

-			maxLength = MAXSYMLEN - i;

-			length = CopyMacroArg(&sym[i], maxLength, ch);

-

-			if (length != 0)

-				i += length;

-			else

-				fatalerror("Illegal character escape '%c'\n", ch);

-		} else if (ch == '{') {

-			/* Handle nested symbols */

-			++pLexBuffer;

-			i += yylex_ReadBracketedSymbol(sym, i);

-			--pLexBuffer;

-		} else if (ch == ':' && !mode) { /* Only grab 1st colon */

-			/* Use a whitelist of modes, which does prevent the

-			 * use of some features such as precision,

-			 * but also avoids a security flaw

-			 */

-			const char *acceptedModes = "bxXd";

-			/* Binary isn't natively supported,

-			 * so it's handled differently

-			 */

-			static const char * const formatSpecifiers[] = {

-				"", "%" PRIx32, "%" PRIX32, "%" PRId32

-			};

-			/* Prevent reading out of bounds! */

-			const char *designatedMode;

-

-			if (i != 1)

-				fatalerror("Print types are exactly 1 character long\n");

-

-			designatedMode = strchr(acceptedModes, sym[i - 1]);

-			if (!designatedMode)

-				fatalerror("Illegal print type '%c'\n", sym[i - 1]);

-			mode = formatSpecifiers[designatedMode - acceptedModes];

-			/* Begin writing the symbol again */

-			i = 0;

+	if (capture) {

+		if (lexerState->isMmapped) {

+			captureStart = lexerState->ptr;

 		} else {

-			yylex_SymbolWriteChar(sym, i++, ch);

+			lexerState->captureCapacity = 128; /* The initial size will be twice that */

+			reallocCaptureBuf();

+			captureStart = lexerState->captureBuf;

 		}

 	}

-	/* Properly terminate the string */

-	yylex_SymbolWriteChar(sym, i, 0);

+	for (;;) {

+		int token = yylex();

-	/* It's assumed we're writing to a T_STRING */

-	maxLength = MAXSTRLEN - index;

-	length = symvaluetostring(&dest[index], maxLength, sym, mode);

-

-	if (*pLexBuffer == '}')

-		pLexBuffer++;

-	else

-		fatalerror("Missing }\n");

-

-	return length;

-}

-

-static void yylex_ReadQuotedString(void)

-{

-	size_t index = 0;

-	size_t length, maxLength;

-

-	while (*pLexBuffer != '"' && *pLexBuffer != '\n') {

-		char ch = *pLexBuffer++;

-

-		if (ch == '\\') {

-			ch = *pLexBuffer++;

-

-			switch (ch) {

-			case 'n':

-				ch = '\n';

+		if (level == 0) {

+			if (token == endToken)

 				break;

-			case 'r':

-				ch = '\r';

+			/*

+			 * Hack: skipping after a `if` requires stopping on three different tokens,

+			 * which there is no simple way to make this function support. Instead,

+			 * if ELIF is the end token, ELSE and ENDC are also checked for here.

+			 */

+			if (endToken == T_POP_ELIF && (token == T_POP_ELSE || token == T_POP_ENDC))

 				break;

-			case 't':

-				ch = '\t';

-				break;

-			case '\\':

-				ch = '\\';

-				break;

-			case '"':

-				ch = '"';

-				break;

-			case ',':

-				ch = ',';

-				break;

-			case '{':

-				ch = '{';

-				break;

-			case '}':

-				ch = '}';

-				break;

-			default:

-				maxLength = MAXSTRLEN - index;

-				length = CopyMacroArg(&yylval.tzString[index],

-						      maxLength, ch);

-

-				if (length != 0)

-					index += length;

-				else

-					fatalerror("Illegal character escape '%c'\n", ch);

-

-				ch = 0;

-				break;

-			}

-		} else if (ch == '{') {

-			// Get bracketed symbol within string.

-			index += yylex_ReadBracketedSymbol(yylval.tzString,

-							   index);

-			ch = 0;

 		}

-		if (ch)

-			yylex_StringWriteChar(yylval.tzString, index++, ch);

+		if (token == EOF)

+			error("Unterminated %s\n", name);

+		else if (token == blockStartToken)

+			level++;

+		else if (token == blockEndToken)

+			level--;

 	}

-	yylex_StringWriteChar(yylval.tzString, index, 0);

-

-	if (*pLexBuffer == '"')

-		pLexBuffer++;

-	else

-		fatalerror("Unterminated string\n");

-}

-

-static uint32_t yylex_NORMAL(void)

-{

-	struct sLexString *pLongestFixed = NULL;

-	uint32_t nFloatMask, nFloatLen;

-	uint32_t linestart = AtLineStart;

-

-	AtLineStart = 0;

-

-scanagain:

-	while (*pLexBuffer == ' ' || *pLexBuffer == '\t') {

-		linestart = 0;

-		pLexBuffer++;

+	if (capture) {

+		*capture = captureStart;

+		*size = lexerState->captureSize;

 	}

-

-	if (*pLexBuffer == 0) {

-		// Reached the end of a file, macro, or rept.

-		if (yywrap() == 0) {

-			linestart = AtLineStart;

-			AtLineStart = 0;

-			goto scanagain;

-		}

-	}

-

-	/* Check for line continuation character */

-	if (*pLexBuffer == '\\') {

-		/*

-		 * Look for line continuation character after a series of

-		 * spaces. This is also useful for files that use Windows line

-		 * endings: "\r\n" is replaced by " \n" before the lexer has the

-		 * opportunity to see it.

-		 */

-		if (pLexBuffer[1] == ' ' || pLexBuffer[1] == '\t') {

-			pLexBuffer += 2;

-			while (1) {

-				if (*pLexBuffer == ' ' || *pLexBuffer == '\t') {

-					pLexBuffer++;

-				} else if (*pLexBuffer == '\n') {

-					pLexBuffer++;

-					nLineNo++;

-					goto scanagain;

-				} else {

-					error("Expected a new line after the continuation character.\n");

-					pLexBuffer++;

-				}

-			}

-		}

-

-		/* Line continuation character */

-		if (pLexBuffer[1] == '\n') {

-			pLexBuffer += 2;

-			nLineNo++;

-			goto scanagain;

-		}

-

-		/*

-		 * If there isn't a newline character or a space, ignore the

-		 * character '\'. It will eventually be handled by other

-		 * functions like PutMacroArg().

-		 */

-	}

-

-	/*

-	 * Try to match an identifier, macro argument (e.g. \1),

-	 * or numeric literal.

-	 */

-	yylex_GetFloatMaskAndFloatLen(&nFloatMask, &nFloatLen);

-

-	/* Try to match a keyword or operator. */

-	pLongestFixed = yylex_GetLongestFixed();

-

-	if (nFloatLen == 0 && pLongestFixed == NULL) {

-		/*

-		 * No keyword, identifier, operator, or numerical literal

-		 * matches.

-		 */

-

-		if (*pLexBuffer == '"') {

-			pLexBuffer++;

-			yylex_ReadQuotedString();

-			return T_STRING;

-		} else if (*pLexBuffer == '{') {

-			pLexBuffer++;

-			size_t len = yylex_ReadBracketedSymbol(yylval.tzString,

-							       0);

-			yylval.tzString[len] = 0;

-			return T_STRING;

-		}

-

-		/*

-		 * It's not a keyword, operator, identifier, macro argument,

-		 * numeric literal, string, or bracketed symbol, so just return

-		 * the ASCII character.

-		 */

-		unsigned char ch = *pLexBuffer++;

-

-		if (ch == '\n')

-			AtLineStart = 1;

-

-		/*

-		 * Check for invalid unprintable characters.

-		 * They may not be readily apparent in a text editor,

-		 * so this is useful for identifying encoding problems.

-		 */

-		if (ch != 0

-		 && ch != '\n'

-		 && !(ch >= 0x20 && ch <= 0x7E))

-			fatalerror("Found garbage character: 0x%02X\n", ch);

-

-		return ch;

-	}

-

-	if (pLongestFixed == NULL || nFloatLen > pLongestFixed->nNameLength) {

-		/*

-		 * Longest match was an identifier, macro argument, or numeric

-		 * literal.

-		 */

-		struct sLexFloat *token = lexgetfloat(nFloatMask);

-

-		if (token->Callback) {

-			int32_t done = token->Callback(pLexBuffer, nFloatLen);

-

-			if (!done)

-				goto scanagain;

-		}

-

-		uint32_t type = token->nToken;

-

-		if (type == T_ID && strchr(yylval.tzSym, '.'))

-			type = T_LOCAL_ID;

-

-		if (linestart && type == T_ID)

-			return T_LABEL;

-		return type;

-	}

-

-	/* Longest match was a keyword or operator. */

-	pLexBuffer += pLongestFixed->nNameLength;

-	yylval.nConstValue = pLongestFixed->nToken;

-	return pLongestFixed->nToken;

-}

-

-static uint32_t yylex_MACROARGS(void)

-{

-	size_t index = 0;

-	size_t length, maxLength;

-

-	while ((*pLexBuffer == ' ') || (*pLexBuffer == '\t'))

-		pLexBuffer++;

-

-	while ((*pLexBuffer != ',') && (*pLexBuffer != '\n')) {

-		char ch = *pLexBuffer++;

-

-		if (ch == '\\') {

-			ch = *pLexBuffer++;

-

-			switch (ch) {

-			case 'n':

-				ch = '\n';

-				break;

-			case 't':

-				ch = '\t';

-				break;

-			case '\\':

-				ch = '\\';

-				break;

-			case '"':

-				ch = '\"';

-				break;

-			case ',':

-				ch = ',';

-				break;

-			case '{':

-				ch = '{';

-				break;

-			case '}':

-				ch = '}';

-				break;

-			case ' ':

-			case '\t':

-				/*

-				 * Look for line continuation character after a

-				 * series of spaces. This is also useful for

-				 * files that use Windows line endings: "\r\n"

-				 * is replaced by " \n" before the lexer has the

-				 * opportunity to see it.

-				 */

-				while (1) {

-					if (*pLexBuffer == ' '

-					 || *pLexBuffer == '\t') {

-						pLexBuffer++;

-					} else if (*pLexBuffer == '\n') {

-						pLexBuffer++;

-						nLineNo++;

-						ch = 0;

-						break;

-					} else {

-						error("Expected a new line after the continuation character.\n");

-					}

-				}

-				break;

-			case '\n':

-				/* Line continuation character */

-				nLineNo++;

-				ch = 0;

-				break;

-			default:

-				maxLength = MAXSTRLEN - index;

-				length = CopyMacroArg(&yylval.tzString[index],

-						      maxLength, ch);

-

-				if (length != 0)

-					index += length;

-				else

-					fatalerror("Illegal character escape '%c'\n", ch);

-

-				ch = 0;

-				break;

-			}

-		} else if (ch == '{') {

-			index += yylex_ReadBracketedSymbol(yylval.tzString,

-							   index);

-			ch = 0;

-		}

-		if (ch)

-			yylex_StringWriteChar(yylval.tzString, index++, ch);

-	}

-

-	if (index) {

-		yylex_StringWriteChar(yylval.tzString, index, 0);

-

-		/* trim trailing white space at the end of the line */

-		if (*pLexBuffer == '\n')

-			yylex_TrimEnd(yylval.tzString, index);

-

-		return T_STRING;

-	} else if (*pLexBuffer == '\n') {

-		pLexBuffer++;

-		AtLineStart = 1;

-		return '\n';

-	} else if (*pLexBuffer == ',') {

-		pLexBuffer++;

-		return ',';

-	}

-

-	fatalerror("Internal error in %s\n", __func__);

-}

-

-int yylex(void)

-{

-	int returnedChar;

-

-	switch (lexerstate) {

-	case LEX_STATE_NORMAL:

-		returnedChar = yylex_NORMAL();

-		break;

-	case LEX_STATE_MACROARGS:

-		returnedChar = yylex_MACROARGS();

-		break;

-	default:

-		fatalerror("%s: Internal error.\n", __func__);

-	}

-

-	/* Check if string expansions were fully read */

-	while (pCurrentStringExpansion

-	    && pCurrentStringExpansion->pBuffer == pLexBufferRealStart

-	    && pCurrentStringExpansion->pBufferPos <= pLexBuffer) {

-		struct sStringExpansionPos *pParent =

-			pCurrentStringExpansion->pParent;

-		free(pCurrentStringExpansion->tzName);

-		free(pCurrentStringExpansion);

-

-		pCurrentStringExpansion = pParent;

-		nNbStringExpansions--;

-	}

-

-	return returnedChar;

+	lexerState->captureBuf = NULL;

 }

--- a/src/asm/macro.c

+++ b/src/asm/macro.c

@@ -61,7 +61,7 @@

 #define macArgs (*argPtr)

 	if (macArgs->nbArgs == MAXMACROARGS)

 		error("A maximum of " EXPAND_AND_STR(MAXMACROARGS)

-			" arguments is allowed\n");

+		      " arguments is allowed\n");

 	if (macArgs->nbArgs >= macArgs->capacity) {

 		macArgs->capacity *= 2;

 		/* Check that overflow didn't roll us back */

--- a/src/asm/main.c

+++ b/src/asm/main.c

@@ -6,6 +6,7 @@

  * SPDX-License-Identifier: MIT

  */

+#include <ctype.h>

 #include <errno.h>

 #include <float.h>

 #include <inttypes.h>

@@ -41,11 +42,7 @@

 clock_t nStartClock, nEndClock;

 uint32_t nTotalLines, nIFDepth;

-bool skipElif;

-uint32_t unionStart[128], unionSize[128];

-int32_t nLineNo;

-

 #if defined(YYDEBUG) && YYDEBUG

 extern int yydebug;

 #endif

@@ -76,64 +73,8 @@

 void opt_SetCurrentOptions(struct sOptions *pOpt)

 {

-	if (nGBGfxID != -1) {

-		lex_FloatDeleteRange(nGBGfxID, CurrentOptions.gbgfx[0],

-				     CurrentOptions.gbgfx[0]);

-		lex_FloatDeleteRange(nGBGfxID, CurrentOptions.gbgfx[1],

-				     CurrentOptions.gbgfx[1]);

-		lex_FloatDeleteRange(nGBGfxID, CurrentOptions.gbgfx[2],

-				     CurrentOptions.gbgfx[2]);

-		lex_FloatDeleteRange(nGBGfxID, CurrentOptions.gbgfx[3],

-				     CurrentOptions.gbgfx[3]);

-		lex_FloatDeleteSecondRange(nGBGfxID, CurrentOptions.gbgfx[0],

-					   CurrentOptions.gbgfx[0]);

-		lex_FloatDeleteSecondRange(nGBGfxID, CurrentOptions.gbgfx[1],

-					   CurrentOptions.gbgfx[1]);

-		lex_FloatDeleteSecondRange(nGBGfxID, CurrentOptions.gbgfx[2],

-					   CurrentOptions.gbgfx[2]);

-		lex_FloatDeleteSecondRange(nGBGfxID, CurrentOptions.gbgfx[3],

-					   CurrentOptions.gbgfx[3]);

-	}

-	if (nBinaryID != -1) {

-		lex_FloatDeleteRange(nBinaryID, CurrentOptions.binary[0],

-				     CurrentOptions.binary[0]);

-		lex_FloatDeleteRange(nBinaryID, CurrentOptions.binary[1],

-				     CurrentOptions.binary[1]);

-		lex_FloatDeleteSecondRange(nBinaryID, CurrentOptions.binary[0],

-					   CurrentOptions.binary[0]);

-		lex_FloatDeleteSecondRange(nBinaryID, CurrentOptions.binary[1],

-					   CurrentOptions.binary[1]);

-	}

-	CurrentOptions = *pOpt;

-

-	if (nGBGfxID != -1) {

-		lex_FloatAddRange(nGBGfxID, CurrentOptions.gbgfx[0],

-				  CurrentOptions.gbgfx[0]);

-		lex_FloatAddRange(nGBGfxID, CurrentOptions.gbgfx[1],

-				  CurrentOptions.gbgfx[1]);

-		lex_FloatAddRange(nGBGfxID, CurrentOptions.gbgfx[2],

-				  CurrentOptions.gbgfx[2]);

-		lex_FloatAddRange(nGBGfxID, CurrentOptions.gbgfx[3],

-				  CurrentOptions.gbgfx[3]);

-		lex_FloatAddSecondRange(nGBGfxID, CurrentOptions.gbgfx[0],

-					CurrentOptions.gbgfx[0]);

-		lex_FloatAddSecondRange(nGBGfxID, CurrentOptions.gbgfx[1],

-					CurrentOptions.gbgfx[1]);

-		lex_FloatAddSecondRange(nGBGfxID, CurrentOptions.gbgfx[2],

-					CurrentOptions.gbgfx[2]);

-		lex_FloatAddSecondRange(nGBGfxID, CurrentOptions.gbgfx[3],

-					CurrentOptions.gbgfx[3]);

-	}

-	if (nBinaryID != -1) {

-		lex_FloatAddRange(nBinaryID, CurrentOptions.binary[0],

-				  CurrentOptions.binary[0]);

-		lex_FloatAddRange(nBinaryID, CurrentOptions.binary[1],

-				  CurrentOptions.binary[1]);

-		lex_FloatAddSecondRange(nBinaryID, CurrentOptions.binary[0],

-					CurrentOptions.binary[0]);

-		lex_FloatAddSecondRange(nBinaryID, CurrentOptions.binary[1],

-					CurrentOptions.binary[1]);

-	}

+	/* TODO */

+	(void)pOpt;

 }

 void opt_Parse(char *s)

@@ -251,6 +192,22 @@

 		sym_AddString(cldefines[i], cldefines[i + 1]);

 }

+void upperstring(char *s)

+{

+	while (*s) {

+		*s = toupper(*s);

+		s++;

+	}

+}

+

+void lowerstring(char *s)

+{

+	while (*s) {

+		*s = tolower(*s);

+		s++;

+	}

+}

+

 /* Escapes Make-special chars from a string */

 static char *make_escape(const char *str)

 {

@@ -516,8 +473,6 @@

 	tzMainfile = argv[argc - 1];

-	setup_lexer();

-

 	if (verbose)

 		printf("Assembling %s\n", tzMainfile);

@@ -530,17 +485,20 @@

 	nStartClock = clock();

-	nLineNo = 1;

 	nTotalLines = 0;

 	nIFDepth = 0;

-	skipElif = true;

 	sym_Init();

 	sym_SetExportAll(exportall);

 	fstk_Init(tzMainfile);

+	struct LexerState *state = lexer_OpenFile(tzMainfile);

+

+	if (!state)

+		fatalerror("Failed to open main file!");

+	lexer_SetState(state);

+

 	opt_ParseDefines();

 	charmap_New("main", NULL);

-	yy_set_state(LEX_STATE_NORMAL);

 	opt_SetCurrentOptions(&DefaultOptions);

 	if (yyparse() != 0 || nbErrors != 0)

--- a/src/asm/section.c

+++ b/src/asm/section.c

@@ -656,9 +656,15 @@

 		startPos = 0;

 	}

-	FILE *f = fstk_FindFile(s, NULL);

+	char *fullPath = NULL;

+	size_t size = 0;

+	FILE *f = NULL;

+	if (fstk_FindFile(s, &fullPath, &size))

+		f = fopen(fullPath, "rb");

+

 	if (!f) {

+		free(fullPath);

 		if (oGeneratedMissingIncludes) {

 			oFailedOnMissingInclude = true;

 			return;

@@ -699,6 +705,7 @@

 		error("Error reading INCBIN file '%s': %s\n", s, strerror(errno));

 	fclose(f);

+	free(fullPath);

 }

 void out_BinaryFileSlice(char const *s, int32_t start_pos, int32_t length)

@@ -715,9 +722,15 @@

 	if (length == 0) /* Don't even bother with 0-byte slices */

 		return;

-	FILE *f = fstk_FindFile(s, NULL);

+	char *fullPath = NULL;

+	size_t size = 0;

+	FILE *f = NULL;

+	if (fstk_FindFile(s, &fullPath, &size))

+		f = fopen(fullPath, "rb");

+

 	if (!f) {

+		free(fullPath);

 		if (oGeneratedMissingIncludes) {

 			oFailedOnMissingInclude = true;

 			return;

@@ -767,6 +780,7 @@

 	}

 	fclose(f);

+	free(fullPath);

 }

 /*

--- a/src/asm/symbol.c

+++ b/src/asm/symbol.c

@@ -82,7 +82,7 @@

 static int32_t Callback__LINE__(void)

 {

-	return nLineNo;

+	return lexer_GetLineNo();

 }

 static int32_t CallbackPC(void)

@@ -113,8 +113,9 @@

 static void updateSymbolFilename(struct Symbol *sym)

 {

 	if (snprintf(sym->fileName, _MAX_PATH + 1, "%s",

-		     tzCurrentFileName) > _MAX_PATH)

-		fatalerror("%s: File name is too long: '%s'\n", __func__, tzCurrentFileName);

+		     lexer_GetFileName()) > _MAX_PATH)

+		fatalerror("%s: File name is too long: '%s'\n", __func__,

+			   lexer_GetFileName());

 	sym->fileLine = fstk_GetLine();

 }

--- a/src/asm/warning.c

+++ b/src/asm/warning.c

@@ -204,7 +204,7 @@

 	fstk_Dump();

 	fprintf(stderr, flag ? ": [-Werror=%s]\n    " : ":\n    ", flag);

 	vfprintf(stderr, fmt, args);

-	fstk_DumpStringExpansions();

+	lexer_DumpStringExpansions();

 	nbErrors++;

 }

@@ -256,7 +256,7 @@

 	fstk_Dump();

 	fprintf(stderr, ": [-W%s]\n    ", flag);

 	vfprintf(stderr, fmt, args);

-	fstk_DumpStringExpansions();

+	lexer_DumpStringExpansions();

 	va_end(args);

 }