public: add new features to ParseFileSafe, such as disabling interpreting certain characters as separate tokens, or instead forcing them to be separate tokens

engine/client/cl_game.c

@@ -3218,7 +3218,8 @@ handle colon separately
 */
 static char *pfnParseFile( char *data, char *token )
 {
-	return COM_ParseFileSafe( data, token, PFILE_TOKEN_MAX_LENGTH, PFILE_HANDLECOLON, NULL, NULL );
+	// GoldSrc uses 1024 byte tokens
+	return COM_ParseFileSafe( data, token, 1024, PFILE_HANDLECOLON, NULL, NULL );
 }
 
 /*

filesystem/VFileSystem009.cpp

@@ -412,7 +412,8 @@ public:
 		qboolean qquoted;
 		char *p;
 
-		p = COM_ParseFileSafe( buf, token, PFILE_FS_TOKEN_MAX_LENGTH, 0, nullptr, &qquoted );
+		// filesystem_stdio expects 512 byte buffers
+		p = COM_ParseFileSafe( buf, token, 512, 0, nullptr, &qquoted );
 
 		if( quoted )
 			*quoted = qquoted;

public/crtlib.c

@@ -666,14 +666,23 @@ interpert this character as single
 */
 static int COM_IsSingleChar( unsigned int flags, char c )
 {
-	if( c == '{' || c == '}' || c == '\'' || c == ',' )
+	switch( c )
-		return true;
+	{
-
+	case '}':
-	if( !FBitSet( flags, PFILE_IGNOREBRACKET ) && ( c == ')' || c == '(' ))
+	case '{':
-		return true;
-
-	if( FBitSet( flags, PFILE_HANDLECOLON ) && c == ':' )
 		return true;
+	case ',':
+		return !FBitSet( flags, PFILE_NO_COMMA_AS_TOKEN );
+	case '\'':
+		return !FBitSet( flags, PFILE_NO_SINGLE_QUOTE_AS_TOKEN );
+	case '(':
+	case ')':
+		return !FBitSet( flags, PFILE_NO_BRACKETS_AS_TOKEN );
+	case ':':
+		return FBitSet( flags, PFILE_COLON_AS_TOKEN );
+	case '\n':
+		return FBitSet( flags, PFILE_NEWLINE_AS_TOKEN );
+	}
 
 	return false;
 }
@@ -707,6 +716,9 @@ char *COM_ParseFileSafe( char *data, char *token, const int size, unsigned int f
 skipwhite:
 	while(( c = ((byte)*data)) <= ' ' )
 	{
+		if( FBitSet( flags, PFILE_NEWLINE_AS_TOKEN ) && c == '\n' )
+			break;
+
 		if( c == 0 )
 		{
 			if( plen ) *plen = overflow ? -1 : len;
@@ -716,7 +728,7 @@ skipwhite:
 	}
 
 	// skip // or #, if requested, comments
-	if(( c == '/' && data[1] == '/' ) || ( c == '#' && FBitSet( flags, PFILE_IGNOREHASHCMT )))
+	if(( c == '/' && data[1] == '/' ) || ( c == '#' && FBitSet( flags, PFILE_HASH_AS_COMMENT )))
 	{
 		while( *data && *data != '\n' )
 			data++;
@@ -724,7 +736,7 @@ skipwhite:
 	}
 
 	// handle quoted strings specially
-	if( c == '\"' )
+	if( c == '\"' && !FBitSet( flags, PFILE_NO_QUOTED_TOKENS ))
 	{
 		if( quoted )
 			*quoted = true;

public/crtlib.h

@@ -41,12 +41,29 @@ enum
 
 // a1ba: not using BIT macro, so flags can be copypasted into
 // exported APIs headers and will not get warning in case of changing values
-#define PFILE_IGNOREBRACKET (1<<0)
-#define PFILE_HANDLECOLON   (1<<1)
-#define PFILE_IGNOREHASHCMT (1<<2)
 
-#define PFILE_TOKEN_MAX_LENGTH 1024
+// do NOT interpret brackets as separate token
-#define PFILE_FS_TOKEN_MAX_LENGTH 512
+// e.g. (a b) will be parsed as "(a", "b)"
+#define PFILE_NO_BRACKETS_AS_TOKEN     ( 1U << 0 )
+
+// interpret ':' as separate token
+// e.g. a:b will be parsed as "a", ":", "b"
+#define PFILE_COLON_AS_TOKEN           ( 1U << 1 )
+
+// ignore the whole line starting with '#' interpreting it as comment
+#define PFILE_HASH_AS_COMMENT          ( 1U << 2 )
+
+// when encountering double quotes, do not interpret what's inside as single token
+#define PFILE_NO_QUOTED_TOKENS         ( 1U << 3 )
+
+// do NOT interpret ''' as separate token
+#define PFILE_NO_SINGLE_QUOTE_AS_TOKEN ( 1U << 4 )
+
+// do NOT interpret comma as token
+#define PFILE_NO_COMMA_AS_TOKEN        ( 1U << 5 )
+
+// interpret '\n' as separate token
+#define PFILE_NEWLINE_AS_TOKEN         ( 1U << 6 )
 
 #ifdef __cplusplus
 #define restrict

public/tests/test_parsefile.c

@@ -1,3 +1,4 @@
+#include <stdio.h>
 #include "crtlib.h"
 
 static const char *test_file =
@@ -7,7 +8,15 @@ static const char *test_file =
 "bashlikecomment	#notignored	test\n"
 "#ignore comment\n"
 "thisshall #be ignored\n"
-"test_sentinel\n";
+"test_sentinel\n"
+"ignore:colon\n"
+"notignore:colon\n"
+"(a b)\n"
+"\"test1\"\n"
+"// comment\n"
+"'a 'b\n"
+",a ,b\n"
+;
 
 int main( void )
 {
@@ -29,9 +38,25 @@ int main( void )
 	{ 5, "bark", 4 },
 	{ 32, "bashlikecomment", 15 },
 	{ 32, "#notignored", 11 },
-	{ 32, "test", 4, PFILE_IGNOREHASHCMT },
+	{ 32, "test", 4, PFILE_HASH_AS_COMMENT },
-	{ 32, "thisshall", 9, PFILE_IGNOREHASHCMT },
+	{ 32, "thisshall", 9, PFILE_HASH_AS_COMMENT },
-	{ 32, "test_sentinel", 13, PFILE_IGNOREHASHCMT },
+	{ 32, "test_sentinel", 13, PFILE_HASH_AS_COMMENT },
+	{ 32, "ignore:colon", 12, },
+	{ 32, "notignore", 9, PFILE_COLON_AS_TOKEN },
+	{ 32, ":", 1, PFILE_COLON_AS_TOKEN },
+	{ 32, "colon", 5, PFILE_COLON_AS_TOKEN },
+	{ 32, "(a", 2, PFILE_NO_BRACKETS_AS_TOKEN },
+	{ 32, "b", 1, },
+	{ 32, ")", 1, },
+	{ 32, "\"test1\"", 7, PFILE_NO_QUOTED_TOKENS },
+	{ 32, "\n", 1, PFILE_NEWLINE_AS_TOKEN },
+	{ 32, "'a", 2, PFILE_NO_SINGLE_QUOTE_AS_TOKEN },
+	{ 32, "'", 1, },
+	{ 32, "b", 1, },
+	{ 32, ",a", 2, PFILE_NO_COMMA_AS_TOKEN },
+	{ 32, ",", 1, },
+	{ 32, "b", 1, },
+	{ 32, NULL }, // EOF
 	};
 
 	for( i = 0; i < sizeof( testdata ) / sizeof( testdata[0] ); i++ )
@@ -41,11 +66,26 @@ int main( void )
 
 		file = COM_ParseFileSafe( file, buf, testdata[i].bufsize, testdata[i].flags, &len, NULL );
 
-		if( file == NULL )
+		if( testdata[i].expected == NULL && file != NULL )
+		{
+			printf( "expected NULL, but got no NULL\n" );
 			return i;
+		}
+
+		if( testdata[i].expected != NULL && file == NULL )
+		{
+			printf( "expected no NULL, but got NULL\n" );
+			return i;
+		}
+
+		if( testdata[i].expected == NULL && file == NULL )
+			continue;
 
 		if( !( !Q_strcmp( buf, testdata[i].expected ) && len == testdata[i].expected_len ))
+		{
+			printf( "expected '%s' with length %d, got '%s' with length %d\n", testdata[i].expected, testdata[i].expected_len, buf, len );
 			return i;
+		}
 	}
 
 	return 0;