Make strings a first-class token type; defer evaluation

Make strings a proper, first-class token type, instead of relying on
the "TOKEN_NUM with tv_charptr" hack.  Only convert a string to a
number if requested in an expression context; this also makes it
possible to actually issue a warning when it overflows.

eval.c

@@ -668,6 +668,8 @@ static expr *expr6(int critical)
     expr *e;
     int32_t label_seg;
     int64_t label_ofs;
+    int64_t tmpval;
+    bool rn_warn;
     char *scope;
 
     switch (i) {
@@ -741,6 +743,7 @@ static expr *expr6(int critical)
         return e;
 
     case TOKEN_NUM:
+    case TOKEN_STR:
     case TOKEN_REG:
     case TOKEN_ID:
     case TOKEN_INSN:		/* Opcodes that occur here are really labels */
@@ -751,6 +754,12 @@ static expr *expr6(int critical)
         case TOKEN_NUM:
             addtotemp(EXPR_SIMPLE, tokval->t_integer);
             break;
+	case TOKEN_STR:
+	    tmpval = readstrnum(tokval->t_charptr, tokval->t_inttwo, &rn_warn);
+	    if (rn_warn)
+		error(ERR_WARNING|ERR_PASS1, "character constant too long");
+            addtotemp(EXPR_SIMPLE, tmpval);
+	    break;
         case TOKEN_REG:
             addtotemp(tokval->t_integer, 1L);
             if (hint && hint->type == EAH_NOHINT)

nasm.h

@@ -165,8 +165,14 @@ enum token_type {		/* token types, other than chars */
     TOKEN_INVALID = -1,         /* a placeholder value */
     TOKEN_EOS = 0,              /* end of string */
     TOKEN_EQ = '=', TOKEN_GT = '>', TOKEN_LT = '<',     /* aliases */
-    TOKEN_ID = 256, TOKEN_NUM, TOKEN_REG, TOKEN_INSN,   /* major token types */
-    TOKEN_ERRNUM,               /* numeric constant with error in */
+    TOKEN_ID = 256,		/* identifier */
+    TOKEN_NUM,			/* numeric constant */
+    TOKEN_ERRNUM,		/* malformed numeric constant */
+    TOKEN_STR,			/* string constant */
+    TOKEN_ERRSTR,               /* unterminated string constant */
+    TOKEN_FLOAT,                /* floating-point constant */
+    TOKEN_REG,			/* register name */
+    TOKEN_INSN,			/* instruction name */
     TOKEN_HERE, TOKEN_BASE,     /* $ and $$ */
     TOKEN_SPECIAL,              /* BYTE, WORD, DWORD, QWORD, FAR, NEAR, etc */
     TOKEN_PREFIX,               /* A32, O16, LOCK, REPNZ, TIMES, etc */
@@ -175,7 +181,6 @@ enum token_type {		/* token types, other than chars */
     TOKEN_GE, TOKEN_LE, TOKEN_NE,       /* >=, <= and <> (!= is same as <>) */
     TOKEN_DBL_AND, TOKEN_DBL_OR, TOKEN_DBL_XOR, /* &&, || and ^^ */
     TOKEN_SEG, TOKEN_WRT,       /* SEG and WRT */
-    TOKEN_FLOAT,                /* floating-point constant */
     TOKEN_FLOATIZE,		/* __floatX__ */
 };
 

parser.c

@@ -356,7 +356,7 @@ restart_parse:
             eop->type = EOT_NOTHING;
             oper_num++;
 
-            if (i == TOKEN_NUM && tokval.t_charptr && is_comma_next()) {
+            if (i == TOKEN_STR && is_comma_next()) {
                 eop->type = EOT_DB_STRING;
                 eop->stringval = tokval.t_charptr;
                 eop->stringlen = tokval.t_inttwo;

preproc.c

@@ -1134,10 +1134,11 @@ static int ppscan(void *private_data, struct tokenval *tokval)
     if (tline->type == TOK_NUMBER) {
 	bool rn_error;
 	tokval->t_integer = readnum(tline->text, &rn_error);
-	if (rn_error)
-	    return tokval->t_type = TOKEN_ERRNUM;   /* some malformation occurred */
 	tokval->t_charptr = tline->text;
-	return tokval->t_type = TOKEN_NUM;
+	if (rn_error)
+	    return tokval->t_type = TOKEN_ERRNUM;
+	else
+	    return tokval->t_type = TOKEN_NUM;
     }
 
     if (tline->type == TOK_FLOAT) {
@@ -1146,23 +1147,15 @@ static int ppscan(void *private_data, struct tokenval *tokval)
 
     if (tline->type == TOK_STRING) {
 	char bq, *ep;
-	bool errquote;
-	bool rn_warn;
-        size_t l;
 
 	bq = tline->text[0];
-        l = nasm_unquote(tline->text, &ep);
-	if (ep[0] != bq || ep[1] != '\0')
-	    errquote = true;
+        tokval->t_charptr = tline->text;
+        tokval->t_inttwo = nasm_unquote(tline->text, &ep);
 	
-	if (errquote)
-	    return tokval->t_type = TOKEN_ERRNUM;
-
-        tokval->t_integer = readstrnum(tline->text, l, &rn_warn);
-        if (rn_warn)
-            error(ERR_WARNING | ERR_PASS1, "character constant too long");
-        tokval->t_charptr = NULL;
-        return tokval->t_type = TOKEN_NUM;
+	if (ep[0] != bq || ep[1] != '\0')
+	    return tokval->t_type = TOKEN_ERRSTR;
+	else
+	    return tokval->t_type = TOKEN_STR;
     }
 
     if (tline->type == TOK_OTHER) {

stdscan.c

@@ -177,17 +177,13 @@ int stdscan(void *private_data, struct tokenval *tv)
     } else if (*stdscan_bufptr == '\'' || *stdscan_bufptr == '"' ||
 	       *stdscan_bufptr == '`') {
 	/* a quoted string */
-        bool rn_warn;
 	char start_quote = *stdscan_bufptr;
 	tv->t_charptr = stdscan_bufptr;
 	tv->t_inttwo = nasm_unquote(tv->t_charptr, &stdscan_bufptr);
 	if (*stdscan_bufptr != start_quote)
-	    return tv->t_type = TOKEN_ERRNUM;
+	    return tv->t_type = TOKEN_ERRSTR;
 	stdscan_bufptr++;	/* Skip final quote */
-        tv->t_integer = readstrnum(tv->t_charptr, tv->t_inttwo, &rn_warn);
-	/* Issue: can't readily check rn_warn, because we might be in
-	   a db family context... */
-        return tv->t_type = TOKEN_NUM;
+        return tv->t_type = TOKEN_STR;
     } else if (*stdscan_bufptr == ';') {
         /* a comment has happened - stay */
         return tv->t_type = 0;