Makefile.in (c-lex.o, [...]): Update.

* Makefile.in (c-lex.o, LIBCPP_OBJS, cpplex.o): Update.
	* c-lex.c (MULTIBYTE_CHARS): Remove conditionals.
	(lex_string): Take cpp_string with full spelling.
	(cb_ident): Update.
	(c_lex): Update diagnostics.
	* cpplex.c (SPELL_NUMBER, SPELL_STRING): Combine into SPELL_LITERAL.
	(create_literal): New.
	(lex_string): Unterminated literals have type CPP_OTHER.
	(_cpp_lex_direct): Update calls to lex_string.  Use create_literal
	for CPP_OTHER.
	(cpp_token_len, cpp_spell_token, cpp_output_token): Simplify.
	(_cpp_equiv_tokens, cpp_interpret_charconst): Update.
	* cpplib.c (parse_include, do_line, do_linemarker,
	destringize_and_run): Update for token storing full spelling.
	* cpplib.h: Update token spelling types.
	* cppmacro.c (stringify_arg, check_trad_stringification):
	Update for token storing full spelling.
cp:
	* Make-lang.in (lex.o): Remove mbchar.h.
	* lex.c (MULTIBYTE_CHARS): Lose.
	* parser.c (cp_lexer_get_preprocessor_token): CPP_OTHER handled
	in c-lex.c.
testsuite:
	* gcc.dg/cpp/include2.c: Update.
	* gcc.dg/cpp/multiline-2.c: New.
	* gcc.dg/cpp/multiline.c: Update.
	* gcc.dg/cpp/strify2.c: Update.
	* gcc.dg/cpp/trad/literals-2.c: Update.

From-SVN: r66019

gcc/ChangeLog

@@ -1,3 +1,23 @@
+2003-04-23  Neil Booth  <neil@daikokuya.co.uk>
+
+	* Makefile.in (c-lex.o, LIBCPP_OBJS, cpplex.o): Update.
+	* c-lex.c (MULTIBYTE_CHARS): Remove conditionals.
+	(lex_string): Take cpp_string with full spelling.
+	(cb_ident): Update.
+	(c_lex): Update diagnostics.
+	* cpplex.c (SPELL_NUMBER, SPELL_STRING): Combine into SPELL_LITERAL.
+	(create_literal): New.
+	(lex_string): Unterminated literals have type CPP_OTHER.
+	(_cpp_lex_direct): Update calls to lex_string.  Use create_literal
+	for CPP_OTHER.
+	(cpp_token_len, cpp_spell_token, cpp_output_token): Simplify.
+	(_cpp_equiv_tokens, cpp_interpret_charconst): Update.
+	* cpplib.c (parse_include, do_line, do_linemarker,
+	destringize_and_run): Update for token storing full spelling.
+	* cpplib.h: Update token spelling types.
+	* cppmacro.c (stringify_arg, check_trad_stringification):
+	Update for token storing full spelling.
+
 2003-04-23  Ulrich Weigand  <uweigand@de.ibm.com>
 
 	* config/s390/s390.c (s390_expand_cmpstr): Disable CLC loop.

gcc/Makefile.in

@@ -1267,7 +1267,7 @@ c-lang.o : c-lang.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(TREE_H) $(C_TR
 c-lex.o : c-lex.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(TREE_H) \
     $(RTL_H) debug.h $(C_TREE_H) c-common.h real.h c-incpath.h cppdefault.h \
     c-pragma.h input.h intl.h flags.h toplev.h output.h \
-    mbchar.h $(CPPLIB_H) $(EXPR_H) $(TM_P_H)
+    $(CPPLIB_H) $(EXPR_H) $(TM_P_H)
 c-ppoutput.o : c-ppoutput.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) \
     c-common.h $(TREE_H) $(CPPLIB_H) cpphash.h $(TM_P_H) c-pragma.h
 c-objc-common.o : c-objc-common.c $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H) $(TREE_H) \
@@ -2326,7 +2326,7 @@ PREPROCESSOR_DEFINES = \
 
 LIBCPP_OBJS =	cpplib.o cpplex.o cppmacro.o cppexp.o cppfiles.o cpptrad.o \
 		cpphash.o cpperror.o cppinit.o cppcharset.o \
-		hashtable.o line-map.o mkdeps.o mbchar.o cpppch.o
+		hashtable.o line-map.o mkdeps.o cpppch.o
 
 LIBCPP_DEPS =	$(CPPLIB_H) cpphash.h line-map.h hashtable.h intl.h \
 		$(OBSTACK_H) $(CONFIG_H) $(SYSTEM_H) coretypes.h $(TM_H)
@@ -2341,7 +2341,7 @@ libcpp.a: $(LIBCPP_OBJS)
 cppcharset.o: cppcharset.c $(LIBCPP_DEPS)
 cpperror.o: cpperror.c $(LIBCPP_DEPS)
 cppexp.o:   cppexp.c   $(LIBCPP_DEPS)
-cpplex.o:   cpplex.c   $(LIBCPP_DEPS) mbchar.h
+cpplex.o:   cpplex.c   $(LIBCPP_DEPS)
 cppmacro.o: cppmacro.c $(LIBCPP_DEPS)
 cpplib.o:   cpplib.c   $(LIBCPP_DEPS)
 cpphash.o:  cpphash.c  $(LIBCPP_DEPS)

gcc/c-lex.c

@@ -42,11 +42,6 @@ Software Foundation, 59 Temple Place - Suite 330, Boston, MA
 #include "splay-tree.h"
 #include "debug.h"
 
-#ifdef MULTIBYTE_CHARS
-#include "mbchar.h"
-#include <locale.h>
-#endif /* MULTIBYTE_CHARS */
-
 /* The current line map.  */
 static const struct line_map *map;
 
@@ -78,8 +73,7 @@ static enum integer_type_kind
   narrowest_unsigned_type	PARAMS ((tree, unsigned int));
 static enum integer_type_kind
   narrowest_signed_type		PARAMS ((tree, unsigned int));
-static tree lex_string		PARAMS ((const unsigned char *, unsigned int,
+static tree lex_string		PARAMS ((const cpp_string *));
-					 int));
 static tree lex_charconst	PARAMS ((const cpp_token *));
 static void update_header_times	PARAMS ((const char *));
 static int dump_one_header	PARAMS ((splay_tree_node, void *));
@@ -201,7 +195,7 @@ cb_ident (pfile, line, str)
   if (! flag_no_ident)
     {
       /* Convert escapes in the string.  */
-      tree value ATTRIBUTE_UNUSED = lex_string (str->text, str->len, 0);
+      tree value ATTRIBUTE_UNUSED = lex_string (str);
       ASM_OUTPUT_IDENT (asm_out_file, TREE_STRING_POINTER (value));
     }
 #endif
@@ -329,7 +323,7 @@ c_lex (value)
 {
   const cpp_token *tok;
 
-  retry:
+ retry:
   timevar_push (TV_CPP);
   do
     tok = cpp_get_token (parse_in);
@@ -344,11 +338,6 @@ c_lex (value)
   *value = NULL_TREE;
   switch (tok->type)
     {
-    case CPP_OTHER:
-      error ("stray token \"%s\" in program",
-	     cpp_token_as_text (parse_in, tok));
-      goto retry;
-      
     case CPP_NAME:
       *value = HT_IDENT_TO_GCC_IDENT (HT_NODE (tok->val.node));
       break;
@@ -378,6 +367,19 @@ c_lex (value)
       }
       break;
 
+    case CPP_OTHER:
+      {
+	cppchar_t c = tok->val.str.text[0];
+
+	if (c == '"' || c == '\'')
+	  error ("missing terminating %c character", (int) c);
+	else if (ISGRAPH (c))
+	  error ("stray '%c' in program", (int) c);
+	else
+	  error ("stray '\\%o' in program", (int) c);
+      }
+      goto retry;
+
     case CPP_CHAR:
     case CPP_WCHAR:
       *value = lex_charconst (tok);
@@ -385,8 +387,7 @@ c_lex (value)
 
     case CPP_STRING:
     case CPP_WSTRING:
-      *value = lex_string (tok->val.str.text, tok->val.str.len,
+      *value = lex_string (&tok->val.str);
-			   tok->type == CPP_WSTRING);
       break;
 
       /* These tokens should not be visible outside cpplib.  */
@@ -601,43 +602,23 @@ interpret_float (token, flags)
 }
 
 static tree
-lex_string (str, len, wide)
+lex_string (str)
-     const unsigned char *str;
+     const cpp_string *str;
-     unsigned int len;
-     int wide;
 {
+  bool wide;
   tree value;
-  char *buf = alloca ((len + 1) * (wide ? WCHAR_BYTES : 1));
+  char *buf, *q;
-  char *q = buf;
-  const unsigned char *p = str, *limit = str + len;
   cppchar_t c;
-
+  const unsigned char *p, *limit;
-#ifdef MULTIBYTE_CHARS
+  
-  /* Reset multibyte conversion state.  */
+  wide = str->text[0] == 'L';
-  (void) local_mbtowc (NULL, NULL, 0);
+  p = str->text + 1 + wide;
-#endif
+  limit = str->text + str->len - 1;
+  q = buf = alloca ((str->len + 1) * (wide ? WCHAR_BYTES : 1));
 
   while (p < limit)
     {
-#ifdef MULTIBYTE_CHARS
-      wchar_t wc;
-      int char_len;
-
-      char_len = local_mbtowc (&wc, (const char *) p, limit - p);
-      if (char_len == -1)
-	{
-	  warning ("ignoring invalid multibyte character");
-	  char_len = 1;
-	  c = *p++;
-	}
-      else
-	{
-	  p += char_len;
-	  c = wc;
-	}
-#else
       c = *p++;
-#endif
 
       if (c == '\\' && !ignore_escape_flag)
 	c = cpp_parse_escape (parse_in, &p, limit, wide);
@@ -664,16 +645,6 @@ lex_string (str, len, wide)
 	    }
 	  q += WCHAR_BYTES;
 	}
-#ifdef MULTIBYTE_CHARS
-      else if (char_len > 1)
-	{
-	  /* We're dealing with a multibyte character.  */
-	  for ( ; char_len >0; --char_len)
-	    {
-	      *q++ = *(p - char_len);
-	    }
-	}
-#endif
       else
 	{
 	  *q++ = c;

gcc/cp/ChangeLog

@@ -1,3 +1,10 @@
+2003-04-23  Neil Booth  <neil@daikokuya.co.uk>
+
+	* Make-lang.in (lex.o): Remove mbchar.h.
+	* lex.c (MULTIBYTE_CHARS): Lose.
+	* parser.c (cp_lexer_get_preprocessor_token): CPP_OTHER handled
+	in c-lex.c.
+
 2003-04-23  Mark Mitchell  <mark@codesourcery.com>
 
 	PR c++/9847

gcc/cp/Make-lang.in

@@ -228,7 +228,7 @@ CXX_TREE_H = $(TREE_H) cp/name-lookup.h cp/cp-tree.h c-common.h \
 	$(srcdir)/../include/hashtab.h $(srcdir)/../include/splay-tree.h
 
 cp/lex.o: cp/lex.c $(CXX_TREE_H) $(TM_H) flags.h cp/lex.h \
-  c-pragma.h toplev.h output.h mbchar.h input.h diagnostic.h \
+  c-pragma.h toplev.h output.h input.h diagnostic.h \
   cp/operators.def $(TM_P_H)
 cp/cp-lang.o: cp/cp-lang.c $(CXX_TREE_H) $(TM_H) toplev.h langhooks.h \
   $(LANGHOOKS_DEF_H) c-common.h

gcc/cp/lex.c

@@ -40,11 +40,6 @@ Boston, MA 02111-1307, USA.  */
 #include "timevar.h"
 #include "diagnostic.h"
 
-#ifdef MULTIBYTE_CHARS
-#include "mbchar.h"
-#include <locale.h>
-#endif
-
 static int interface_strcmp PARAMS ((const char *));
 static void init_cp_pragma PARAMS ((void));
 

gcc/cp/parser.c

@@ -644,10 +644,6 @@ cp_lexer_get_preprocessor_token (cp_lexer *lexer ATTRIBUTE_UNUSED ,
 	  error ("invalid token");
 	  break;
 
-	case CPP_OTHER:
-	  /* These tokens are already warned about by c_lex.  */
-	  break;
-
 	default:
 	  /* This is a good token, so we exit the loop.  */
 	  done = true;

gcc/cpplex.c

@@ -26,14 +26,11 @@ Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.  */
 #include "cpplib.h"
 #include "cpphash.h"
 
-/* Tokens with SPELL_STRING store their spelling in the token list,
-   and it's length in the token->val.name.len.  */
 enum spell_type
 {
   SPELL_OPERATOR = 0,
   SPELL_IDENT,
-  SPELL_NUMBER,
+  SPELL_LITERAL,
-  SPELL_STRING,
   SPELL_NONE
 };
 
@@ -61,9 +58,11 @@ static void skip_whitespace PARAMS ((cpp_reader *, cppchar_t));
 static cpp_hashnode *lex_identifier PARAMS ((cpp_reader *, const uchar *));
 static void lex_number PARAMS ((cpp_reader *, cpp_string *));
 static bool forms_identifier_p PARAMS ((cpp_reader *, int));
-static void lex_string PARAMS ((cpp_reader *, cpp_token *));
+static void lex_string PARAMS ((cpp_reader *, cpp_token *, const uchar *));
 static void save_comment PARAMS ((cpp_reader *, cpp_token *, const uchar *,
 				  cppchar_t));
+static void create_literal PARAMS ((cpp_reader *, cpp_token *, const uchar *,
+				    unsigned int, enum cpp_ttype));
 static int name_p PARAMS ((cpp_reader *, const cpp_string *));
 static cppchar_t maybe_read_ucn PARAMS ((cpp_reader *, const uchar **));
 static tokenrun *next_tokenrun PARAMS ((tokenrun *));
@@ -468,63 +467,77 @@ lex_number (pfile, number)
   number->text = dest;
 }
 
-/* Lexes a string, character constant, or angle-bracketed header file
+/* Create a token of type TYPE with a literal spelling.  */
-   name.  The stored string is guaranteed NUL-terminated, but it is
-   not guaranteed that this is the first NUL since embedded NULs are
-   preserved.  */
 static void
-lex_string (pfile, token)
+create_literal (pfile, token, base, len, type)
      cpp_reader *pfile;
      cpp_token *token;
+     const uchar *base;
+     unsigned int len;
+     enum cpp_ttype type;
 {
-  cpp_buffer *buffer = pfile->buffer;
+  uchar *dest = _cpp_unaligned_alloc (pfile, len + 1);
-  bool warned_nulls = false;
-  const uchar *base;
-  uchar *dest;
-  cppchar_t terminator;
 
-  base = buffer->cur;
+  memcpy (dest, base, len);
-  terminator = base[-1];
+  dest[len] = '\0';
-  if (terminator == '<')
+  token->type = type;
-    terminator = '>';
+  token->val.str.len = len;
+  token->val.str.text = dest;
+}
+
+/* Lexes a string, character constant, or angle-bracketed header file
+   name.  The stored string contains the spelling, including opening
+   quote and leading any leading 'L'.  It returns the type of the
+   literal, or CPP_OTHER if it was not properly terminated.
+
+   The spelling is NUL-terminated, but it is not guaranteed that this
+   is the first NUL since embedded NULs are preserved.  */
+static void
+lex_string (pfile, token, base)
+     cpp_reader *pfile;
+     cpp_token *token;
+     const uchar *base;
+{
+  bool saw_NUL = false;
+  const uchar *cur;
+  cppchar_t terminator;
+  enum cpp_ttype type;
+
+  cur = base;
+  terminator = *cur++;
+  if (terminator == 'L')
+    terminator = *cur++;
+  if (terminator == '\"')
+    type = *base == 'L' ? CPP_WSTRING: CPP_STRING;
+  else if (terminator == '\'')
+    type = *base == 'L' ? CPP_WCHAR: CPP_CHAR;
+  else
+    terminator = '>', type = CPP_HEADER_NAME;
 
   for (;;)
     {
-      cppchar_t c = *buffer->cur++;
+      cppchar_t c = *cur++;
 
       /* In #include-style directives, terminators are not escapable.  */
-      if (c == '\\' && !pfile->state.angled_headers && *buffer->cur != '\n')
+      if (c == '\\' && !pfile->state.angled_headers && *cur != '\n')
-	buffer->cur++;
+	cur++;
-      else if (c == terminator || c == '\n')
+      else if (c == terminator)
 	break;
-      else if (c == '\0')
+      else if (c == '\n')
 	{
-	  if (!warned_nulls)
+	  cur--;
-	    {
+	  type = CPP_OTHER;
-	      warned_nulls = true;
+	  break;
-	      cpp_error (pfile, DL_WARNING,
-			 "null character(s) preserved in literal");
-	    }
 	}
+      else if (c == '\0')
+	saw_NUL = true;
     }
 
-  token->val.str.len = buffer->cur - base - 1;
+  if (saw_NUL && !pfile->state.skipping)
-  dest = _cpp_unaligned_alloc (pfile, token->val.str.len + 1);
+    cpp_error (pfile, DL_WARNING, "null character(s) preserved in literal");
-  memcpy (dest, base, token->val.str.len);
-  dest[token->val.str.len] = '\0';
-  token->val.str.text = dest;
 
-  if (buffer->cur[-1] == '\n')
+  pfile->buffer->cur = cur;
-    {
+  create_literal (pfile, token, base, cur - base, type);
-      /* No string literal may extend over multiple lines.  In
-	 assembly language, suppress the error except for <>
-	 includes.  This is a kludge around not knowing where
-	 comments are.  */
-      if (CPP_OPTION (pfile, lang) != CLK_ASM || terminator == '>')
-	cpp_error (pfile, DL_ERROR, "missing terminating %c character",
-		   (int) terminator);
-      buffer->cur--;
-    }
 }
 
 /* The stored comment includes the comment start and any terminator.  */
@@ -817,9 +830,7 @@ _cpp_lex_direct (pfile)
       /* 'L' may introduce wide characters or strings.  */
       if (*buffer->cur == '\'' || *buffer->cur == '"')
 	{
-	  result->type = (*buffer->cur == '"' ? CPP_WSTRING: CPP_WCHAR);
+	  lex_string (pfile, result, buffer->cur - 1);
-	  buffer->cur++;
-	  lex_string (pfile, result);
 	  break;
 	}
       /* Fall through.  */
@@ -848,8 +859,7 @@ _cpp_lex_direct (pfile)
 
     case '\'':
     case '"':
-      result->type = c == '"' ? CPP_STRING: CPP_CHAR;
+      lex_string (pfile, result, buffer->cur - 1);
-      lex_string (pfile, result);
       break;
 
     case '/':
@@ -905,8 +915,7 @@ _cpp_lex_direct (pfile)
     case '<':
       if (pfile->state.angled_headers)
 	{
-	  result->type = CPP_HEADER_NAME;
+	  lex_string (pfile, result, buffer->cur - 1);
-	  lex_string (pfile, result);
 	  break;
 	}
 
@@ -1078,15 +1087,8 @@ _cpp_lex_direct (pfile)
       }
 
     default:
-      {
+      create_literal (pfile, result, buffer->cur - 1, 1, CPP_OTHER);
-	uchar *dest = _cpp_unaligned_alloc (pfile, 1 + 1);
+      break;
-	dest[0] = c;
-	dest[1] = '\0';
-	result->type = CPP_OTHER;
-	result->val.str.len = 1;
-	result->val.str.text = dest;
-	break;
-      }
     }
 
   return result;
@@ -1103,8 +1105,7 @@ cpp_token_len (token)
   switch (TOKEN_SPELL (token))
     {
     default:		len = 0;				break;
-    case SPELL_NUMBER:
+    case SPELL_LITERAL:	len = token->val.str.len;		break;
-    case SPELL_STRING:	len = token->val.str.len;		break;
     case SPELL_IDENT:	len = NODE_LEN (token->val.node);	break;
     }
   /* 1 for whitespace, 4 for comment delimiters.  */
@@ -1147,34 +1148,11 @@ cpp_spell_token (pfile, token, buffer)
       buffer += NODE_LEN (token->val.node);
       break;
 
-    case SPELL_NUMBER:
+    case SPELL_LITERAL:
       memcpy (buffer, token->val.str.text, token->val.str.len);
       buffer += token->val.str.len;
       break;
 
-    case SPELL_STRING:
-      {
-	int left, right, tag;
-	switch (token->type)
-	  {
-	  case CPP_STRING:	left = '"';  right = '"';  tag = '\0'; break;
-	  case CPP_WSTRING:	left = '"';  right = '"';  tag = 'L';  break;
-	  case CPP_CHAR:	left = '\''; right = '\''; tag = '\0'; break;
-    	  case CPP_WCHAR:	left = '\''; right = '\''; tag = 'L';  break;
-	  case CPP_HEADER_NAME:	left = '<';  right = '>';  tag = '\0'; break;
-	  default:
-	    cpp_error (pfile, DL_ICE, "unknown string token %s\n",
-		       TOKEN_NAME (token));
-	    return buffer;
-	  }
-	if (tag) *buffer++ = tag;
-	*buffer++ = left;
-	memcpy (buffer, token->val.str.text, token->val.str.len);
-	buffer += token->val.str.len;
-	*buffer++ = right;
-      }
-      break;
-
     case SPELL_NONE:
       cpp_error (pfile, DL_ICE, "unspellable token %s", TOKEN_NAME (token));
       break;
@@ -1243,31 +1221,10 @@ cpp_output_token (token, fp)
       fwrite (NODE_NAME (token->val.node), 1, NODE_LEN (token->val.node), fp);
     break;
 
-    case SPELL_NUMBER:
+    case SPELL_LITERAL:
       fwrite (token->val.str.text, 1, token->val.str.len, fp);
       break;
 
-    case SPELL_STRING:
-      {
-	int left, right, tag;
-	switch (token->type)
-	  {
-	  case CPP_STRING:	left = '"';  right = '"';  tag = '\0'; break;
-	  case CPP_WSTRING:	left = '"';  right = '"';  tag = 'L';  break;
-	  case CPP_CHAR:	left = '\''; right = '\''; tag = '\0'; break;
-    	  case CPP_WCHAR:	left = '\''; right = '\''; tag = 'L';  break;
-	  case CPP_HEADER_NAME:	left = '<';  right = '>';  tag = '\0'; break;
-	  default:
-	    fprintf (stderr, "impossible STRING token %s\n", TOKEN_NAME (token));
-	    return;
-	  }
-	if (tag) putc (tag, fp);
-	putc (left, fp);
-	fwrite (token->val.str.text, 1, token->val.str.len, fp);
-	putc (right, fp);
-      }
-      break;
-
     case SPELL_NONE:
       /* An error, most probably.  */
       break;
@@ -1289,8 +1246,7 @@ _cpp_equiv_tokens (a, b)
 	return (a->type != CPP_MACRO_ARG || a->val.arg_no == b->val.arg_no);
       case SPELL_IDENT:
 	return a->val.node == b->val.node;
-      case SPELL_NUMBER:
+      case SPELL_LITERAL:
-      case SPELL_STRING:
 	return (a->val.str.len == b->val.str.len
 		&& !memcmp (a->val.str.text, b->val.str.text,
 			    a->val.str.len));
@@ -1588,14 +1544,15 @@ cpp_interpret_charconst (pfile, token, pchars_seen, unsignedp)
      unsigned int *pchars_seen;
      int *unsignedp;
 {
-  const unsigned char *str = token->val.str.text;
+  const unsigned char *str, *limit;
-  const unsigned char *limit = str + token->val.str.len;
   unsigned int chars_seen = 0;
   size_t width, max_chars;
   cppchar_t c, mask, result = 0;
   bool unsigned_p;
 
-  /* Width in bits.  */
+  str = token->val.str.text + 1 + (token->type == CPP_WCHAR);
+  limit = token->val.str.text + token->val.str.len - 1;
+
   if (token->type == CPP_CHAR)
     {
       width = CPP_OPTION (pfile, char_precision);

gcc/cpplib.c

@@ -627,9 +627,9 @@ parse_include (pfile, pangle_brackets)
   header = get_token_no_padding (pfile);
   if (header->type == CPP_STRING || header->type == CPP_HEADER_NAME)
     {
-      fname = xmalloc (header->val.str.len + 1);
+      fname = xmalloc (header->val.str.len - 1);
-      memcpy (fname, header->val.str.text, header->val.str.len);
+      memcpy (fname, header->val.str.text + 1, header->val.str.len - 2);
-      fname[header->val.str.len] = '\0';
+      fname[header->val.str.len - 2] = '\0';
       *pangle_brackets = header->type == CPP_HEADER_NAME;
     }
   else if (header->type == CPP_LESS)
@@ -832,8 +832,8 @@ do_line (pfile)
   token = cpp_get_token (pfile);
   if (token->type == CPP_STRING)
     {
-      new_file = (const char *) dequote_string (pfile, token->val.str.text,
+      new_file = (const char *) dequote_string (pfile, token->val.str.text + 1,
-						token->val.str.len);
+						token->val.str.len - 2);
       check_eol (pfile);
     }
   else if (token->type != CPP_EOF)
@@ -881,8 +881,8 @@ do_linemarker (pfile)
   token = cpp_get_token (pfile);
   if (token->type == CPP_STRING)
     {
-      new_file = (const char *) dequote_string (pfile, token->val.str.text,
+      new_file = (const char *) dequote_string (pfile, token->val.str.text + 1,
-						token->val.str.len);
+						token->val.str.len - 2);
       new_sysp = 0;
       flag = read_flag (pfile, 0);
       if (flag == 1)
@@ -1369,8 +1369,10 @@ destringize_and_run (pfile, in)
   const unsigned char *src, *limit;
   char *dest, *result;
 
-  dest = result = alloca (in->len + 1);
+  dest = result = alloca (in->len - 1);
-  for (src = in->text, limit = src + in->len; src < limit;)
+  src = in->text + 1 + (in->text[0] == 'L');
+  limit = in->text + in->len - 1;
+  while (src < limit)
     {
       /* We know there is a character following the backslash.  */
       if (*src == '\\' && (src[1] == '\\' || src[1] == '"'))

gcc/cpplib.h

@@ -124,18 +124,18 @@ struct file_name_map_list;
   OP(CPP_ATSIGN,	"@")  /* used in Objective-C */ \
 \
   TK(CPP_NAME,		SPELL_IDENT)	/* word */			\
-  TK(CPP_NUMBER,	SPELL_NUMBER)	/* 34_be+ta  */			\
+  TK(CPP_NUMBER,	SPELL_LITERAL)	/* 34_be+ta  */			\
 \
-  TK(CPP_CHAR,		SPELL_STRING)	/* 'char' */			\
+  TK(CPP_CHAR,		SPELL_LITERAL)	/* 'char' */			\
-  TK(CPP_WCHAR,		SPELL_STRING)	/* L'char' */			\
+  TK(CPP_WCHAR,		SPELL_LITERAL)	/* L'char' */			\
-  TK(CPP_OTHER,		SPELL_NUMBER)	/* stray punctuation */		\
+  TK(CPP_OTHER,		SPELL_LITERAL)	/* stray punctuation */		\
 \
-  TK(CPP_STRING,	SPELL_STRING)	/* "string" */			\
+  TK(CPP_STRING,	SPELL_LITERAL)	/* "string" */			\
-  TK(CPP_WSTRING,	SPELL_STRING)	/* L"string" */			\
+  TK(CPP_WSTRING,	SPELL_LITERAL)	/* L"string" */			\
-  TK(CPP_HEADER_NAME,	SPELL_STRING)	/* <stdio.h> in #include */	\
+  TK(CPP_HEADER_NAME,	SPELL_LITERAL)	/* <stdio.h> in #include */	\
 \
-  TK(CPP_COMMENT,	SPELL_NUMBER)	/* Only if output comments.  */ \
+  TK(CPP_COMMENT,	SPELL_LITERAL)	/* Only if output comments.  */ \
-                                        /* SPELL_NUMBER happens to DTRT.  */ \
+                                        /* SPELL_LITERAL happens to DTRT.  */ \
   TK(CPP_MACRO_ARG,	SPELL_NONE)	/* Macro argument.  */		\
   TK(CPP_PADDING,	SPELL_NONE)	/* Whitespace for cpp0.  */
 

gcc/cppmacro.c

@@ -340,11 +340,16 @@ stringify_arg (pfile, arg)
      cpp_reader *pfile;
      macro_arg *arg;
 {
-  unsigned char *dest = BUFF_FRONT (pfile->u_buff);
+  unsigned char *dest;
   unsigned int i, escape_it, backslash_count = 0;
   const cpp_token *source = NULL;
   size_t len;
 
+  if (BUFF_ROOM (pfile->u_buff) < 3)
+    _cpp_extend_buff (pfile, &pfile->u_buff, 3);
+  dest = BUFF_FRONT (pfile->u_buff);
+  *dest++ = '"';
+
   /* Loop, reading in the argument's tokens.  */
   for (i = 0; i < arg->count; i++)
     {
@@ -361,11 +366,11 @@ stringify_arg (pfile, arg)
 		   || token->type == CPP_CHAR || token->type == CPP_WCHAR);
 
       /* Room for each char being written in octal, initial space and
-	 final NUL.  */
+	 final quote and NUL.  */
       len = cpp_token_len (token);
       if (escape_it)
 	len *= 4;
-      len += 2;
+      len += 3;
 
       if ((size_t) (BUFF_LIMIT (pfile->u_buff) - dest) < len)
 	{
@@ -375,7 +380,7 @@ stringify_arg (pfile, arg)
 	}
 
       /* Leading white space?  */
-      if (dest != BUFF_FRONT (pfile->u_buff))
+      if (dest - 1 != BUFF_FRONT (pfile->u_buff))
 	{
 	  if (source == NULL)
 	    source = token;
@@ -410,12 +415,7 @@ stringify_arg (pfile, arg)
     }
 
   /* Commit the memory, including NUL, and return the token.  */
-  if ((size_t) (BUFF_LIMIT (pfile->u_buff) - dest) < 1)
+  *dest++ = '"';
-    {
-      size_t len_so_far = dest - BUFF_FRONT (pfile->u_buff);
-      _cpp_extend_buff (pfile, &pfile->u_buff, 1);
-      dest = BUFF_FRONT (pfile->u_buff) + len_so_far;
-    }
   len = dest - BUFF_FRONT (pfile->u_buff);
   BUFF_FRONT (pfile->u_buff) = dest + 1;
   return new_string_token (pfile, dest - len, len);
@@ -1638,10 +1638,11 @@ check_trad_stringification (pfile, macro, string)
      const cpp_string *string;
 {
   unsigned int i, len;
-  const uchar *p, *q, *limit = string->text + string->len;
+  const uchar *p, *q, *limit;
 
   /* Loop over the string.  */
-  for (p = string->text; p < limit; p = q)
+  limit = string->text + string->len - 1;
+  for (p = string->text + 1; p < limit; p = q)
     {
       /* Find the start of an identifier.  */
       while (p < limit && !is_idstart (*p))

gcc/testsuite/ChangeLog

@@ -1,3 +1,11 @@
+2003-04-23  Neil Booth  <neil@daikokuya.co.uk>
+
+	* gcc.dg/cpp/include2.c: Update.
+	* gcc.dg/cpp/multiline-2.c: New.
+	* gcc.dg/cpp/multiline.c: Update.
+	* gcc.dg/cpp/strify2.c: Update.
+	* gcc.dg/cpp/trad/literals-2.c: Update.
+
 2003-04-23  John David Anglin  <dave.anglin@nrc-cnrc.gc.ca>
 
 	* g++.dg/other/packed1.C: XFAIL hppa*-*-*.

gcc/testsuite/gcc.dg/cpp/include2.c

@@ -8,10 +8,9 @@
 /* Source: Neil Booth, 4 Nov 2000.  */
 
 #include <silly\>>  /* { dg-warning "extra tokens" "" } */
-#include "silly\""  /* { dg-error "missing" "" } */
+#include "silly\""  /* { dg-warning "extra tokens" "" } */
 
 /* These first 2 errors are No such file or directory.  However, this
    message is locale-dependent, so don't test for it.  */
 /* { dg-error "silly" "" { target *-*-* } 10 } */
 /* { dg-error "silly" "" { target *-*-* } 11 } */
-/* { dg-warning "extra tokens" "" { target *-*-* } 11 } */

gcc/testsuite/gcc.dg/cpp/multiline-2.c

@@ -0,0 +1,14 @@
+/* Copyright (C) 2000, 2003 Free Software Foundation, Inc.  */
+
+/* { dg-do compile } */
+
+/* Test that multi-line tokens are rejected by the compiler.  Source:
+   Neil Booth.  */
+
+const char *p = "line 1
+"
+"";			      /* The compiler front end sees this.  */
+
+/* { dg-error "missing term" "multiline strings" { target *-*-* } 8 } */
+/* { dg-error "missing term" "multiline strings" { target *-*-* } 9 } */
+

gcc/testsuite/gcc.dg/cpp/multiline.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2000 Free Software Foundation, Inc.  */
+/* Copyright (C) 2000, 2003 Free Software Foundation, Inc.  */
 /* { dg-do preprocess } */
 /* { dg-options "-C" } */
 /* Test that multi-line tokens are recognized by cpp0 as being
@@ -22,11 +22,5 @@ L"line 1
    { dg-final { if \{ [grep multiline.i "^$"] == "" \} \{               } }
    { dg-final { return \}                                               } }
    { dg-final { fail "multiline.c: multi-line tokens"                   } } */
-/* { dg-error "missing term" "multiline strings" { target *-*-* } 11 } */
+/* { dg-bogus "missing term" "multiline strings" { target *-*-* } 11 } */
-/* { dg-error "missing term" "multiline strings" { target *-*-* } 14 } */
-/* { dg-error "missing term" "multiline strings" { target *-*-* } 15 } */
-/* { dg-error "missing term" "multiline strings" { target *-*-* } 18 } */
-/* { dg-bogus "warning" "warning in place of error" { target *-*-* } 11 } */
-/* { dg-bogus "warning" "warning in place of error" { target *-*-* } 14 } */
 /* { dg-bogus "warning" "warning in place of error" { target *-*-* } 15 } */
-/* { dg-bogus "warning" "warning in place of error" { target *-*-* } 18 } */

gcc/testsuite/gcc.dg/cpp/strify2.c

@@ -1,7 +1,7 @@
 /* Copyright (C) 2000 Free Software Foundation, Inc.  */
 
 /* { dg-do run } */
-/* { dg-options "-std=c99 -pedantic-errors" } */
+/* { dg-options "-std=c99 -pedantic-errors -fno-show-column" } */
 
 /* Tests a whole bunch of things are correctly stringified.  */
 

gcc/testsuite/gcc.dg/cpp/trad/literals-2.c

@@ -3,6 +3,6 @@
 
 /* { dg-do preprocess } */
 
-/* { dg-error "missing terminating" "bad charconst" { target *-*-* } 7 } */
+/* { dg-error "not valid" "bad charconst" { target *-*-* } 7 } */
 #if 'x
 #endif