// -*- C++ -*-
/* Copyright (C) 1989, 1990, 1991, 1992, 2000, 2001, 2002, 2003, 2004, 2005
   Free Software Foundation, Inc.
     Written by James Clark (jjc@jclark.com)

This file is part of groff.

groff is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 2, or (at your option) any later
version.

groff is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License along
with groff; see the file COPYING.  If not, write to the Free Software
Foundation, 51 Franklin St - Fifth Floor, Boston, MA 02110-1301, USA. */

#define DEBUGGING

#include "troff.h"
#include "dictionary.h"
#include "hvunits.h"
#include "stringclass.h"
#include "mtsm.h"
#include "env.h"
#include "request.h"
#include "node.h"
#include "token.h"
#include "div.h"
#include "reg.h"
#include "charinfo.h"
#include "macropath.h"
#include "input.h"
#include "defs.h"
#include "font.h"
#include "unicode.h"

// Needed for getpid() and isatty()
#include "posix.h"

#include "nonposix.h"

#ifdef NEED_DECLARATION_PUTENV
extern "C" {
  int putenv(const char *);
}
#endif /* NEED_DECLARATION_PUTENV */

#define MACRO_PREFIX "tmac."
#define MACRO_POSTFIX ".tmac"
#define INITIAL_STARTUP_FILE "troffrc"
#define FINAL_STARTUP_FILE   "troffrc-end"
#define DEFAULT_INPUT_STACK_LIMIT 1000

#ifndef DEFAULT_WARNING_MASK
// warnings that are enabled by default
#define DEFAULT_WARNING_MASK \
     (WARN_CHAR|WARN_NUMBER|WARN_BREAK|WARN_SPACE|WARN_FONT)
#endif

// initial size of buffer for reading names; expanded as necessary
#define ABUF_SIZE 16

extern "C" const char *program_name;
extern "C" const char *Version_string;

#ifdef COLUMN
void init_column_requests();
#endif /* COLUMN */

static node *read_draw_node();
static void read_color_draw_node(token &);
static void push_token(const token &);
void copy_file();
#ifdef COLUMN
void vjustify();
#endif /* COLUMN */
void transparent_file();

token tok;
int break_flag = 0;
int color_flag = 1;		// colors are on by default
static int backtrace_flag = 0;
#ifndef POPEN_MISSING
char *pipe_command = 0;
#endif
charinfo *charset_table[256];
unsigned char hpf_code_table[256];

static int warning_mask = DEFAULT_WARNING_MASK;
static int inhibit_errors = 0;
static int ignoring = 0;

static void enable_warning(const char *);
static void disable_warning(const char *);

static int escape_char = '\\';
static symbol end_macro_name;
static symbol blank_line_macro_name;
static int compatible_flag = 0;
int ascii_output_flag = 0;
int suppress_output_flag = 0;
int is_html = 0;
int begin_level = 0;		// number of nested \O escapes

int have_input = 0;		// whether \f, \F, \D'F...', \H, \m, \M,
				// \R, \s, or \S has been processed in
				// token::next()
int old_have_input = 0;		// value of have_input right before \n
int tcommand_flag = 0;
int safer_flag = 1;		// safer by default

int have_string_arg = 0;	// whether we have \*[foo bar...]

double spread_limit = -3.0 - 1.0;	// negative means deactivated

double warn_scale;
char warn_scaling_indicator;
int debug_state = 0;            // turns on debugging of the html troff state

search_path *mac_path = &safer_macro_path;

// Defaults to the current directory.
search_path include_search_path(0, 0, 0, 1);

static int get_copy(node**, int = 0);
static void copy_mode_error(const char *,
			    const errarg & = empty_errarg,
			    const errarg & = empty_errarg,
			    const errarg & = empty_errarg);

enum read_mode { ALLOW_EMPTY, WITH_ARGS, NO_ARGS };
static symbol read_escape_name(read_mode mode = NO_ARGS);
static symbol read_long_escape_name(read_mode mode = NO_ARGS);
static void interpolate_string(symbol);
static void interpolate_string_with_args(symbol);
static void interpolate_macro(symbol);
static void interpolate_number_format(symbol);
static void interpolate_environment_variable(symbol);

static symbol composite_glyph_name(symbol);
static void interpolate_arg(symbol);
static request_or_macro *lookup_request(symbol);
static int get_delim_number(units *, unsigned char);
static int get_delim_number(units *, unsigned char, units);
static symbol do_get_long_name(int, char);
static int get_line_arg(units *res, unsigned char si, charinfo **cp);
static int read_size(int *);
static symbol get_delim_name();
static void init_registers();
static void trapping_blank_line();

class input_iterator;
input_iterator *make_temp_iterator(const char *);
const char *input_char_description(int);

void process_input_stack();
void chop_macro();		// declare to avoid friend name injection


void set_escape_char()
{
  if (has_arg()) {
    if (tok.ch() == 0) {
      error("bad escape character");
      escape_char = '\\';
    }
    else
      escape_char = tok.ch();
  }
  else
    escape_char = '\\';
  skip_line();
}

void escape_off()
{
  escape_char = 0;
  skip_line();
}

static int saved_escape_char = '\\';

void save_escape_char()
{
  saved_escape_char = escape_char;
  skip_line();
}

void restore_escape_char()
{
  escape_char = saved_escape_char;
  skip_line();
}

class input_iterator {
public:
  input_iterator();
  input_iterator(int is_div);
  virtual ~input_iterator() {}
  int get(node **);
  friend class input_stack;
  int is_diversion;
  statem *diversion_state;
protected:
  const unsigned char *ptr;
  const unsigned char *eptr;
  input_iterator *next;
private:
  virtual int fill(node **);
  virtual int peek();
  virtual int has_args() { return 0; }
  virtual int nargs() { return 0; }
  virtual input_iterator *get_arg(int) { return 0; }
  virtual int get_location(int, const char **, int *) { return 0; }
  virtual void backtrace() {}
  virtual int set_location(const char *, int) { return 0; }
  virtual int next_file(FILE *, const char *) { return 0; }
  virtual void shift(int) {}
  virtual int is_boundary() {return 0; }
  virtual int is_file() { return 0; }
  virtual int is_macro() { return 0; }
  virtual void save_compatible_flag(int) {}
  virtual int get_compatible_flag() { return 0; }
};

input_iterator::input_iterator()
: is_diversion(0), ptr(0), eptr(0)
{
}

input_iterator::input_iterator(int is_div)
: is_diversion(is_div), ptr(0), eptr(0)
{
}

int input_iterator::fill(node **)
{
  return EOF;
}

int input_iterator::peek()
{
  return EOF;
}

inline int input_iterator::get(node **p)
{
  return ptr < eptr ? *ptr++ : fill(p);
}

class input_boundary : public input_iterator {
public:
  int is_boundary() { return 1; }
};

class input_return_boundary : public input_iterator {
public:
  int is_boundary() { return 2; }
};

class file_iterator : public input_iterator {
  FILE *fp;
  int lineno;
  const char *filename;
  int popened;
  int newline_flag;
  int seen_escape;
  enum { BUF_SIZE = 512 };
  unsigned char buf[BUF_SIZE];
  void close();
public:
  file_iterator(FILE *, const char *, int = 0);
  ~file_iterator();
  int fill(node **);
  int peek();
  int get_location(int, const char **, int *);
  void backtrace();
  int set_location(const char *, int);
  int next_file(FILE *, const char *);
  int is_file();
};

file_iterator::file_iterator(FILE *f, const char *fn, int po)
: fp(f), lineno(1), filename(fn), popened(po),
  newline_flag(0), seen_escape(0)
{
  if ((font::use_charnames_in_special) && (fn != 0)) {
    if (!the_output)
      init_output();
    the_output->put_filename(fn);
  }
}

file_iterator::~file_iterator()
{
  close();
}

void file_iterator::close()
{
  if (fp == stdin)
    clearerr(stdin);
#ifndef POPEN_MISSING
  else if (popened)
    pclose(fp);
#endif /* not POPEN_MISSING */
  else
    fclose(fp);
}

int file_iterator::is_file()
{
  return 1;
}

int file_iterator::next_file(FILE *f, const char *s)
{
  close();
  filename = s;
  fp = f;
  lineno = 1;
  newline_flag = 0;
  seen_escape = 0;
  popened = 0;
  ptr = 0;
  eptr = 0;
  return 1;
}

int file_iterator::fill(node **)
{
  if (newline_flag)
    lineno++;
  newline_flag = 0;
  unsigned char *p = buf;
  ptr = p;
  unsigned char *e = p + BUF_SIZE;
  while (p < e) {
    int c = getc(fp);
    if (c == EOF)
      break;
    if (invalid_input_char(c))
      warning(WARN_INPUT, "invalid input character code %1", int(c));
    else {
      *p++ = c;
      if (c == '\n') {
	seen_escape = 0;
	newline_flag = 1;
	break;
      }
      seen_escape = (c == '\\');
    }
  }
  if (p > buf) {
    eptr = p;
    return *ptr++;
  }
  else {
    eptr = p;
    return EOF;
  }
}

int file_iterator::peek()
{
  int c = getc(fp);
  while (invalid_input_char(c)) {
    warning(WARN_INPUT, "invalid input character code %1", int(c));
    c = getc(fp);
  }
  if (c != EOF)
    ungetc(c, fp);
  return c;
}

int file_iterator::get_location(int /*allow_macro*/,
				const char **filenamep, int *linenop)
{
  *linenop = lineno;
  if (filename != 0 && strcmp(filename, "-") == 0)
    *filenamep = "<standard input>";
  else
    *filenamep = filename;
  return 1;
}

void file_iterator::backtrace()
{
  errprint("%1:%2: backtrace: %3 `%1'\n", filename, lineno,
	   popened ? "process" : "file");
}

int file_iterator::set_location(const char *f, int ln)
{
  if (f) {
    filename = f;
    if (!the_output)
      init_output();
    the_output->put_filename(f);
  }
  lineno = ln;
  return 1;
}

input_iterator nil_iterator;

class input_stack {
public:
  static int get(node **);
  static int peek();
  static void push(input_iterator *);
  static input_iterator *get_arg(int);
  static int nargs();
  static int get_location(int, const char **, int *);
  static int set_location(const char *, int);
  static void backtrace();
  static void backtrace_all();
  static void next_file(FILE *, const char *);
  static void end_file();
  static void shift(int n);
  static void add_boundary();
  static void add_return_boundary();
  static int is_return_boundary();
  static void remove_boundary();
  static int get_level();
  static int get_div_level();
  static void increase_level();
  static void decrease_level();
  static void clear();
  static void pop_macro();
  static void save_compatible_flag(int);
  static int get_compatible_flag();
  static statem *get_diversion_state();
  static void check_end_diversion(input_iterator *t);
  static int limit;
  static int div_level;
  static statem *diversion_state;
private:
  static input_iterator *top;
  static int level;
  static int finish_get(node **);
  static int finish_peek();
};

input_iterator *input_stack::top = &nil_iterator;
int input_stack::level = 0;
int input_stack::limit = DEFAULT_INPUT_STACK_LIMIT;
int input_stack::div_level = 0;
statem *input_stack::diversion_state = NULL;
int suppress_push=0;


inline int input_stack::get_level()
{
  return level;
}

inline void input_stack::increase_level()
{
  level++;
}

inline void input_stack::decrease_level()
{
  level--;
}

inline int input_stack::get_div_level()
{
  return div_level;
}

inline int input_stack::get(node **np)
{
  int res = (top->ptr < top->eptr) ? *top->ptr++ : finish_get(np);
  if (res == '\n') {
    old_have_input = have_input;
    have_input = 0;
  }
  return res;
}

int input_stack::finish_get(node **np)
{
  for (;;) {
    int c = top->fill(np);
    if (c != EOF || top->is_boundary())
      return c;
    if (top == &nil_iterator)
      break;
    input_iterator *tem = top;
    check_end_diversion(tem);
#if defined(DEBUGGING)
  if (debug_state)
    if (tem->is_diversion)
      fprintf(stderr,
	      "in diversion level = %d\n", input_stack::get_div_level());
#endif
    top = top->next;
    level--;
    delete tem;
    if (top->ptr < top->eptr)
      return *top->ptr++;
  }
  assert(level == 0);
  return EOF;
}

inline int input_stack::peek()
{
  return (top->ptr < top->eptr) ? *top->ptr : finish_peek();
}

void input_stack::check_end_diversion(input_iterator *t)
{
  if (t->is_diversion) {
    div_level--;
    diversion_state = t->diversion_state;
  }
}

int input_stack::finish_peek()
{
  for (;;) {
    int c = top->peek();
    if (c != EOF || top->is_boundary())
      return c;
    if (top == &nil_iterator)
      break;
    input_iterator *tem = top;
    check_end_diversion(tem);
    top = top->next;
    level--;
    delete tem;
    if (top->ptr < top->eptr)
      return *top->ptr;
  }
  assert(level == 0);
  return EOF;
}

void input_stack::add_boundary()
{
  push(new input_boundary);
}

void input_stack::add_return_boundary()
{
  push(new input_return_boundary);
}

int input_stack::is_return_boundary()
{
  return top->is_boundary() == 2;
}

void input_stack::remove_boundary()
{
  assert(top->is_boundary());
  input_iterator *temp = top->next;
  check_end_diversion(top);

  delete top;
  top = temp;
  level--;
}

void input_stack::push(input_iterator *in)
{
  if (in == 0)
    return;
  if (++level > limit && limit > 0)
    fatal("input stack limit exceeded (probable infinite loop)");
  in->next = top;
  top = in;
  if (top->is_diversion) {
    div_level++;
    in->diversion_state = diversion_state;
    diversion_state = curenv->construct_state(0);
#if defined(DEBUGGING)
    if (debug_state) {
      curenv->dump_troff_state();
      fflush(stderr);
    }
#endif
  }
#if defined(DEBUGGING)
  if (debug_state)
    if (top->is_diversion) {
      fprintf(stderr,
	      "in diversion level = %d\n", input_stack::get_div_level());
      fflush(stderr);
    }
#endif
}

statem *get_diversion_state()
{
  return input_stack::get_diversion_state();
}

statem *input_stack::get_diversion_state()
{
  if (diversion_state == NULL)
    return NULL;
  else
    return new statem(diversion_state);
}

input_iterator *input_stack::get_arg(int i)
{
  input_iterator *p;
  for (p = top; p != 0; p = p->next)
    if (p->has_args())
      return p->get_arg(i);
  return 0;
}

void input_stack::shift(int n)
{
  for (input_iterator *p = top; p; p = p->next)
    if (p->has_args()) {
      p->shift(n);
      return;
    }
}

int input_stack::nargs()
{
  for (input_iterator *p =top; p != 0; p = p->next)
    if (p->has_args())
      return p->nargs();
  return 0;
}

int input_stack::get_location(int allow_macro, const char **filenamep, int *linenop)
{
  for (input_iterator *p = top; p; p = p->next)
    if (p->get_location(allow_macro, filenamep, linenop))
      return 1;
  return 0;
}

void input_stack::backtrace()
{
  const char *f;
  int n;
  // only backtrace down to (not including) the topmost file
  for (input_iterator *p = top;
       p && !p->get_location(0, &f, &n);
       p = p->next)
    p->backtrace();
}

void input_stack::backtrace_all()
{
  for (input_iterator *p = top; p; p = p->next)
    p->backtrace();
}

int input_stack::set_location(const char *filename, int lineno)
{
  for (input_iterator *p = top; p; p = p->next)
    if (p->set_location(filename, lineno))
      return 1;
  return 0;
}

void input_stack::next_file(FILE *fp, const char *s)
{
  input_iterator **pp;
  for (pp = &top; *pp != &nil_iterator; pp = &(*pp)->next)
    if ((*pp)->next_file(fp, s))
      return;
  if (++level > limit && limit > 0)
    fatal("input stack limit exceeded");
  *pp = new file_iterator(fp, s);
  (*pp)->next = &nil_iterator;
}

void input_stack::end_file()
{
  for (input_iterator **pp = &top; *pp != &nil_iterator; pp = &(*pp)->next)
    if ((*pp)->is_file()) {
      input_iterator *tem = *pp;
      check_end_diversion(tem);
      *pp = (*pp)->next;
      delete tem;
      level--;
      return;
    }
}

void input_stack::clear()
{
  int nboundaries = 0;
  while (top != &nil_iterator) {
    if (top->is_boundary())
      nboundaries++;
    input_iterator *tem = top;
    check_end_diversion(tem);
    top = top->next;
    level--;
    delete tem;
  }
  // Keep while_request happy.
  for (; nboundaries > 0; --nboundaries)
    add_return_boundary();
}

void input_stack::pop_macro()
{
  int nboundaries = 0;
  int is_macro = 0;
  do {
    if (top->next == &nil_iterator)
      break;
    if (top->is_boundary())
      nboundaries++;
    is_macro = top->is_macro();
    input_iterator *tem = top;
    check_end_diversion(tem);
    top = top->next;
    level--;
    delete tem;
  } while (!is_macro);
  // Keep while_request happy.
  for (; nboundaries > 0; --nboundaries)
    add_return_boundary();
}

inline void input_stack::save_compatible_flag(int f)
{
  top->save_compatible_flag(f);
}

inline int input_stack::get_compatible_flag()
{
  return top->get_compatible_flag();
}

void backtrace_request()
{
  input_stack::backtrace_all();
  fflush(stderr);
  skip_line();
}

void next_file()
{
  symbol nm = get_long_name();
  while (!tok.newline() && !tok.eof())
    tok.next();
  if (nm.is_null())
    input_stack::end_file();
  else {
    errno = 0;
    FILE *fp = include_search_path.open_file_cautious(nm.contents());
    if (!fp)
      error("can't open `%1': %2", nm.contents(), strerror(errno));
    else
      input_stack::next_file(fp, nm.contents());
  }
  tok.next();
}

void shift()
{
  int n;
  if (!has_arg() || !get_integer(&n))
    n = 1;
  input_stack::shift(n);
  skip_line();
}

static char get_char_for_escape_name(int allow_space = 0)
{
  int c = get_copy(0);
  switch (c) {
  case EOF:
    copy_mode_error("end of input in escape name");
    return '\0';
  default:
    if (!invalid_input_char(c))
      break;
    // fall through
  case '\n':
    if (c == '\n')
      input_stack::push(make_temp_iterator("\n"));
    // fall through
  case ' ':
    if (c == ' ' && allow_space)
      break;
    // fall through
  case '\t':
  case '\001':
  case '\b':
    copy_mode_error("%1 is not allowed in an escape name",
		    input_char_description(c));
    return '\0';
  }
  return c;
}

static symbol read_two_char_escape_name()
{
  char buf[3];
  buf[0] = get_char_for_escape_name();
  if (buf[0] != '\0') {
    buf[1] = get_char_for_escape_name();
    if (buf[1] == '\0')
      buf[0] = 0;
    else
      buf[2] = 0;
  }
  return symbol(buf);
}

static symbol read_long_escape_name(read_mode mode)
{
  int start_level = input_stack::get_level();
  char abuf[ABUF_SIZE];
  char *buf = abuf;
  int buf_size = ABUF_SIZE;
  int i = 0;
  char c;
  int have_char = 0;
  for (;;) {
    c = get_char_for_escape_name(have_char && mode == WITH_ARGS);
    if (c == 0) {
      if (buf != abuf)
	a_delete buf;
      return NULL_SYMBOL;
    }
    have_char = 1;
    if (mode == WITH_ARGS && c == ' ')
      break;
    if (i + 2 > buf_size) {
      if (buf == abuf) {
	buf = new char[ABUF_SIZE*2];
	memcpy(buf, abuf, buf_size);
	buf_size = ABUF_SIZE*2;
      }
      else {
	char *old_buf = buf;
	buf = new char[buf_size*2];
	memcpy(buf, old_buf, buf_size);
	buf_size *= 2;
	a_delete old_buf;
      }
    }
    if (c == ']' && input_stack::get_level() == start_level)
      break;
    buf[i++] = c;
  }
  buf[i] = 0;
  if (c == ' ')
    have_string_arg = 1;
  if (buf == abuf) {
    if (i == 0) {
      if (mode != ALLOW_EMPTY)
        copy_mode_error("empty escape name");
      return EMPTY_SYMBOL;
    }
    return symbol(abuf);
  }
  else {
    symbol s(buf);
    a_delete buf;
    return s;
  }
}

static symbol read_escape_name(read_mode mode)
{
  char c = get_char_for_escape_name();
  if (c == 0)
    return NULL_SYMBOL;
  if (c == '(')
    return read_two_char_escape_name();
  if (c == '[' && !compatible_flag)
    return read_long_escape_name(mode);
  char buf[2];
  buf[0] = c;
  buf[1] = '\0';
  return symbol(buf);
}

static symbol read_increment_and_escape_name(int *incp)
{
  char c = get_char_for_escape_name();
  switch (c) {
  case 0:
    *incp = 0;
    return NULL_SYMBOL;
  case '(':
    *incp = 0;
    return read_two_char_escape_name();
  case '+':
    *incp = 1;
    return read_escape_name();
  case '-':
    *incp = -1;
    return read_escape_name();
  case '[':
    if (!compatible_flag) {
      *incp = 0;
      return read_long_escape_name();
    }
    break;
  }
  *incp = 0;
  char buf[2];
  buf[0] = c;
  buf[1] = '\0';
  return symbol(buf);
}

static int get_copy(node **nd, int defining)
{
  for (;;) {
    int c = input_stack::get(nd);
    if (c == PUSH_GROFF_MODE) {
      input_stack::save_compatible_flag(compatible_flag);
      compatible_flag = 0;
      continue;
    }
    if (c == PUSH_COMP_MODE) {
      input_stack::save_compatible_flag(compatible_flag);
      compatible_flag = 1;
      continue;
    }
    if (c == POP_GROFFCOMP_MODE) {
      compatible_flag = input_stack::get_compatible_flag();
      continue;
    }
    if (c == BEGIN_QUOTE) {
      input_stack::increase_level();
      continue;
    }
    if (c == END_QUOTE) {
      input_stack::decrease_level();
      continue;
    }
    if (c == ESCAPE_NEWLINE) {
      if (defining)
	return c;
      do {
	c = input_stack::get(nd);
      } while (c == ESCAPE_NEWLINE);
    }
    if (c != escape_char || escape_char <= 0)
      return c;
    c = input_stack::peek();
    switch(c) {
    case 0:
      return escape_char;
    case '"':
      (void)input_stack::get(0);
      while ((c = input_stack::get(0)) != '\n' && c != EOF)
	;
      return c;
    case '#':			// Like \" but newline is ignored.
      (void)input_stack::get(0);
      while ((c = input_stack::get(0)) != '\n')
	if (c == EOF)
	  return EOF;
      break;
    case '$':
      {
	(void)input_stack::get(0);
	symbol s = read_escape_name();
	if (!(s.is_null() || s.is_empty()))
	  interpolate_arg(s);
	break;
      }
    case '*':
      {
	(void)input_stack::get(0);
	symbol s = read_escape_name(WITH_ARGS);
	if (!(s.is_null() || s.is_empty())) {
	  if (have_string_arg) {
	    have_string_arg = 0;
	    interpolate_string_with_args(s);
	  }
	  else
	    interpolate_string(s);
	}
	break;
      }
    case 'a':
      (void)input_stack::get(0);
      return '\001';
    case 'e':
      (void)input_stack::get(0);
      return ESCAPE_e;
    case 'E':
      (void)input_stack::get(0);
      return ESCAPE_E;
    case 'n':
      {
	(void)input_stack::get(0);
	int inc;
	symbol s = read_increment_and_escape_name(&inc);
	if (!(s.is_null() || s.is_empty()))
	  interpolate_number_reg(s, inc);
	break;
      }
    case 'g':
      {
	(void)input_stack::get(0);
	symbol s = read_escape_name();
	if (!(s.is_null() || s.is_empty()))
	  interpolate_number_format(s);
	break;
      }
    case 't':
      (void)input_stack::get(0);
      return '\t';
    case 'V':
      {
	(void)input_stack::get(0);
	symbol s = read_escape_name();
	if (!(s.is_null() || s.is_empty()))
	  interpolate_environment_variable(s);
	break;
      }
    case '\n':
      (void)input_stack::get(0);
      if (defining)
	return ESCAPE_NEWLINE;
      break;
    case ' ':
      (void)input_stack::get(0);
      return ESCAPE_SPACE;
    case '~':
      (void)input_stack::get(0);
      return ESCAPE_TILDE;
    case ':':
      (void)input_stack::get(0);
      return ESCAPE_COLON;
    case '|':
      (void)input_stack::get(0);
      return ESCAPE_BAR;
    case '^':
      (void)input_stack::get(0);
      return ESCAPE_CIRCUMFLEX;
    case '{':
      (void)input_stack::get(0);
      return ESCAPE_LEFT_BRACE;
    case '}':
      (void)input_stack::get(0);
      return ESCAPE_RIGHT_BRACE;
    case '`':
      (void)input_stack::get(0);
      return ESCAPE_LEFT_QUOTE;
    case '\'':
      (void)input_stack::get(0);
      return ESCAPE_RIGHT_QUOTE;
    case '-':
      (void)input_stack::get(0);
      return ESCAPE_HYPHEN;
    case '_':
      (void)input_stack::get(0);
      return ESCAPE_UNDERSCORE;
    case 'c':
      (void)input_stack::get(0);
      return ESCAPE_c;
    case '!':
      (void)input_stack::get(0);
      return ESCAPE_BANG;
    case '?':
      (void)input_stack::get(0);
      return ESCAPE_QUESTION;
    case '&':
      (void)input_stack::get(0);
      return ESCAPE_AMPERSAND;
    case ')':
      (void)input_stack::get(0);
      return ESCAPE_RIGHT_PARENTHESIS;
    case '.':
      (void)input_stack::get(0);
      return c;			
    case '%':
      (void)input_stack::get(0);
      return ESCAPE_PERCENT;
    default:
      if (c == escape_char) {
	(void)input_stack::get(0);
	return c;
      }
      else
	return escape_char;
    }
  }
}

class non_interpreted_char_node : public node {
  unsigned char c;
public:
  non_interpreted_char_node(unsigned char);
  node *copy();
  int interpret(macro *);
  int same(node *);
  const char *type();
  int force_tprint();
  int is_tag();
};

int non_interpreted_char_node::same(node *nd)
{
  return c == ((non_interpreted_char_node *)nd)->c;
}

const char *non_interpreted_char_node::type()
{
  return "non_interpreted_char_node";
}

int non_interpreted_char_node::force_tprint()
{
  return 0;
}

int non_interpreted_char_node::is_tag()
{
  return 0;
}

non_interpreted_char_node::non_interpreted_char_node(unsigned char n) : c(n)
{
  assert(n != 0);
}

node *non_interpreted_char_node::copy()
{
  return new non_interpreted_char_node(c);
}

int non_interpreted_char_node::interpret(macro *mac)
{
  mac->append(c);
  return 1;
}

static void do_width();
static node *do_non_interpreted();
static node *do_special();
static node *do_suppress(symbol nm);
static void do_register();

dictionary color_dictionary(501);

static color *lookup_color(symbol nm)
{
  assert(!nm.is_null());
  if (nm == default_symbol)
    return &default_color;
  color *c = (color *)color_dictionary.lookup(nm);
  if (c == 0)
    warning(WARN_COLOR, "color `%1' not defined", nm.contents());
  return c;
}

void do_glyph_color(symbol nm)
{
  if (nm.is_null())
    return;
  if (nm.is_empty())
    curenv->set_glyph_color(curenv->get_prev_glyph_color());
  else {
    color *tem = lookup_color(nm);
    if (tem)
      curenv->set_glyph_color(tem);
    else
      (void)color_dictionary.lookup(nm, new color(nm));
  }
}

void do_fill_color(symbol nm)
{
  if (nm.is_null())
    return;
  if (nm.is_empty())
    curenv->set_fill_color(curenv->get_prev_fill_color());
  else {
    color *tem = lookup_color(nm);
    if (tem)
      curenv->set_fill_color(tem);
    else
      (void)color_dictionary.lookup(nm, new color(nm));
  }
}

static unsigned int get_color_element(const char *scheme, const char *col)
{
  units val;
  if (!get_number(&val, 'f')) {
    warning(WARN_COLOR, "%1 in %2 definition set to 0", col, scheme);
    tok.next();
    return 0;
  }
  if (val < 0) {
    warning(WARN_RANGE, "%1 cannot be negative: set to 0", col);
    return 0;
  }
  if (val > color::MAX_COLOR_VAL+1) {
    warning(WARN_RANGE, "%1 cannot be greater than 1", col);
    // we change 0x10000 to 0xffff
    return color::MAX_COLOR_VAL;
  }
  return (unsigned int)val;
}

static color *read_rgb(char end = 0)
{
  symbol component = do_get_long_name(0, end);
  if (component.is_null()) {
    warning(WARN_COLOR, "missing rgb color values");
    return 0;
  }
  const char *s = component.contents();
  color *col = new color;
  if (*s == '#') {
    if (!col->read_rgb(s)) {
      warning(WARN_COLOR, "expecting rgb color definition not `%1'", s);
      delete col;
      return 0;
    }
  }
  else {
    if (!end)
      input_stack::push(make_temp_iterator(" "));
    input_stack::push(make_temp_iterator(s));
    tok.next();
    unsigned int r = get_color_element("rgb color", "red component");
    unsigned int g = get_color_element("rgb color", "green component");
    unsigned int b = get_color_element("rgb color", "blue component");
    col->set_rgb(r, g, b);
  }
  return col;
}

static color *read_cmy(char end = 0)
{
  symbol component = do_get_long_name(0, end);
  if (component.is_null()) {
    warning(WARN_COLOR, "missing cmy color values");
    return 0;
  }
  const char *s = component.contents();
  color *col = new color;
  if (*s == '#') {
    if (!col->read_cmy(s)) {
      warning(WARN_COLOR, "expecting cmy color definition not `%1'", s);
      delete col;
      return 0;
    }
  }
  else {
    if (!end)
      input_stack::push(make_temp_iterator(" "));
    input_stack::push(make_temp_iterator(s));
    tok.next();
    unsigned int c = get_color_element("cmy color", "cyan component");
    unsigned int m = get_color_element("cmy color", "magenta component");
    unsigned int y = get_color_element("cmy color", "yellow component");
    col->set_cmy(c, m, y);
  }
  return col;
}

static color *read_cmyk(char end = 0)
{
  symbol component = do_get_long_name(0, end);
  if (component.is_null()) {
    warning(WARN_COLOR, "missing cmyk color values");
    return 0;
  }
  const char *s = component.contents();
  color *col = new color;
  if (*s == '#') {
    if (!col->read_cmyk(s)) {
      warning(WARN_COLOR, "`expecting a cmyk color definition not `%1'", s);
      delete col;
      return 0;
    }
  }
  else {
    if (!end)
      input_stack::push(make_temp_iterator(" "));
    input_stack::push(make_temp_iterator(s));
    tok.next();
    unsigned int c = get_color_element("cmyk color", "cyan component");
    unsigned int m = get_color_element("cmyk color", "magenta component");
    unsigned int y = get_color_element("cmyk color", "yellow component");
    unsigned int k = get_color_element("cmyk color", "black component");
    col->set_cmyk(c, m, y, k);
  }
  return col;
}

static color *read_gray(char end = 0)
{
  symbol component = do_get_long_name(0, end);
  if (component.is_null()) {
    warning(WARN_COLOR, "missing gray values");
    return 0;
  }
  const char *s = component.contents();
  color *col = new color;
  if (*s == '#') {
    if (!col->read_gray(s)) {
      warning(WARN_COLOR, "`expecting a gray definition not `%1'", s);
      delete col;
      return 0;
    }
  }
  else {
    if (!end)
      input_stack::push(make_temp_iterator("\n"));
    input_stack::push(make_temp_iterator(s));
    tok.next();
    unsigned int g = get_color_element("gray", "gray value");
    col->set_gray(g);
  }
  return col;
}

static void activate_color()
{
  int n;
  if (has_arg() && get_integer(&n))
    color_flag = n != 0;
  else
    color_flag = 1;
  skip_line();
}

static void define_color()
{
  symbol color_name = get_long_name(1);
  if (color_name.is_null()) {
    skip_line();
    return;
  }
  if (color_name == default_symbol) {
    warning(WARN_COLOR, "default color can't be redefined");
    skip_line();
    return;
  }
  symbol style = get_long_name(1);
  if (style.is_null()) {
    skip_line();
    return;
  }
  color *col;
  if (strcmp(style.contents(), "rgb") == 0)
    col = read_rgb();
  else if (strcmp(style.contents(), "cmyk") == 0)
    col = read_cmyk();
  else if (strcmp(style.contents(), "gray") == 0)
    col = read_gray();
  else if (strcmp(style.contents(), "grey") == 0)
    col = read_gray();
  else if (strcmp(style.contents(), "cmy") == 0)
    col = read_cmy();
  else {
    warning(WARN_COLOR,
	    "unknown color space `%1'; use rgb, cmyk, gray or cmy",
	    style.contents());
    skip_line();
    return;
  }
  if (col) {
    col->nm = color_name;
    (void)color_dictionary.lookup(color_name, col);
  }
  skip_line();
}

static node *do_overstrike()
{
  token start;
  overstrike_node *on = new overstrike_node;
  int start_level = input_stack::get_level();
  start.next();
  for (;;) {
    tok.next();
    if (tok.newline() || tok.eof()) {
      warning(WARN_DELIM, "missing closing delimiter");
      input_stack::push(make_temp_iterator("\n"));
      break;
    }
    if (tok == start
	&& (compatible_flag || input_stack::get_level() == start_level))
      break;
    charinfo *ci = tok.get_char(1);
    if (ci) {
      node *n = curenv->make_char_node(ci);
      if (n)
	on->overstrike(n);
    }
  }
  return on;
}

static node *do_bracket()
{
  token start;
  bracket_node *bn = new bracket_node;
  start.next();
  int start_level = input_stack::get_level();
  for (;;) {
    tok.next();
    if (tok.eof()) {
      warning(WARN_DELIM, "missing closing delimiter");
      break;
    }
    if (tok.newline()) {
      warning(WARN_DELIM, "missing closing delimiter");
      input_stack::push(make_temp_iterator("\n"));
      break;
    }
    if (tok == start
	&& (compatible_flag || input_stack::get_level() == start_level))
      break;
    charinfo *ci = tok.get_char(1);
    if (ci) {
      node *n = curenv->make_char_node(ci);
      if (n)
	bn->bracket(n);
    }
  }
  return bn;
}

static int do_name_test()
{
  token start;
  start.next();
  int start_level = input_stack::get_level();
  int bad_char = 0;
  int some_char = 0;
  for (;;) {
    tok.next();
    if (tok.newline() || tok.eof()) {
      warning(WARN_DELIM, "missing closing delimiter");
      input_stack::push(make_temp_iterator("\n"));
      break;
    }
    if (tok == start
	&& (compatible_flag || input_stack::get_level() == start_level))
      break;
    if (!tok.ch())
      bad_char = 1;
    some_char = 1;
  }
  return some_char && !bad_char;
}

static int do_expr_test()
{
  token start;
  start.next();
  int start_level = input_stack::get_level();
  if (!start.delimiter(1))
    return 0;
  tok.next();
  // disable all warning and error messages temporarily
  int saved_warning_mask = warning_mask;
  int saved_inhibit_errors = inhibit_errors;
  warning_mask = 0;
  inhibit_errors = 1;
  int dummy;
  int result = get_number_rigidly(&dummy, 'u');
  warning_mask = saved_warning_mask;
  inhibit_errors = saved_inhibit_errors;
  if (tok == start && input_stack::get_level() == start_level)
    return result;
  // ignore everything up to the delimiter in case we aren't right there
  for (;;) {
    tok.next();
    if (tok.newline() || tok.eof()) {
      warning(WARN_DELIM, "missing closing delimiter");
      input_stack::push(make_temp_iterator("\n"));
      break;
    }
    if (tok == start && input_stack::get_level() == start_level)
      break;
  }
  return 0;
}

#if 0
static node *do_zero_width()
{
  token start;
  start.next();
  int start_level = input_stack::get_level();
  environment env(curenv);
  environment *oldenv = curenv;
  curenv = &env;
  for (;;) {
    tok.next();
    if (tok.newline() || tok.eof()) {
      error("missing closing delimiter");
      break;
    }
    if (tok == start
	&& (compatible_flag || input_stack::get_level() == start_level))
      break;
    tok.process();
  }
  curenv = oldenv;
  node *rev = env.extract_output_line();
  node *n = 0;
  while (rev) {
    node *tem = rev;
    rev = rev->next;
    tem->next = n;
    n = tem;
  }
  return new zero_width_node(n);
}

#else

// It's undesirable for \Z to change environments, because then
// \n(.w won't work as expected.

static node *do_zero_width()
{
  node *rev = new dummy_node;
  token start;
  start.next();
  int start_level = input_stack::get_level();
  for (;;) {
    tok.next();
    if (tok.newline() || tok.eof()) {
      warning(WARN_DELIM, "missing closing delimiter");
      input_stack::push(make_temp_iterator("\n"));
      break;
    }
    if (tok == start
	&& (compatible_flag || input_stack::get_level() == start_level))
      break;
    if (!tok.add_to_node_list(&rev))
      error("invalid token in argument to \\Z");
  }
  node *n = 0;
  while (rev) {
    node *tem = rev;
    rev = rev->next;
    tem->next = n;
    n = tem;
  }
  return new zero_width_node(n);
}

#endif

token_node *node::get_token_node()
{
  return 0;
}

class token_node : public node {
public:
  token tk;
  token_node(const token &t);
  node *copy();
  token_node *get_token_node();
  int same(node *);
  const char *type();
  int force_tprint();
  int is_tag();
};

token_node::token_node(const token &t) : tk(t)
{
}

node *token_node::copy()
{
  return new token_node(tk);
}

token_node *token_node::get_token_node()
{
  return this;
}

int token_node::same(node *nd)
{
  return tk == ((token_node *)nd)->tk;
}

const char *token_node::type()
{
  return "token_node";
}

int token_node::force_tprint()
{
  return 0;
}

int token_node::is_tag()
{
  return 0;
}

token::token() : nd(0), type(TOKEN_EMPTY)
{
}

token::~token()
{
  delete nd;
}

token::token(const token &t)
: nm(t.nm), c(t.c), val(t.val), dim(t.dim), type(t.type)
{
  // Use two statements to work around bug in SGI C++.
  node *tem = t.nd;
  nd = tem ? tem->copy() : 0;
}

void token::operator=(const token &t)
{
  delete nd;
  nm = t.nm;
  // Use two statements to work around bug in SGI C++.
  node *tem = t.nd;
  nd = tem ? tem->copy() : 0;
  c = t.c;
  val = t.val;
  dim = t.dim;
  type = t.type;
}

void token::skip()
{
  while (space())
    next();
}

int has_arg()
{
  while (tok.space())
    tok.next();
  return !tok.newline();
}

void token::make_space()
{
  type = TOKEN_SPACE;
}

void token::make_newline()
{
  type = TOKEN_NEWLINE;
}

void token::next()
{
  if (nd) {
    delete nd;
    nd = 0;
  }
  units x;
  for (;;) {
    node *n = 0;
    int cc = input_stack::get(&n);
    if (cc != escape_char || escape_char == 0) {
    handle_normal_char:
      switch(cc) {
      case PUSH_GROFF_MODE:
	input_stack::save_compatible_flag(compatible_flag);
	compatible_flag = 0;
	continue;
      case PUSH_COMP_MODE:
	input_stack::save_compatible_flag(compatible_flag);
	compatible_flag = 1;
	continue;
      case POP_GROFFCOMP_MODE:
	compatible_flag = input_stack::get_compatible_flag();
	continue;
      case BEGIN_QUOTE:
	input_stack::increase_level();
	continue;
      case END_QUOTE:
	input_stack::decrease_level();
	continue;
      case EOF:
	type = TOKEN_EOF;
	return;
      case TRANSPARENT_FILE_REQUEST:
      case TITLE_REQUEST:
      case COPY_FILE_REQUEST:
#ifdef COLUMN
      case VJUSTIFY_REQUEST:
#endif /* COLUMN */
	type = TOKEN_REQUEST;
	c = cc;
	return;
      case BEGIN_TRAP:
	type = TOKEN_BEGIN_TRAP;
	return;
      case END_TRAP:
	type = TOKEN_END_TRAP;
	return;
      case LAST_PAGE_EJECTOR:
	seen_last_page_ejector = 1;
	// fall through
      case PAGE_EJECTOR:
	type = TOKEN_PAGE_EJECTOR;
	return;
      case ESCAPE_PERCENT:
      ESCAPE_PERCENT:
	type = TOKEN_HYPHEN_INDICATOR;
	return;
      case ESCAPE_SPACE:
      ESCAPE_SPACE:
	type = TOKEN_UNSTRETCHABLE_SPACE;
	return;
      case ESCAPE_TILDE:
      ESCAPE_TILDE:
	type = TOKEN_STRETCHABLE_SPACE;
	return;
      case ESCAPE_COLON:
      ESCAPE_COLON:
	type = TOKEN_ZERO_WIDTH_BREAK;
	return;
      case ESCAPE_e:
      ESCAPE_e:
	type = TOKEN_ESCAPE;
	return;
      case ESCAPE_E:
	goto handle_escape_char;
      case ESCAPE_BAR:
      ESCAPE_BAR:
	type = TOKEN_NODE;
	nd = new hmotion_node(curenv->get_narrow_space_width(),
			      curenv->get_fill_color());
	return;
      case ESCAPE_CIRCUMFLEX:
      ESCAPE_CIRCUMFLEX:
	type = TOKEN_NODE;
	nd = new hmotion_node(curenv->get_half_narrow_space_width(),
			      curenv->get_fill_color());
	return;
      case ESCAPE_NEWLINE:
	have_input = 0;
	break;
      case ESCAPE_LEFT_BRACE:
      ESCAPE_LEFT_BRACE:
	type = TOKEN_LEFT_BRACE;
	return;
      case ESCAPE_RIGHT_BRACE:
      ESCAPE_RIGHT_BRACE:
	type = TOKEN_RIGHT_BRACE;
	return;
      case ESCAPE_LEFT_QUOTE:
      ESCAPE_LEFT_QUOTE:
	type = TOKEN_SPECIAL;
	nm = symbol("ga");
	return;
      case ESCAPE_RIGHT_QUOTE:
      ESCAPE_RIGHT_QUOTE:
	type = TOKEN_SPECIAL;
	nm = symbol("aa");
	return;
      case ESCAPE_HYPHEN:
      ESCAPE_HYPHEN:
	type = TOKEN_SPECIAL;
	nm = symbol("-");
	return;
      case ESCAPE_UNDERSCORE:
      ESCAPE_UNDERSCORE:
	type = TOKEN_SPECIAL;
	nm = symbol("ul");
	return;
      case ESCAPE_c:
      ESCAPE_c:
	type = TOKEN_INTERRUPT;
	return;
      case ESCAPE_BANG:
      ESCAPE_BANG:
	type = TOKEN_TRANSPARENT;
	return;
      case ESCAPE_QUESTION:
      ESCAPE_QUESTION:
	nd = do_non_interpreted();
	if (nd) {
	  type = TOKEN_NODE;
	  return;
	}
	break;
      case ESCAPE_AMPERSAND:
      ESCAPE_AMPERSAND:
	type = TOKEN_DUMMY;
	return;
      case ESCAPE_RIGHT_PARENTHESIS:
      ESCAPE_RIGHT_PARENTHESIS:
	type = TOKEN_TRANSPARENT_DUMMY;
	return;
      case '\b':
	type = TOKEN_BACKSPACE;
	return;
      case ' ':
	type = TOKEN_SPACE;
	return;
      case '\t':
	type = TOKEN_TAB;
	return;
      case '\n':
	type = TOKEN_NEWLINE;
	return;
      case '\001':
	type = TOKEN_LEADER;
	return;
      case 0:
	{
	  assert(n != 0);
	  token_node *tn = n->get_token_node();
	  if (tn) {
	    *this = tn->tk;
	    delete tn;
	  }
	  else {
	    nd = n;
	    type = TOKEN_NODE;
	  }
	}
	return;
      default:
	type = TOKEN_CHAR;
	c = cc;
	return;
      }
    }
    else {
    handle_escape_char:
      cc = input_stack::get(&n);
      switch(cc) {
      case '(':
	nm = read_two_char_escape_name();
	type = TOKEN_SPECIAL;
	return;
      case EOF:
	type = TOKEN_EOF;
	error("end of input after escape character");
	return;
      case '`':
	goto ESCAPE_LEFT_QUOTE;
      case '\'':
	goto ESCAPE_RIGHT_QUOTE;
      case '-':
	goto ESCAPE_HYPHEN;
      case '_':
	goto ESCAPE_UNDERSCORE;
      case '%':
	goto ESCAPE_PERCENT;
      case ' ':
	goto ESCAPE_SPACE;
      case '0':
	nd = new hmotion_node(curenv->get_digit_width(),
			      curenv->get_fill_color());
	type = TOKEN_NODE;
	return;
      case '|':
	goto ESCAPE_BAR;
      case '^':
	goto ESCAPE_CIRCUMFLEX;
      case '/':
	type = TOKEN_ITALIC_CORRECTION;
	return;
      case ',':
	type = TOKEN_NODE;
	nd = new left_italic_corrected_node;
	return;
      case '&':
	goto ESCAPE_AMPERSAND;
      case ')':
	goto ESCAPE_RIGHT_PARENTHESIS;
      case '!':
	goto ESCAPE_BANG;
      case '?':
	goto ESCAPE_QUESTION;
      case '~':
	goto ESCAPE_TILDE;
      case ':':
	goto ESCAPE_COLON;
      case '"':
	while ((cc = input_stack::get(0)) != '\n' && cc != EOF)
	  ;
	if (cc == '\n')
	  type = TOKEN_NEWLINE;
	else
	  type = TOKEN_EOF;
	return;
      case '#':			// Like \" but newline is ignored.
	while ((cc = input_stack::get(0)) != '\n')
	  if (cc == EOF) {
	    type = TOKEN_EOF;
	    return;
	  }
	break;
      case '$':
	{
	  symbol s = read_escape_name();
	  if (!(s.is_null() || s.is_empty()))
	    interpolate_arg(s);
	  break;
	}
      case '*':
	{
	  symbol s = read_escape_name(WITH_ARGS);
	  if (!(s.is_null() || s.is_empty())) {
	    if (have_string_arg) {
	      have_string_arg = 0;
	      interpolate_string_with_args(s);
	    }
	    else
	      interpolate_string(s);
	  }
	  break;
	}
      case 'a':
	nd = new non_interpreted_char_node('\001');
	type = TOKEN_NODE;
	return;
      case 'A':
	c = '0' + do_name_test();
	type = TOKEN_CHAR;
	return;
      case 'b':
	nd = do_bracket();
	type = TOKEN_NODE;
	return;
      case 'B':
	c = '0' + do_expr_test();
	type = TOKEN_CHAR;
	return;
      case 'c':
	goto ESCAPE_c;
      case 'C':
	nm = get_delim_name();
	if (nm.is_null())
	  break;
	type = TOKEN_SPECIAL;
	return;
      case 'd':
	type = TOKEN_NODE;
	nd = new vmotion_node(curenv->get_size() / 2,
			      curenv->get_fill_color());
	return;
      case 'D':
	nd = read_draw_node();
	if (!nd)
	  break;
	type = TOKEN_NODE;
	return;
      case 'e':
	goto ESCAPE_e;
      case 'E':
	goto handle_escape_char;
      case 'f':
	{
	  symbol s = read_escape_name(ALLOW_EMPTY);
	  if (s.is_null())
	    break;
	  const char *p;
	  for (p = s.contents(); *p != '\0'; p++)
	    if (!csdigit(*p))
	      break;
	  if (*p || s.is_empty())
	    curenv->set_font(s);
	  else
	    curenv->set_font(atoi(s.contents()));
	  if (!compatible_flag)
	    have_input = 1;
	  break;
	}
      case 'F':
	{
	  symbol s = read_escape_name(ALLOW_EMPTY);
	  if (s.is_null())
	    break;
	  curenv->set_family(s);
	  have_input = 1;
	  break;
	}
      case 'g':
	{
	  symbol s = read_escape_name();
	  if (!(s.is_null() || s.is_empty()))
	    interpolate_number_format(s);
	  break;
	}
      case 'h':
	if (!get_delim_number(&x, 'm'))
	  break;
	type = TOKEN_NODE;
	nd = new hmotion_node(x, curenv->get_fill_color());
	return;
      case 'H':
	// don't take height increments relative to previous height if
	// in compatibility mode
	if (!compatible_flag && curenv->get_char_height())
	{
	  if (get_delim_number(&x, 'z', curenv->get_char_height()))
	    curenv->set_char_height(x);
	}
	else
	{
	  if (get_delim_number(&x, 'z', curenv->get_requested_point_size()))
	    curenv->set_char_height(x);
	}
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 'k':
	nm = read_escape_name();
	if (nm.is_null() || nm.is_empty())
	  break;
	type = TOKEN_MARK_INPUT;
	return;
      case 'l':
      case 'L':
	{
	  charinfo *s = 0;
	  if (!get_line_arg(&x, (cc == 'l' ? 'm': 'v'), &s))
	    break;
	  if (s == 0)
	    s = get_charinfo(cc == 'l' ? "ru" : "br");
	  type = TOKEN_NODE;
	  node *char_node = curenv->make_char_node(s);
	  if (cc == 'l')
	    nd = new hline_node(x, char_node);
	  else
	    nd = new vline_node(x, char_node);
	  return;
	}
      case 'm':
	do_glyph_color(read_escape_name(ALLOW_EMPTY));
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 'M':
	do_fill_color(read_escape_name(ALLOW_EMPTY));
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 'n':
	{
	  int inc;
	  symbol s = read_increment_and_escape_name(&inc);
	  if (!(s.is_null() || s.is_empty()))
	    interpolate_number_reg(s, inc);
	  break;
	}
      case 'N':
	if (!get_delim_number(&val, 0))
	  break;
	type = TOKEN_NUMBERED_CHAR;
	return;
      case 'o':
	nd = do_overstrike();
	type = TOKEN_NODE;
	return;
      case 'O':
	nd = do_suppress(read_escape_name());
	if (!nd)
	  break;
	type = TOKEN_NODE;
	return;
      case 'p':
	type = TOKEN_SPREAD;
	return;
      case 'r':
	type = TOKEN_NODE;
	nd = new vmotion_node(-curenv->get_size(), curenv->get_fill_color());
	return;
      case 'R':
	do_register();
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 's':
	if (read_size(&x))
	  curenv->set_size(x);
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 'S':
	if (get_delim_number(&x, 0))
	  curenv->set_char_slant(x);
	if (!compatible_flag)
	  have_input = 1;
	break;
      case 't':
	type = TOKEN_NODE;
	nd = new non_interpreted_char_node('\t');
	return;
      case 'u':
	type = TOKEN_NODE;
	nd = new vmotion_node(-curenv->get_size() / 2,
			      curenv->get_fill_color());
	return;
      case 'v':
	if (!get_delim_number(&x, 'v'))
	  break;
	type = TOKEN_NODE;
	nd = new vmotion_node(x, curenv->get_fill_color());
	return;
      case 'V':
	{
	  symbol s = read_escape_name();
	  if (!(s.is_null() || s.is_empty()))
	    interpolate_environment_variable(s);
	  break;
	}
      case 'w':
	do_width();
	break;
      case 'x':
	if (!get_delim_number(&x, 'v'))
	  break;
	type = TOKEN_NODE;
	nd = new extra_size_node(x);
	return;
      case 'X':
	nd = do_special();
	if (!nd)
	  break;
	type = TOKEN_NODE;
	return;
      case 'Y':
	{
	  symbol s = read_escape_name();
	  if (s.is_null() || s.is_empty())
	    break;
	  request_or_macro *p = lookup_request(s);
	  macro *m = p->to_macro();
	  if (!m) {
	    error("can't transparently throughput a request");
	    break;
	  }
	  nd = new special_node(*m);
	  type = TOKEN_NODE;
	  return;
	}
      case 'z':
	{
	  next();
	  if (type == TOKEN_NODE)
	    nd = new zero_width_node(nd);
	  else {
  	    charinfo *ci = get_char(1);
	    if (ci == 0)
	      break;
	    node *gn = curenv->make_char_node(ci);
	    if (gn == 0)
	      break;
	    nd = new zero_width_node(gn);
	    type = TOKEN_NODE;
	  }
	  return;
	}
      case 'Z':
	nd = do_zero_width();
	if (nd == 0)
	  break;
	type = TOKEN_NODE;
	return;
      case '{':
	goto ESCAPE_LEFT_BRACE;
      case '}':
	goto ESCAPE_RIGHT_BRACE;
      case '\n':
	break;
      case '[':
	if (!compatible_flag) {
	  symbol s = read_long_escape_name(WITH_ARGS);
	  if (s.is_null() || s.is_empty())
	    break;
	  if (have_string_arg) {
	    have_string_arg = 0;
	    nm = composite_glyph_name(s);
	  }
	  else {
	    const char *gn = check_unicode_name(s.contents());
	    if (gn) {
	      const char *gn_decomposed = decompose_unicode(gn);
	      if (gn_decomposed)
		gn = &gn_decomposed[1];
	      const char *groff_gn = unicode_to_glyph_name(gn);
	      if (groff_gn)
		nm = symbol(groff_gn);
	      else {
		char *buf = new char[strlen(gn) + 1 + 1];
		strcpy(buf, "u");
		strcat(buf, gn);
		nm = symbol(buf);
		a_delete buf;
	      }
	    }
	    else
	      nm = symbol(s.contents());
	  }
	  type = TOKEN_SPECIAL;
	  return;
	}
	goto handle_normal_char;
      default:
	if (cc != escape_char && cc != '.')
	  warning(WARN_ESCAPE, "escape character ignored before %1",
		  input_char_description(cc));
	goto handle_normal_char;
      }
    }
  }
}

int token::operator==(const token &t)
{
  if (type != t.type)
    return 0;
  switch(type) {
  case TOKEN_CHAR:
    return c == t.c;
  case TOKEN_SPECIAL:
    return nm == t.nm;
  case TOKEN_NUMBERED_CHAR:
    return val == t.val;
  default:
    return 1;
  }
}

int token::operator!=(const token &t)
{
  return !(*this == t);
}

// is token a suitable delimiter (like ')?

int token::delimiter(int err)
{
  switch(type) {
  case TOKEN_CHAR:
    switch(c) {
    case '0':
    case '1':
    case '2':
    case '3':
    case '4':
    case '5':
    case '6':
    case '7':
    case '8':
    case '9':
    case '+':
    case '-':
    case '/':
    case '*':
    case '%':
    case '<':
    case '>':
    case '=':
    case '&':
    case ':':
    case '(':
    case ')':
    case '.':
      if (err)
	error("cannot use character `%1' as a starting delimiter", char(c));
      return 0;
    default:
      return 1;
    }
  case TOKEN_NODE:
  case TOKEN_SPACE:
  case TOKEN_STRETCHABLE_SPACE:
  case TOKEN_UNSTRETCHABLE_SPACE:
  case TOKEN_TAB:
  case TOKEN_NEWLINE:
    if (err)
      error("cannot use %1 as a starting delimiter", description());
    return 0;
  default:
    return 1;
  }
}

const char *token::description()
{
  static char buf[4];
  switch (type) {
  case TOKEN_BACKSPACE:
    return "a backspace character";
  case TOKEN_CHAR:
    buf[0] = '`';
    buf[1] = c;
    buf[2] = '\'';
    buf[3] = '\0';
    return buf;
  case TOKEN_DUMMY:
    return "`\\&'";
  case TOKEN_ESCAPE:
    return "`\\e'";
  case TOKEN_HYPHEN_INDICATOR:
    return "`\\%'";
  case TOKEN_INTERRUPT:
    return "`\\c'";
  case TOKEN_ITALIC_CORRECTION:
    return "`\\/'";
  case TOKEN_LEADER:
    return "a leader character";
  case TOKEN_LEFT_BRACE:
    return "`\\{'";
  case TOKEN_MARK_INPUT:
    return "`\\k'";
  case TOKEN_NEWLINE:
    return "newline";
  case TOKEN_NODE:
    return "a node";
  case TOKEN_NUMBERED_CHAR:
    return "`\\N'";
  case TOKEN_RIGHT_BRACE:
    return "`\…