Listing 3: The header that defines scanner and token classes that can look ahead an unlimited number of tokens

//
// scanner.h
//
// Copyright (C) 1996 by Dan Saks.
// May be copied for private, non-commercial
// use, provided this copyright notice 
// remains intact.
// All other rights reserved.
//

#ifndef SCANNER_H_INCLUDED
#define SCANNER_H_INCLUDED

#include <assert.h>
#include <deque.h>
#include <iostream.h>
#include <limits.h>
#include <string>

class token
    {
    friend class scanner;
public:
    enum category
        {
        AMPERSAND = '&',
        COMMA = ',',
        LEFT_BRACKET = '[',
        LEFT_PAREN = '(',
        RIGHT_BRACKET = ']',
        RIGHT_PAREN = ')',
        SEMICOLON = ';',
        STAR = '*',
        NO_MORE = CHAR_MAX + 1,
        SCOPE,
        INT_LITERAL,
        IDENTIFIER,
        NAME,
        TYPE_KEYWORD,
        CONST,
        VOLATILE,
        NO_SUCH,
        NO_VALUE
        };
    token();
    string text() const;
    category kind() const;
private:
    token(category, const string &);
    category the_kind;
    string the_text;
    };

const char *image(token::category);

class scanner
    {
public:
    scanner(istream &);
    token backup();
    token current() const;
    token get();
    void mark();
    void reset();
private:
    token scan();
    token current_token;
    istream &in_stream;
    deque<token> in_queue;
    deque<token>::iterator in_queue_iterator;
    bool getting_from_queue;
    bool looking_ahead;
    struct pair
        {
        char const *text;
        token::category kind;
        };
    static pair const keyword[];
    scanner(const scanner &);
    scanner &operator=(const scanner &);
    };

//
// token inline functions
//
inline
token::token()
    :   the_kind(NO_VALUE)
    {
    }

inline
token::token(category c, const string &s)
    :   the_kind(c), the_text(s)
    {
    }

inline
string token::text() const
    {
    return the_text;
    }

inline
token::category token::kind() const
    {
    return the_kind;
    }

//
// scanner inline functions
//
inline
scanner::scanner(istream &s)
    :   in_stream(s),
        looking_ahead(false),
        getting_from_queue(false)
    {
    }

inline
token scanner::current() const
    {
    return current_token;
    }

#endif
/* End of File */