Listing 3 Definition of the class Tokenizer

_CLASSDEF(Tokenizer)
class _CLASSTYPE Tokenizer
{
public:
   Tokenizer(){}      // constructor
   Tokenizer(ifstream&);    // constructor which takes
                        // a file stream
   Tokenizer(const LPSTR);  // constructor which takes
                        // a file name
   ~Tokenizer()       // destructor
   {fis->close();
   delete fis;}
   Token * lookToken();     // look at the current token
   Token * getToken();      // get a new token from the
                        // file stream
   Token * peekToken();     // look ahead a token without moving
                        // the file pointer
   int getLineNo() { return nLineNo;}
   BOOL eof() { return seof;}
   BOOL good() { return status;}

   virtual void initialize(){}  // virtual function defined
                           // in child class

protected:
   BOOL seof;               // end of file indicator
   BOOL status;             // status of the constructor
   char chcurr;             // current char
   Token tcurr;             // current token
   Token tpeek;             // peek token
   int nLineNo;
private:
   void getChar();          // get one char from the ifstream
   void getString();        // get one string
   void skipComments();     // skip comments inside a file
   void searchTokenType();  // determine the token type
   ifstream * fis;      // file stream of the tokenizer
};
// End of File