Listing 1: Interface for TokenIter class

#ifndef TOKENITERATOR_H
#define TOKENITERATOR_H

//*************************************************************
//
// TokenIter class header.
//
// TokenIter is a template class that provides an iterator 
// over a string, supplying the tokens indicated by the 
// function object 'Finder'.
// 
// Finder is a function object that finds a token in a string 
// by setting the supplied start and end pointers to the start, 
// and just past the end, of the first token respectively.
//
// Finder should have a default constructor and a function 
// signature of:
//
//      int operator()(const char*& start, const char*& end)
//
// It will be called with 'end' pointing to the beginning of 
// the string and should set 'start' to point to the beginning 
// of the first token, 'end' to point just past the end of the 
// token, and return the token length.
// 
// See TokenFinder.h for a canonical example of Finder.
//
//*************************************************************

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#include <iterator>
#include <string>

template<class TFinder>
class TokenIter : public std::iterator<std::input_iterator_tag, 
                             std::string>
{
public:
    static const char EndOfString;
    static const TokenIter<TFinder> EOS;

    TokenIter() : start(&EndOfString), end(&EndOfString),
                  length(0)
    {}

    TokenIter(const char* str, TFinder finder = TFinder()) 
        : findToken(finder), end(str) 
    { length = findToken(start, end); }

    TokenIter(const std::string& str, TFinder finder = TFinder()) 
        : findToken(finder), end(str.c_str()) 
    { length = findToken(start, end); }

    TokenIter& operator=(const char* const str)
    { end = str; length = findToken(start, end); return *this; }

    TokenIter& operator=(const std::string& str)
    { 
        end = str.c_str(); length = findToken(start, end); 
        return *this; 
    }

    std::string operator*() const 
    { return std::string(start, length); }

    bool operator == (TokenIter rhs) const
    {
        bool retv(false);
        if (*start == *rhs.start)
            if (*start == EndOfString || start == rhs.start)
                retv = true;
        return retv;
    }

    bool operator != (TokenIter rhs) const 
    { return !operator==(rhs); }

    TokenIter& operator ++ () 
    { length = findToken(start, end); return *this; }

    TokenIter& operator ++ (int) 
    { 
        TokenIter retv(*this); 
        length = findToken(start, end); 
        return retv; 
    }

private:
    TFinder     findToken;
    const char* start;
    const char* end;
    size_t      length;
};

// Define a string terminator for comparisons

template<typename TFinder>
const char TokenIter<TFinder>::EndOfString = '\0';


// define default iterator for end of sequence

template<typename TFinder>
const TokenIter<TFinder> TokenIter<TFinder>::EOS;

#endif // #ifndef TOKENITERATOR_H