// Tokenizer.h     Dave Reed      9/30/02
//
// Tokenizer class -- used to read tokens (language elements)
//     Note: utilizes the Token class from Token.h
//
//     constructor: create a Tokenizer connected to an input stream
//       (if no argument is specified, the default stream is cin)
//       e.g., ifstream myin("prog1.txt");
//             Tokenizer program(myin);
//             Tokenizer p;
//
//      public member functions:
//         bool TokensRemain();  	// returns true if tokens remain to be read
//         Token GetToken();        // returns next token from input stream
//         Token PeekAhead();       // returns next token without reading it
/////////////////////////////////////////////////////////////////////////////////

#ifndef _TOKENIZER_
#define _TOKENIZER_

#include <iostream>
#include <cctype>
#include <string>
#include <vector>
#include "Token.h"
using namespace std;


class Tokenizer								
{									
  public:
	Tokenizer(istream & instr = cin);
	bool TokensRemain() const;
	Token GetToken();
	Token PeekAhead() const;
  private:
	istream instream;    
	Token nextToken;
	bool remaining;
	bool ReadToken(istream & instr, Token & t);
};

#endif