// Tokenizer.cpp     Dave Reed      9/30/02
////////////////////////////////////////////////

#include "Tokenizer.h"

Tokenizer::Tokenizer(istream & instr) : instream(instr)
// constructor: initializes input stream and reads first token into
//              nextToken buffer (setting remaining flag as a result)
{
	remaining = ReadToken(instream, nextToken);
}

bool Tokenizer::TokensRemain() const
// returns true if tokens remain to be processed (based on remaining flag)
{
	return remaining;
}

Token Tokenizer::GetToken()
// returns next token from input stream (already buffered in nextToken)
//   and reads ahead, storing the nextToken and updating the remaining flag
{
	Token retValue = nextToken;
	remaining = ReadToken(instream, nextToken);
	return retValue;
}

Token Tokenizer::PeekAhead() const
// returns next token from input stream (already buffered in nextToken) 
//   but does not remove it from the stream
{
	return nextToken;
}

bool Tokenizer::ReadToken(istream & instr, Token & t)
// private member function: reads a token t from input stream instr
//   returns whether the read was successful
{
	t.SetValue("");
	t.SetType(UNKNOWN);

	string temp;
	if (instr >> temp) {
		t.SetValue(temp);

        if (temp == "begin" || temp == "end" || temp == "output") {
			t.SetType(KEYWORD);
		}
		else if (temp == "+" || temp == "=") {
			t.SetType(OPERATOR);
		}
		else if (temp[0] == '\"') {
			char ch;
			while (temp[temp.length()-1] != '\"' && instr.get(ch) && ch != '\n') {
					temp = temp + ch;
			}
			t.SetValue(temp);
			if (temp[temp.length()-1] == '\"') {
			    t.SetType(STRING);
			}
		}
		else if (isdigit(temp[0])) {
			bool integer = true;
			for (int k = 1; k < temp.length(); k++) {
				if (!isdigit(temp[k])) {
					integer = false;
				}
			}
			if (integer) {
				t.SetType(INTEGER);
			}
		}
		else if (isalpha(temp[0]) &&
			     (temp.length() == 1 || 
				  (temp.length() == 2 && isdigit(temp[1])))) {
			t.SetType(IDENTIFIER);
		}
		return true;
	}
	return false;
}