Marsyas  0.6.0-alpha
/usr/src/RPM/BUILD/marsyas-0.6.0/src/marsyas/expr/ExScanner.cpp
Go to the documentation of this file.
00001 #include <marsyas/expr/ExScanner.h>
00002 
00003 #include <memory.h>
00004 #include <string.h>
00005 
00006 // string handling, wide character
00007 
00008 char* coco_string_create(const char* value) {
00009   char* data;
00010   int len = 0;
00011   if (value) { len =(int)strlen(value); }
00012   data = new char[len + 1];
00013   strncpy(data, value, len);
00014   data[len] = 0;
00015   return data;
00016 }
00017 
00018 char* coco_string_create(const char* value , int startIndex, int length) {
00019   int len = 0;
00020   char* data;
00021 
00022   if (value) { len = length; }
00023   data = new char[len + 1];
00024   strncpy(data, &(value[startIndex]), len);
00025   data[len] = 0;
00026 
00027   return data;
00028 }
00029 
00030 char* coco_string_create_upper(char* data) {
00031   if (!data) { return NULL; }
00032 
00033   int dataLen = 0;
00034   if (data) { dataLen =(int)strlen(data); }
00035 
00036   char* newData = new char[dataLen + 1];
00037 
00038   for (int i = 0; i <= dataLen; ++i) {
00039     if (('a' <= data[i]) && (data[i] <= 'z')) {
00040       newData[i] = data[i] + ('A' - 'a');
00041     }
00042     else { newData[i] = data[i]; }
00043   }
00044 
00045   newData[dataLen] = '\0';
00046   return newData;
00047 }
00048 
00049 char* coco_string_create_lower(char* data) {
00050   if (!data) { return NULL; }
00051 
00052   int dataLen = 0;
00053   if (data) { dataLen =(int)strlen(data); }
00054 
00055   char* newData = new char[dataLen + 1];
00056 
00057   for (int i = 0; i <= dataLen; ++i) {
00058     if (('A' <= data[i]) && (data[i] <= 'Z')) {
00059       newData[i] = data[i] - ('A'- 'a');
00060     }
00061     else { newData[i] = data[i]; }
00062   }
00063   newData[dataLen] = '\0';
00064   return newData;
00065 }
00066 
00067 char* coco_string_create_append(const char* data1, const char* data2) {
00068   char* data;
00069   int data1Len = 0;
00070   int data2Len = 0;
00071 
00072   if (data1) { data1Len =(int)strlen(data1); }
00073   if (data2) {data2Len =(int)strlen(data2); }
00074 
00075   data = new char[data1Len + data2Len + 1];
00076 
00077   if (data1) { strcpy(data, data1); }
00078   if (data2) { strcpy(data + data1Len, data2); }
00079 
00080   data[data1Len + data2Len] = 0;
00081 
00082   return data;
00083 }
00084 
00085 char* coco_string_create_append(const char* target, const char appendix) {
00086   int targetLen = coco_string_length(target);
00087   char* data = new char[targetLen + 2];
00088   strncpy(data, target, targetLen);
00089   data[targetLen] = appendix;
00090   data[targetLen + 1] = 0;
00091   return data;
00092 }
00093 
00094 void coco_string_delete(char* &data) {
00095   delete [] data;
00096   data = NULL;
00097 }
00098 
00099 int coco_string_length(const char* data) {
00100   if (data) { return (int)strlen(data); }
00101   return 0;
00102 }
00103 
00104 bool coco_string_endswith(char* data, char* end) {
00105   int dataLen =(int)strlen(data);
00106   int endLen =(int)strlen(end);
00107   return (endLen <= dataLen) && (strcmp(data + dataLen - endLen, end) == 0);
00108 }
00109 
00110 int coco_string_indexof(char* data, char value) {
00111   char* chr = strchr(data, value);
00112 
00113   if (chr) { return (int)(chr-data); }
00114   return -1;
00115 }
00116 
00117 int coco_string_lastindexof(char* data, char value) {
00118   char* chr = strrchr(data, value);
00119 
00120   if (chr) { return (int)(chr-data); }
00121   return -1;
00122 }
00123 
00124 void coco_string_merge(char* &target, char* appendix) {
00125   if (!appendix) { return; }
00126   char* data = coco_string_create_append(target, appendix);
00127   delete [] target;
00128   target = data;
00129 }
00130 
00131 bool coco_string_equal(char* data1, char* data2) {
00132   return strcmp( data1, data2 ) == 0;
00133 }
00134 
00135 int coco_string_compareto(char* data1, char* data2) {
00136   return strcmp(data1, data2);
00137 }
00138 
00139 int coco_string_hash(char* data) {
00140   int h = 0;
00141   if (!data) { return 0; }
00142   while (*data != 0) {
00143     h = (h * 7) ^ *data;
00144     ++data;
00145   }
00146   if (h < 0) { h = -h; }
00147   return h;
00148 }
00149 
00150 using namespace Marsyas;
00151 
00152 Token::Token() {
00153   kind = 0;
00154   pos  = 0;
00155   col  = 0;
00156   line = 0;
00157   val  = NULL;
00158   next = NULL;
00159 }
00160 
00161 Token::~Token() {
00162   coco_string_delete(val);
00163 }
00164 
00165 #include <iostream>
00166 Buffer::Buffer(const char* s) {
00167   stream = NULL; // not a file
00168   this->isUserStream = true; // so Buffer doesn't try to close file
00169   int l; for (l=0; s[l]!='\0'; l++); // find string length
00170   fileLen = bufLen = l;
00171 //    std::cout << "len=" << bufLen << std::endl;
00172   if (bufLen>MAX_BUFFER_LENGTH) { bufLen=MAX_BUFFER_LENGTH; }
00173   buf = new char[bufLen];
00174   for (int i=0; i<fileLen; ++i) { buf[i]=s[i]; }
00175   bufStart = 0; // set to start of buffer
00176   SetPos(0);          // setup  buffer to position 0 (start)
00177   if (bufLen == fileLen) Close();
00178 }
00179 
00180 Buffer::Buffer(FILE* s, bool isUserStream) {
00181   stream = s; this->isUserStream = isUserStream;
00182   fseek(s, 0, SEEK_END);
00183   fileLen = bufLen = ftell(s);
00184   fseek(s, 0, SEEK_SET);
00185   buf = new char[MAX_BUFFER_LENGTH];
00186   bufStart = INT_MAX; // nothing in the buffer so far
00187   SetPos(0);          // setup  buffer to position 0 (start)
00188   if (bufLen == fileLen) Close();
00189 }
00190 
00191 Buffer::Buffer(Buffer *b) {
00192   buf = b->buf;
00193   b->buf = NULL;
00194   bufStart = b->bufStart;
00195   bufLen = b->bufLen;
00196   fileLen = b->fileLen;
00197   pos = b->pos;
00198   stream = b->stream;
00199   b->stream = NULL;
00200   isUserStream = b->isUserStream;
00201 }
00202 
00203 Buffer::~Buffer() {
00204   Close();
00205   if (buf != NULL) {
00206     delete [] buf;
00207     buf = NULL;
00208   }
00209 }
00210 
00211 void Buffer::Close() {
00212   if (!isUserStream && stream != NULL) {
00213     fclose(stream);
00214     stream = NULL;
00215   }
00216 }
00217 
00218 int Buffer::Read() {
00219   if (pos < bufLen) {
00220     return buf[pos++];
00221   } else if (GetPos() < fileLen) {
00222     SetPos(GetPos()); // shift buffer start to Pos
00223     return buf[pos++];
00224   } else {
00225     return EoF;
00226   }
00227 }
00228 
00229 int Buffer::Peek() {
00230   int curPos = GetPos();
00231   int ch = Read();
00232   SetPos(curPos);
00233   return ch;
00234 }
00235 
00236 char* Buffer::GetString(int beg, int end) {
00237   int len = end - beg;
00238   char *buf = new char[len];
00239   int oldPos = GetPos();
00240   SetPos(beg);
00241   for (int i = 0; i < len; ++i) buf[i] = (char) Read();
00242   SetPos(oldPos);
00243   return buf;
00244 }
00245 
00246 int Buffer::GetPos() {
00247   return pos + bufStart;
00248 }
00249 
00250 void Buffer::SetPos(int value) {
00251   if (value < 0) value = 0;
00252   else if (value > fileLen) value = fileLen;
00253   if (value >= bufStart && value < bufStart + bufLen) { // already in buffer
00254     pos = value - bufStart;
00255   } else if (stream != NULL) { // must be swapped in
00256     fseek(stream, value, SEEK_SET);
00257     bufLen = (int)fread(buf, sizeof(char), MAX_BUFFER_LENGTH, stream);
00258     bufStart = value; pos = 0;
00259   } else {
00260     pos = fileLen - bufStart; // make Pos return fileLen
00261   }
00262 }
00263 
00264 int UTF8Buffer::Read() {
00265   int ch;
00266   do {
00267     ch = Buffer::Read();
00268     // until we find a uft8 start (0xxxxxxx or 11xxxxxx)
00269   } while ((ch >= 128) && ((ch & 0xC0) != 0xC0) && (ch != EOF));
00270   if (ch < 128 || ch == EOF) {
00271     // nothing to do, first 127 chars are the same in ascii and utf8
00272     // 0xxxxxxx or end of file character
00273   } else if ((ch & 0xF0) == 0xF0) {
00274     // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
00275     int c1 = ch & 0x07; ch = Buffer::Read();
00276     int c2 = ch & 0x3F; ch = Buffer::Read();
00277     int c3 = ch & 0x3F; ch = Buffer::Read();
00278     int c4 = ch & 0x3F;
00279     ch = (((((c1 << 6) | c2) << 6) | c3) << 6) | c4;
00280   } else if ((ch & 0xE0) == 0xE0) {
00281     // 1110xxxx 10xxxxxx 10xxxxxx
00282     int c1 = ch & 0x0F; ch = Buffer::Read();
00283     int c2 = ch & 0x3F; ch = Buffer::Read();
00284     int c3 = ch & 0x3F;
00285     ch = (((c1 << 6) | c2) << 6) | c3;
00286   } else if ((ch & 0xC0) == 0xC0) {
00287     // 110xxxxx 10xxxxxx
00288     int c1 = ch & 0x1F; ch = Buffer::Read();
00289     int c2 = ch & 0x3F;
00290     ch = (c1 << 6) | c2;
00291   }
00292   return ch;
00293 }
00294 
00295 ExScanner::ExScanner(char* fileName) {
00296   FILE* stream;
00297   char *chFileName = coco_string_create(fileName);
00298   if ((stream = fopen(chFileName, "rb")) == NULL) {
00299     MRSWARN((std::string)"ExScanner: Cannot open file %s"+fileName);
00300     exit(1);
00301   }
00302   coco_string_delete(chFileName);
00303   buffer = new Buffer(stream, false);
00304   Init();
00305 }
00306 
00307 ExScanner::ExScanner(FILE* s) {
00308   buffer = new Buffer(s, true);
00309   Init();
00310 }
00311 
00312 ExScanner::ExScanner(char* s, int i) {
00313   (void) i; // FIXME This parameter is unused
00314   buffer = new Buffer(s);
00315   Init();
00316 }
00317 ExScanner::ExScanner() { tval=NULL; buffer=NULL; tail=NULL; }
00318 void ExScanner::setString(const char* s)
00319 {
00320   delete [] tval;
00321   delete buffer;
00322   while (tail!=NULL) { Token* y=tail->next; delete tail; tail=y; }
00323   buffer = new Buffer(s);
00324   Init();
00325 }
00326 
00327 ExScanner::~ExScanner() {
00328   delete [] tval;
00329   delete buffer;
00330   while (tail!=NULL) { Token* y=tail->next; delete tail; tail=y; }
00331 }
00332 
00333 void ExScanner::Init() {
00334   EOL    = '\n';
00335   eofSym = 0;
00336   maxT = 61;
00337   noSym = 61;
00338   for (int i = 48; i <= 57; ++i) start.set(i, 52);
00339   for (int i = 46; i <= 46; ++i) start.set(i, 3);
00340   for (int i = 39; i <= 39; ++i) start.set(i, 4);
00341   for (int i = 65; i <= 90; ++i) start.set(i, 53);
00342   for (int i = 97; i <= 122; ++i) start.set(i, 53);
00343   for (int i = 47; i <= 47; ++i) start.set(i, 54);
00344   start.set(36, 55);
00345   start.set(62, 56);
00346   start.set(60, 57);
00347   start.set(43, 58);
00348   start.set(45, 59);
00349   start.set(42, 60);
00350   start.set(37, 61);
00351   start.set(38, 62);
00352   start.set(124, 63);
00353   start.set(94, 37);
00354   start.set(40, 38);
00355   start.set(41, 39);
00356   start.set(33, 64);
00357   start.set(61, 41);
00358   start.set(44, 45);
00359   start.set(123, 65);
00360   start.set(125, 46);
00361   start.set(64, 48);
00362   start.set(91, 49);
00363   start.set(93, 50);
00364   start.set(58, 51);
00365   start.set(Buffer::EoF, -1);
00366   keywords.set((char *)"/", 26);
00367   keywords.set((char *)".", 47);
00368   keywords.set((char *)"Stream", 51);
00369   keywords.set((char *)"true", 52);
00370   keywords.set((char *)"false", 53);
00371   keywords.set((char *)"map", 54);
00372   keywords.set((char *)"iter", 55);
00373   keywords.set((char *)"for", 56);
00374   keywords.set((char *)"rfor", 57);
00375   keywords.set((char *)"in", 58);
00376   keywords.set((char *)"use", 59);
00377   keywords.set((char *)"load", 60);
00378 
00379 
00380   tvalLength = 128;
00381   tval = new char[tvalLength]; // text of current token
00382 
00383   pos = -1; line = 1; col = 0;
00384   oldEols = 0;
00385   NextCh();
00386   if (ch == 0xEF) { // check optional byte order mark for UTF-8
00387     NextCh(); int ch1 = ch;
00388     NextCh(); int ch2 = ch;
00389     if (ch1 != 0xBB || ch2 != 0xBF) {
00390       MRSWARN("ExScanner: Illegal byte order mark at start of file.");
00391 //          exit(1);
00392     }
00393     Buffer *oldBuf = buffer;
00394     buffer = new UTF8Buffer(buffer); col = 0;
00395     delete oldBuf; oldBuf = NULL;
00396     NextCh();
00397   }
00398 
00399 
00400   tail = pt = tokens = CreateToken(); // first token is a dummy
00401   pt->val=new char[1]; pt->val[0]='\0';
00402 }
00403 
00404 void ExScanner::NextCh() {
00405   if (oldEols > 0) { ch = EOL; oldEols--; }
00406   else {
00407     pos = buffer->GetPos();
00408     ch = buffer->Read(); col++;
00409     // replace isolated '\r' by '\n' in order to make
00410     // eol handling uniform across Windows, Unix and Mac
00411     if (ch == '\r' && buffer->Peek() != '\n') ch = EOL;
00412     if (ch == EOL) { line++; col = 0; }
00413   }
00414 
00415 }
00416 
00417 void ExScanner::AddCh() {
00418   if (tlen >= tvalLength) {
00419     tvalLength *= 2;
00420     char* newBuf = new char[tvalLength];
00421     memcpy(newBuf, tval, tlen*sizeof(char));
00422     delete tval;
00423     tval = newBuf;
00424   }
00425   tval[tlen++] = ch;
00426   NextCh();
00427 }
00428 
00429 
00430 bool ExScanner::Comment0() {
00431   int level = 1, pos0 = pos, line0 = line, col0 = col;
00432   (void) pos0; (void) col0; // FIXME Unused variables
00433   NextCh();
00434   for(;;) {
00435     if (ch == 10) {
00436       level--;
00437       if (level == 0) { oldEols = line - line0; NextCh(); return true; }
00438       NextCh();
00439     } else if (ch == buffer->EoF) return false;
00440     else NextCh();
00441   }
00442 }
00443 
00444 bool ExScanner::Comment1() {
00445   int level = 1, pos0 = pos, line0 = line, col0 = col;
00446   NextCh();
00447   if (ch == '*') {
00448     NextCh();
00449     for(;;) {
00450       if (ch == '*') {
00451         NextCh();
00452         if (ch == ')') {
00453           level--;
00454           if (level == 0) { oldEols = line - line0; NextCh(); return true; }
00455           NextCh();
00456         }
00457       } else if (ch == '(') {
00458         NextCh();
00459         if (ch == '*') {
00460           level++; NextCh();
00461         }
00462       } else if (ch == buffer->EoF) return false;
00463       else NextCh();
00464     }
00465   } else {
00466     buffer->SetPos(pos0); NextCh(); line = line0; col = col0;
00467   }
00468   return false;
00469 }
00470 
00471 
00472 Token* ExScanner::CreateToken() { return new Token(); }
00473 
00474 Token* ExScanner::NextToken() {
00475   while ((ch == L' ') || ((ch >= 9) && (ch <= 10)) || (ch == 13))
00476     NextCh();
00477   if (((ch == '#') && (Comment0())) || ((ch == '(') && (Comment1())))
00478     return NextToken();
00479   int apx = 0;
00480   t = CreateToken();
00481   t->pos = pos; t->col = col; t->line = line;
00482   int state = start.state(ch);
00483   tlen = 0; AddCh();
00484 
00485   switch (state) {
00486   case -1: { t->kind = eofSym; break; } // NextCh already done
00487   case 0: { t->kind = noSym; break; }   // NextCh already done
00488   case 1:
00489 case_1:
00490     {
00491       tlen -= apx;
00492       buffer->SetPos(t->pos); NextCh(); line = t->line; col = t->col;
00493       for (int i = 0; i < tlen; ++i) NextCh();
00494       t->kind = 1; break;
00495     }
00496   case 2:
00497 case_2:
00498     if (ch >= '0' && ch <= '9') {AddCh(); goto case_2;}
00499     else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00500   case 3:
00501 case_3:
00502     if (ch >= '0' && ch <= '9') {AddCh(); goto case_3;}
00503     else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00504   case 4:
00505 case_4:
00506     if (ch == 39) {AddCh(); goto case_5;}
00507     else if ((ch <= '&') || ((ch >= '(') && (ch <= 65535))) {AddCh(); goto case_4;}
00508     else {t->kind = noSym; break;}
00509   case 5:
00510 case_5:
00511     {t->kind = 3; break;}
00512   case 6:
00513 case_6:
00514     {t->kind = 4; break;}
00515   case 7:
00516 case_7:
00517     if (ch == '/') {AddCh(); goto case_8;}
00518     else if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '_') || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_7;}
00519     else {t->kind = noSym; break;}
00520   case 8:
00521 case_8:
00522     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_9;}
00523     else {t->kind = noSym; break;}
00524   case 9:
00525 case_9:
00526     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '_') || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_10;}
00527     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00528   case 10:
00529 case_10:
00530     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_10;}
00531     else if (ch == '/') {AddCh(); goto case_11;}
00532     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00533   case 11:
00534 case_11:
00535     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_12;}
00536     else {t->kind = noSym; break;}
00537   case 12:
00538 case_12:
00539     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_66;}
00540     else if (ch == '/') {AddCh(); goto case_11;}
00541     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00542   case 13:
00543 case_13:
00544     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_13;}
00545     else if (ch == '/') {AddCh(); goto case_14;}
00546     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00547   case 14:
00548 case_14:
00549     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_15;}
00550     else {t->kind = noSym; break;}
00551   case 15:
00552 case_15:
00553     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_67;}
00554     else if (ch == '/') {AddCh(); goto case_14;}
00555     else if (((ch >= '0') && (ch <= '9')) || (ch == '_')) {AddCh(); goto case_15;}
00556     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00557   case 16:
00558 case_16:
00559     {t->kind = 7; break;}
00560   case 17:
00561 case_17:
00562     if (ch == '>') {AddCh(); goto case_18;}
00563     else {t->kind = noSym; break;}
00564   case 18:
00565 case_18:
00566     {t->kind = 9; break;}
00567   case 19:
00568 case_19:
00569     {t->kind = 10; break;}
00570   case 20:
00571 case_20:
00572     if (ch == '>') {AddCh(); goto case_21;}
00573     else {t->kind = noSym; break;}
00574   case 21:
00575 case_21:
00576     {t->kind = 11; break;}
00577   case 22:
00578 case_22:
00579     if (ch == '>') {AddCh(); goto case_23;}
00580     else {t->kind = noSym; break;}
00581   case 23:
00582 case_23:
00583     {t->kind = 12; break;}
00584   case 24:
00585 case_24:
00586     if (ch == '>') {AddCh(); goto case_25;}
00587     else {t->kind = noSym; break;}
00588   case 25:
00589 case_25:
00590     {t->kind = 13; break;}
00591   case 26:
00592 case_26:
00593     {t->kind = 14; break;}
00594   case 27:
00595 case_27:
00596     {t->kind = 15; break;}
00597   case 28:
00598 case_28:
00599     {t->kind = 16; break;}
00600   case 29:
00601 case_29:
00602     {t->kind = 17; break;}
00603   case 30:
00604 case_30:
00605     {t->kind = 18; break;}
00606   case 31:
00607 case_31:
00608     if (ch == '>') {AddCh(); goto case_32;}
00609     else {t->kind = noSym; break;}
00610   case 32:
00611 case_32:
00612     {t->kind = 19; break;}
00613   case 33:
00614 case_33:
00615     if (ch == '>') {AddCh(); goto case_34;}
00616     else {t->kind = noSym; break;}
00617   case 34:
00618 case_34:
00619     {t->kind = 20; break;}
00620   case 35:
00621 case_35:
00622     {t->kind = 21; break;}
00623   case 36:
00624 case_36:
00625     {t->kind = 22; break;}
00626   case 37:
00627   {t->kind = 28; break;}
00628   case 38:
00629   {t->kind = 29; break;}
00630   case 39:
00631   {t->kind = 30; break;}
00632   case 40:
00633 case_40:
00634     {t->kind = 32; break;}
00635   case 41:
00636   {t->kind = 34; break;}
00637   case 42:
00638 case_42:
00639     {t->kind = 35; break;}
00640   case 43:
00641 case_43:
00642     {t->kind = 37; break;}
00643   case 44:
00644 case_44:
00645     {t->kind = 39; break;}
00646   case 45:
00647   {t->kind = 42; break;}
00648   case 46:
00649   {t->kind = 44; break;}
00650   case 47:
00651 case_47:
00652     {t->kind = 45; break;}
00653   case 48:
00654   {t->kind = 46; break;}
00655   case 49:
00656   {t->kind = 48; break;}
00657   case 50:
00658   {t->kind = 49; break;}
00659   case 51:
00660   {t->kind = 50; break;}
00661   case 52:
00662 case_52:
00663     if ((ch >= '0') && (ch <= '9')) {AddCh(); goto case_52;}
00664     else if (ch == '.') {apx++; AddCh(); goto case_68;}
00665     else {t->kind = 1; break;}
00666   case 53:
00667 case_53:
00668     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_53;}
00669     else if (ch == '/') {AddCh(); goto case_8;}
00670     else if (ch == '_') {AddCh(); goto case_7;}
00671     else {t->kind = 5; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00672   case 54:
00673     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_13;}
00674     else if (ch == '>') {AddCh(); goto case_22;}
00675     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00676   case 55:
00677     if (ch == 'f' || ch == 't') {AddCh(); goto case_6;}
00678     else {t->kind = noSym; break;}
00679   case 56:
00680     if (ch == '>') {AddCh(); goto case_16;}
00681     else if (ch == '=') {AddCh(); goto case_43;}
00682     else {t->kind = 36; break;}
00683   case 57:
00684     if (ch == '<') {AddCh(); goto case_69;}
00685     else if (ch == '-') {AddCh(); goto case_40;}
00686     else if (ch == '=') {AddCh(); goto case_44;}
00687     else {t->kind = 38; break;}
00688   case 58:
00689     if (ch == '>') {AddCh(); goto case_17;}
00690     else {t->kind = 23; break;}
00691   case 59:
00692     if (ch == '>') {AddCh(); goto case_70;}
00693     else {t->kind = 24; break;}
00694   case 60:
00695     if (ch == '>') {AddCh(); goto case_20;}
00696     else {t->kind = 25; break;}
00697   case 61:
00698     if (ch == '>') {AddCh(); goto case_24;}
00699     else {t->kind = 27; break;}
00700   case 62:
00701     if (ch == '>') {AddCh(); goto case_31;}
00702     else {t->kind = 40; break;}
00703   case 63:
00704     if (ch == '>') {AddCh(); goto case_33;}
00705     else {t->kind = 41; break;}
00706   case 64:
00707     if (ch == '=') {AddCh(); goto case_42;}
00708     else {t->kind = 33; break;}
00709   case 65:
00710     if (ch == '?') {AddCh(); goto case_47;}
00711     else {t->kind = 43; break;}
00712   case 66:
00713 case_66:
00714     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_66;}
00715     else if (ch == '/') {AddCh(); goto case_11;}
00716     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00717   case 67:
00718 case_67:
00719     if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_67;}
00720     else if (ch == '/') {AddCh(); goto case_14;}
00721     else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00722   case 68:
00723 case_68:
00724     if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {apx++; AddCh(); goto case_1;}
00725     else if ((ch >= '0') && (ch <= '9')) {apx = 0; AddCh(); goto case_2;}
00726     else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;}
00727   case 69:
00728 case_69:
00729     if (ch == '+') {AddCh(); goto case_26;}
00730     else if (ch == '-') {AddCh(); goto case_27;}
00731     else if (ch == '*') {AddCh(); goto case_28;}
00732     else if (ch == '/') {AddCh(); goto case_29;}
00733     else if (ch == '%') {AddCh(); goto case_30;}
00734     else if (ch == '&') {AddCh(); goto case_35;}
00735     else if (ch == '|') {AddCh(); goto case_36;}
00736     else {t->kind = 8; break;}
00737   case 70:
00738 case_70:
00739     if (ch == '>') {AddCh(); goto case_19;}
00740     else {t->kind = 31; break;}
00741 
00742   }
00743   t->val=coco_string_create(tval, 0, tlen);
00744   return t;
00745 }
00746 
00747 // get the next token (possibly a token already seen during peeking)
00748 Token* ExScanner::Scan() {
00749   if (tokens->next == NULL) { tokens->next=NextToken(); }
00750   tokens=tokens->next; pt=tokens;
00751   return tokens;
00752 }
00753 
00754 // peek for the next token, ignore pragmas
00755 Token* ExScanner::Peek() {
00756   if (pt->next == NULL) {
00757     do {
00758       pt = pt->next = NextToken();
00759     } while (pt->kind > maxT); // skip pragmas
00760   } else {
00761     do {
00762       pt = pt->next;
00763     } while (pt->kind > maxT);
00764   }
00765   return pt;
00766 }
00767 
00768 // make sure that peeking starts at the current scan position
00769 void ExScanner::ResetPeek() {
00770   pt = tokens;
00771 }