Marsyas
0.6.0-alpha
|
00001 #include <marsyas/expr/ExScanner.h> 00002 00003 #include <memory.h> 00004 #include <string.h> 00005 00006 // string handling, wide character 00007 00008 char* coco_string_create(const char* value) { 00009 char* data; 00010 int len = 0; 00011 if (value) { len =(int)strlen(value); } 00012 data = new char[len + 1]; 00013 strncpy(data, value, len); 00014 data[len] = 0; 00015 return data; 00016 } 00017 00018 char* coco_string_create(const char* value , int startIndex, int length) { 00019 int len = 0; 00020 char* data; 00021 00022 if (value) { len = length; } 00023 data = new char[len + 1]; 00024 strncpy(data, &(value[startIndex]), len); 00025 data[len] = 0; 00026 00027 return data; 00028 } 00029 00030 char* coco_string_create_upper(char* data) { 00031 if (!data) { return NULL; } 00032 00033 int dataLen = 0; 00034 if (data) { dataLen =(int)strlen(data); } 00035 00036 char* newData = new char[dataLen + 1]; 00037 00038 for (int i = 0; i <= dataLen; ++i) { 00039 if (('a' <= data[i]) && (data[i] <= 'z')) { 00040 newData[i] = data[i] + ('A' - 'a'); 00041 } 00042 else { newData[i] = data[i]; } 00043 } 00044 00045 newData[dataLen] = '\0'; 00046 return newData; 00047 } 00048 00049 char* coco_string_create_lower(char* data) { 00050 if (!data) { return NULL; } 00051 00052 int dataLen = 0; 00053 if (data) { dataLen =(int)strlen(data); } 00054 00055 char* newData = new char[dataLen + 1]; 00056 00057 for (int i = 0; i <= dataLen; ++i) { 00058 if (('A' <= data[i]) && (data[i] <= 'Z')) { 00059 newData[i] = data[i] - ('A'- 'a'); 00060 } 00061 else { newData[i] = data[i]; } 00062 } 00063 newData[dataLen] = '\0'; 00064 return newData; 00065 } 00066 00067 char* coco_string_create_append(const char* data1, const char* data2) { 00068 char* data; 00069 int data1Len = 0; 00070 int data2Len = 0; 00071 00072 if (data1) { data1Len =(int)strlen(data1); } 00073 if (data2) {data2Len =(int)strlen(data2); } 00074 00075 data = new char[data1Len + data2Len + 1]; 00076 00077 if (data1) { strcpy(data, data1); } 00078 if (data2) { strcpy(data + data1Len, data2); } 00079 00080 data[data1Len + data2Len] = 0; 00081 00082 return data; 00083 } 00084 00085 char* coco_string_create_append(const char* target, const char appendix) { 00086 int targetLen = coco_string_length(target); 00087 char* data = new char[targetLen + 2]; 00088 strncpy(data, target, targetLen); 00089 data[targetLen] = appendix; 00090 data[targetLen + 1] = 0; 00091 return data; 00092 } 00093 00094 void coco_string_delete(char* &data) { 00095 delete [] data; 00096 data = NULL; 00097 } 00098 00099 int coco_string_length(const char* data) { 00100 if (data) { return (int)strlen(data); } 00101 return 0; 00102 } 00103 00104 bool coco_string_endswith(char* data, char* end) { 00105 int dataLen =(int)strlen(data); 00106 int endLen =(int)strlen(end); 00107 return (endLen <= dataLen) && (strcmp(data + dataLen - endLen, end) == 0); 00108 } 00109 00110 int coco_string_indexof(char* data, char value) { 00111 char* chr = strchr(data, value); 00112 00113 if (chr) { return (int)(chr-data); } 00114 return -1; 00115 } 00116 00117 int coco_string_lastindexof(char* data, char value) { 00118 char* chr = strrchr(data, value); 00119 00120 if (chr) { return (int)(chr-data); } 00121 return -1; 00122 } 00123 00124 void coco_string_merge(char* &target, char* appendix) { 00125 if (!appendix) { return; } 00126 char* data = coco_string_create_append(target, appendix); 00127 delete [] target; 00128 target = data; 00129 } 00130 00131 bool coco_string_equal(char* data1, char* data2) { 00132 return strcmp( data1, data2 ) == 0; 00133 } 00134 00135 int coco_string_compareto(char* data1, char* data2) { 00136 return strcmp(data1, data2); 00137 } 00138 00139 int coco_string_hash(char* data) { 00140 int h = 0; 00141 if (!data) { return 0; } 00142 while (*data != 0) { 00143 h = (h * 7) ^ *data; 00144 ++data; 00145 } 00146 if (h < 0) { h = -h; } 00147 return h; 00148 } 00149 00150 using namespace Marsyas; 00151 00152 Token::Token() { 00153 kind = 0; 00154 pos = 0; 00155 col = 0; 00156 line = 0; 00157 val = NULL; 00158 next = NULL; 00159 } 00160 00161 Token::~Token() { 00162 coco_string_delete(val); 00163 } 00164 00165 #include <iostream> 00166 Buffer::Buffer(const char* s) { 00167 stream = NULL; // not a file 00168 this->isUserStream = true; // so Buffer doesn't try to close file 00169 int l; for (l=0; s[l]!='\0'; l++); // find string length 00170 fileLen = bufLen = l; 00171 // std::cout << "len=" << bufLen << std::endl; 00172 if (bufLen>MAX_BUFFER_LENGTH) { bufLen=MAX_BUFFER_LENGTH; } 00173 buf = new char[bufLen]; 00174 for (int i=0; i<fileLen; ++i) { buf[i]=s[i]; } 00175 bufStart = 0; // set to start of buffer 00176 SetPos(0); // setup buffer to position 0 (start) 00177 if (bufLen == fileLen) Close(); 00178 } 00179 00180 Buffer::Buffer(FILE* s, bool isUserStream) { 00181 stream = s; this->isUserStream = isUserStream; 00182 fseek(s, 0, SEEK_END); 00183 fileLen = bufLen = ftell(s); 00184 fseek(s, 0, SEEK_SET); 00185 buf = new char[MAX_BUFFER_LENGTH]; 00186 bufStart = INT_MAX; // nothing in the buffer so far 00187 SetPos(0); // setup buffer to position 0 (start) 00188 if (bufLen == fileLen) Close(); 00189 } 00190 00191 Buffer::Buffer(Buffer *b) { 00192 buf = b->buf; 00193 b->buf = NULL; 00194 bufStart = b->bufStart; 00195 bufLen = b->bufLen; 00196 fileLen = b->fileLen; 00197 pos = b->pos; 00198 stream = b->stream; 00199 b->stream = NULL; 00200 isUserStream = b->isUserStream; 00201 } 00202 00203 Buffer::~Buffer() { 00204 Close(); 00205 if (buf != NULL) { 00206 delete [] buf; 00207 buf = NULL; 00208 } 00209 } 00210 00211 void Buffer::Close() { 00212 if (!isUserStream && stream != NULL) { 00213 fclose(stream); 00214 stream = NULL; 00215 } 00216 } 00217 00218 int Buffer::Read() { 00219 if (pos < bufLen) { 00220 return buf[pos++]; 00221 } else if (GetPos() < fileLen) { 00222 SetPos(GetPos()); // shift buffer start to Pos 00223 return buf[pos++]; 00224 } else { 00225 return EoF; 00226 } 00227 } 00228 00229 int Buffer::Peek() { 00230 int curPos = GetPos(); 00231 int ch = Read(); 00232 SetPos(curPos); 00233 return ch; 00234 } 00235 00236 char* Buffer::GetString(int beg, int end) { 00237 int len = end - beg; 00238 char *buf = new char[len]; 00239 int oldPos = GetPos(); 00240 SetPos(beg); 00241 for (int i = 0; i < len; ++i) buf[i] = (char) Read(); 00242 SetPos(oldPos); 00243 return buf; 00244 } 00245 00246 int Buffer::GetPos() { 00247 return pos + bufStart; 00248 } 00249 00250 void Buffer::SetPos(int value) { 00251 if (value < 0) value = 0; 00252 else if (value > fileLen) value = fileLen; 00253 if (value >= bufStart && value < bufStart + bufLen) { // already in buffer 00254 pos = value - bufStart; 00255 } else if (stream != NULL) { // must be swapped in 00256 fseek(stream, value, SEEK_SET); 00257 bufLen = (int)fread(buf, sizeof(char), MAX_BUFFER_LENGTH, stream); 00258 bufStart = value; pos = 0; 00259 } else { 00260 pos = fileLen - bufStart; // make Pos return fileLen 00261 } 00262 } 00263 00264 int UTF8Buffer::Read() { 00265 int ch; 00266 do { 00267 ch = Buffer::Read(); 00268 // until we find a uft8 start (0xxxxxxx or 11xxxxxx) 00269 } while ((ch >= 128) && ((ch & 0xC0) != 0xC0) && (ch != EOF)); 00270 if (ch < 128 || ch == EOF) { 00271 // nothing to do, first 127 chars are the same in ascii and utf8 00272 // 0xxxxxxx or end of file character 00273 } else if ((ch & 0xF0) == 0xF0) { 00274 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 00275 int c1 = ch & 0x07; ch = Buffer::Read(); 00276 int c2 = ch & 0x3F; ch = Buffer::Read(); 00277 int c3 = ch & 0x3F; ch = Buffer::Read(); 00278 int c4 = ch & 0x3F; 00279 ch = (((((c1 << 6) | c2) << 6) | c3) << 6) | c4; 00280 } else if ((ch & 0xE0) == 0xE0) { 00281 // 1110xxxx 10xxxxxx 10xxxxxx 00282 int c1 = ch & 0x0F; ch = Buffer::Read(); 00283 int c2 = ch & 0x3F; ch = Buffer::Read(); 00284 int c3 = ch & 0x3F; 00285 ch = (((c1 << 6) | c2) << 6) | c3; 00286 } else if ((ch & 0xC0) == 0xC0) { 00287 // 110xxxxx 10xxxxxx 00288 int c1 = ch & 0x1F; ch = Buffer::Read(); 00289 int c2 = ch & 0x3F; 00290 ch = (c1 << 6) | c2; 00291 } 00292 return ch; 00293 } 00294 00295 ExScanner::ExScanner(char* fileName) { 00296 FILE* stream; 00297 char *chFileName = coco_string_create(fileName); 00298 if ((stream = fopen(chFileName, "rb")) == NULL) { 00299 MRSWARN((std::string)"ExScanner: Cannot open file %s"+fileName); 00300 exit(1); 00301 } 00302 coco_string_delete(chFileName); 00303 buffer = new Buffer(stream, false); 00304 Init(); 00305 } 00306 00307 ExScanner::ExScanner(FILE* s) { 00308 buffer = new Buffer(s, true); 00309 Init(); 00310 } 00311 00312 ExScanner::ExScanner(char* s, int i) { 00313 (void) i; // FIXME This parameter is unused 00314 buffer = new Buffer(s); 00315 Init(); 00316 } 00317 ExScanner::ExScanner() { tval=NULL; buffer=NULL; tail=NULL; } 00318 void ExScanner::setString(const char* s) 00319 { 00320 delete [] tval; 00321 delete buffer; 00322 while (tail!=NULL) { Token* y=tail->next; delete tail; tail=y; } 00323 buffer = new Buffer(s); 00324 Init(); 00325 } 00326 00327 ExScanner::~ExScanner() { 00328 delete [] tval; 00329 delete buffer; 00330 while (tail!=NULL) { Token* y=tail->next; delete tail; tail=y; } 00331 } 00332 00333 void ExScanner::Init() { 00334 EOL = '\n'; 00335 eofSym = 0; 00336 maxT = 61; 00337 noSym = 61; 00338 for (int i = 48; i <= 57; ++i) start.set(i, 52); 00339 for (int i = 46; i <= 46; ++i) start.set(i, 3); 00340 for (int i = 39; i <= 39; ++i) start.set(i, 4); 00341 for (int i = 65; i <= 90; ++i) start.set(i, 53); 00342 for (int i = 97; i <= 122; ++i) start.set(i, 53); 00343 for (int i = 47; i <= 47; ++i) start.set(i, 54); 00344 start.set(36, 55); 00345 start.set(62, 56); 00346 start.set(60, 57); 00347 start.set(43, 58); 00348 start.set(45, 59); 00349 start.set(42, 60); 00350 start.set(37, 61); 00351 start.set(38, 62); 00352 start.set(124, 63); 00353 start.set(94, 37); 00354 start.set(40, 38); 00355 start.set(41, 39); 00356 start.set(33, 64); 00357 start.set(61, 41); 00358 start.set(44, 45); 00359 start.set(123, 65); 00360 start.set(125, 46); 00361 start.set(64, 48); 00362 start.set(91, 49); 00363 start.set(93, 50); 00364 start.set(58, 51); 00365 start.set(Buffer::EoF, -1); 00366 keywords.set((char *)"/", 26); 00367 keywords.set((char *)".", 47); 00368 keywords.set((char *)"Stream", 51); 00369 keywords.set((char *)"true", 52); 00370 keywords.set((char *)"false", 53); 00371 keywords.set((char *)"map", 54); 00372 keywords.set((char *)"iter", 55); 00373 keywords.set((char *)"for", 56); 00374 keywords.set((char *)"rfor", 57); 00375 keywords.set((char *)"in", 58); 00376 keywords.set((char *)"use", 59); 00377 keywords.set((char *)"load", 60); 00378 00379 00380 tvalLength = 128; 00381 tval = new char[tvalLength]; // text of current token 00382 00383 pos = -1; line = 1; col = 0; 00384 oldEols = 0; 00385 NextCh(); 00386 if (ch == 0xEF) { // check optional byte order mark for UTF-8 00387 NextCh(); int ch1 = ch; 00388 NextCh(); int ch2 = ch; 00389 if (ch1 != 0xBB || ch2 != 0xBF) { 00390 MRSWARN("ExScanner: Illegal byte order mark at start of file."); 00391 // exit(1); 00392 } 00393 Buffer *oldBuf = buffer; 00394 buffer = new UTF8Buffer(buffer); col = 0; 00395 delete oldBuf; oldBuf = NULL; 00396 NextCh(); 00397 } 00398 00399 00400 tail = pt = tokens = CreateToken(); // first token is a dummy 00401 pt->val=new char[1]; pt->val[0]='\0'; 00402 } 00403 00404 void ExScanner::NextCh() { 00405 if (oldEols > 0) { ch = EOL; oldEols--; } 00406 else { 00407 pos = buffer->GetPos(); 00408 ch = buffer->Read(); col++; 00409 // replace isolated '\r' by '\n' in order to make 00410 // eol handling uniform across Windows, Unix and Mac 00411 if (ch == '\r' && buffer->Peek() != '\n') ch = EOL; 00412 if (ch == EOL) { line++; col = 0; } 00413 } 00414 00415 } 00416 00417 void ExScanner::AddCh() { 00418 if (tlen >= tvalLength) { 00419 tvalLength *= 2; 00420 char* newBuf = new char[tvalLength]; 00421 memcpy(newBuf, tval, tlen*sizeof(char)); 00422 delete tval; 00423 tval = newBuf; 00424 } 00425 tval[tlen++] = ch; 00426 NextCh(); 00427 } 00428 00429 00430 bool ExScanner::Comment0() { 00431 int level = 1, pos0 = pos, line0 = line, col0 = col; 00432 (void) pos0; (void) col0; // FIXME Unused variables 00433 NextCh(); 00434 for(;;) { 00435 if (ch == 10) { 00436 level--; 00437 if (level == 0) { oldEols = line - line0; NextCh(); return true; } 00438 NextCh(); 00439 } else if (ch == buffer->EoF) return false; 00440 else NextCh(); 00441 } 00442 } 00443 00444 bool ExScanner::Comment1() { 00445 int level = 1, pos0 = pos, line0 = line, col0 = col; 00446 NextCh(); 00447 if (ch == '*') { 00448 NextCh(); 00449 for(;;) { 00450 if (ch == '*') { 00451 NextCh(); 00452 if (ch == ')') { 00453 level--; 00454 if (level == 0) { oldEols = line - line0; NextCh(); return true; } 00455 NextCh(); 00456 } 00457 } else if (ch == '(') { 00458 NextCh(); 00459 if (ch == '*') { 00460 level++; NextCh(); 00461 } 00462 } else if (ch == buffer->EoF) return false; 00463 else NextCh(); 00464 } 00465 } else { 00466 buffer->SetPos(pos0); NextCh(); line = line0; col = col0; 00467 } 00468 return false; 00469 } 00470 00471 00472 Token* ExScanner::CreateToken() { return new Token(); } 00473 00474 Token* ExScanner::NextToken() { 00475 while ((ch == L' ') || ((ch >= 9) && (ch <= 10)) || (ch == 13)) 00476 NextCh(); 00477 if (((ch == '#') && (Comment0())) || ((ch == '(') && (Comment1()))) 00478 return NextToken(); 00479 int apx = 0; 00480 t = CreateToken(); 00481 t->pos = pos; t->col = col; t->line = line; 00482 int state = start.state(ch); 00483 tlen = 0; AddCh(); 00484 00485 switch (state) { 00486 case -1: { t->kind = eofSym; break; } // NextCh already done 00487 case 0: { t->kind = noSym; break; } // NextCh already done 00488 case 1: 00489 case_1: 00490 { 00491 tlen -= apx; 00492 buffer->SetPos(t->pos); NextCh(); line = t->line; col = t->col; 00493 for (int i = 0; i < tlen; ++i) NextCh(); 00494 t->kind = 1; break; 00495 } 00496 case 2: 00497 case_2: 00498 if (ch >= '0' && ch <= '9') {AddCh(); goto case_2;} 00499 else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00500 case 3: 00501 case_3: 00502 if (ch >= '0' && ch <= '9') {AddCh(); goto case_3;} 00503 else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00504 case 4: 00505 case_4: 00506 if (ch == 39) {AddCh(); goto case_5;} 00507 else if ((ch <= '&') || ((ch >= '(') && (ch <= 65535))) {AddCh(); goto case_4;} 00508 else {t->kind = noSym; break;} 00509 case 5: 00510 case_5: 00511 {t->kind = 3; break;} 00512 case 6: 00513 case_6: 00514 {t->kind = 4; break;} 00515 case 7: 00516 case_7: 00517 if (ch == '/') {AddCh(); goto case_8;} 00518 else if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '_') || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_7;} 00519 else {t->kind = noSym; break;} 00520 case 8: 00521 case_8: 00522 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_9;} 00523 else {t->kind = noSym; break;} 00524 case 9: 00525 case_9: 00526 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || (ch == '_') || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_10;} 00527 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00528 case 10: 00529 case_10: 00530 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_10;} 00531 else if (ch == '/') {AddCh(); goto case_11;} 00532 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00533 case 11: 00534 case_11: 00535 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_12;} 00536 else {t->kind = noSym; break;} 00537 case 12: 00538 case_12: 00539 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_66;} 00540 else if (ch == '/') {AddCh(); goto case_11;} 00541 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00542 case 13: 00543 case_13: 00544 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_13;} 00545 else if (ch == '/') {AddCh(); goto case_14;} 00546 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00547 case 14: 00548 case_14: 00549 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_15;} 00550 else {t->kind = noSym; break;} 00551 case 15: 00552 case_15: 00553 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_67;} 00554 else if (ch == '/') {AddCh(); goto case_14;} 00555 else if (((ch >= '0') && (ch <= '9')) || (ch == '_')) {AddCh(); goto case_15;} 00556 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00557 case 16: 00558 case_16: 00559 {t->kind = 7; break;} 00560 case 17: 00561 case_17: 00562 if (ch == '>') {AddCh(); goto case_18;} 00563 else {t->kind = noSym; break;} 00564 case 18: 00565 case_18: 00566 {t->kind = 9; break;} 00567 case 19: 00568 case_19: 00569 {t->kind = 10; break;} 00570 case 20: 00571 case_20: 00572 if (ch == '>') {AddCh(); goto case_21;} 00573 else {t->kind = noSym; break;} 00574 case 21: 00575 case_21: 00576 {t->kind = 11; break;} 00577 case 22: 00578 case_22: 00579 if (ch == '>') {AddCh(); goto case_23;} 00580 else {t->kind = noSym; break;} 00581 case 23: 00582 case_23: 00583 {t->kind = 12; break;} 00584 case 24: 00585 case_24: 00586 if (ch == '>') {AddCh(); goto case_25;} 00587 else {t->kind = noSym; break;} 00588 case 25: 00589 case_25: 00590 {t->kind = 13; break;} 00591 case 26: 00592 case_26: 00593 {t->kind = 14; break;} 00594 case 27: 00595 case_27: 00596 {t->kind = 15; break;} 00597 case 28: 00598 case_28: 00599 {t->kind = 16; break;} 00600 case 29: 00601 case_29: 00602 {t->kind = 17; break;} 00603 case 30: 00604 case_30: 00605 {t->kind = 18; break;} 00606 case 31: 00607 case_31: 00608 if (ch == '>') {AddCh(); goto case_32;} 00609 else {t->kind = noSym; break;} 00610 case 32: 00611 case_32: 00612 {t->kind = 19; break;} 00613 case 33: 00614 case_33: 00615 if (ch == '>') {AddCh(); goto case_34;} 00616 else {t->kind = noSym; break;} 00617 case 34: 00618 case_34: 00619 {t->kind = 20; break;} 00620 case 35: 00621 case_35: 00622 {t->kind = 21; break;} 00623 case 36: 00624 case_36: 00625 {t->kind = 22; break;} 00626 case 37: 00627 {t->kind = 28; break;} 00628 case 38: 00629 {t->kind = 29; break;} 00630 case 39: 00631 {t->kind = 30; break;} 00632 case 40: 00633 case_40: 00634 {t->kind = 32; break;} 00635 case 41: 00636 {t->kind = 34; break;} 00637 case 42: 00638 case_42: 00639 {t->kind = 35; break;} 00640 case 43: 00641 case_43: 00642 {t->kind = 37; break;} 00643 case 44: 00644 case_44: 00645 {t->kind = 39; break;} 00646 case 45: 00647 {t->kind = 42; break;} 00648 case 46: 00649 {t->kind = 44; break;} 00650 case 47: 00651 case_47: 00652 {t->kind = 45; break;} 00653 case 48: 00654 {t->kind = 46; break;} 00655 case 49: 00656 {t->kind = 48; break;} 00657 case 50: 00658 {t->kind = 49; break;} 00659 case 51: 00660 {t->kind = 50; break;} 00661 case 52: 00662 case_52: 00663 if ((ch >= '0') && (ch <= '9')) {AddCh(); goto case_52;} 00664 else if (ch == '.') {apx++; AddCh(); goto case_68;} 00665 else {t->kind = 1; break;} 00666 case 53: 00667 case_53: 00668 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_53;} 00669 else if (ch == '/') {AddCh(); goto case_8;} 00670 else if (ch == '_') {AddCh(); goto case_7;} 00671 else {t->kind = 5; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00672 case 54: 00673 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_13;} 00674 else if (ch == '>') {AddCh(); goto case_22;} 00675 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00676 case 55: 00677 if (ch == 'f' || ch == 't') {AddCh(); goto case_6;} 00678 else {t->kind = noSym; break;} 00679 case 56: 00680 if (ch == '>') {AddCh(); goto case_16;} 00681 else if (ch == '=') {AddCh(); goto case_43;} 00682 else {t->kind = 36; break;} 00683 case 57: 00684 if (ch == '<') {AddCh(); goto case_69;} 00685 else if (ch == '-') {AddCh(); goto case_40;} 00686 else if (ch == '=') {AddCh(); goto case_44;} 00687 else {t->kind = 38; break;} 00688 case 58: 00689 if (ch == '>') {AddCh(); goto case_17;} 00690 else {t->kind = 23; break;} 00691 case 59: 00692 if (ch == '>') {AddCh(); goto case_70;} 00693 else {t->kind = 24; break;} 00694 case 60: 00695 if (ch == '>') {AddCh(); goto case_20;} 00696 else {t->kind = 25; break;} 00697 case 61: 00698 if (ch == '>') {AddCh(); goto case_24;} 00699 else {t->kind = 27; break;} 00700 case 62: 00701 if (ch == '>') {AddCh(); goto case_31;} 00702 else {t->kind = 40; break;} 00703 case 63: 00704 if (ch == '>') {AddCh(); goto case_33;} 00705 else {t->kind = 41; break;} 00706 case 64: 00707 if (ch == '=') {AddCh(); goto case_42;} 00708 else {t->kind = 33; break;} 00709 case 65: 00710 if (ch == '?') {AddCh(); goto case_47;} 00711 else {t->kind = 43; break;} 00712 case 66: 00713 case_66: 00714 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_66;} 00715 else if (ch == '/') {AddCh(); goto case_11;} 00716 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00717 case 67: 00718 case_67: 00719 if (((ch >= '0') && (ch <= '9')) || ((ch >= 'A') && (ch <= 'Z')) || ch == '_' || ((ch >= 'a') && (ch <= 'z'))) {AddCh(); goto case_67;} 00720 else if (ch == '/') {AddCh(); goto case_14;} 00721 else {t->kind = 6; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00722 case 68: 00723 case_68: 00724 if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) {apx++; AddCh(); goto case_1;} 00725 else if ((ch >= '0') && (ch <= '9')) {apx = 0; AddCh(); goto case_2;} 00726 else {t->kind = 2; t->val = coco_string_create(tval, 0, tlen); t->kind = keywords.get(t->val, t->kind); return t;} 00727 case 69: 00728 case_69: 00729 if (ch == '+') {AddCh(); goto case_26;} 00730 else if (ch == '-') {AddCh(); goto case_27;} 00731 else if (ch == '*') {AddCh(); goto case_28;} 00732 else if (ch == '/') {AddCh(); goto case_29;} 00733 else if (ch == '%') {AddCh(); goto case_30;} 00734 else if (ch == '&') {AddCh(); goto case_35;} 00735 else if (ch == '|') {AddCh(); goto case_36;} 00736 else {t->kind = 8; break;} 00737 case 70: 00738 case_70: 00739 if (ch == '>') {AddCh(); goto case_19;} 00740 else {t->kind = 31; break;} 00741 00742 } 00743 t->val=coco_string_create(tval, 0, tlen); 00744 return t; 00745 } 00746 00747 // get the next token (possibly a token already seen during peeking) 00748 Token* ExScanner::Scan() { 00749 if (tokens->next == NULL) { tokens->next=NextToken(); } 00750 tokens=tokens->next; pt=tokens; 00751 return tokens; 00752 } 00753 00754 // peek for the next token, ignore pragmas 00755 Token* ExScanner::Peek() { 00756 if (pt->next == NULL) { 00757 do { 00758 pt = pt->next = NextToken(); 00759 } while (pt->kind > maxT); // skip pragmas 00760 } else { 00761 do { 00762 pt = pt->next; 00763 } while (pt->kind > maxT); 00764 } 00765 return pt; 00766 } 00767 00768 // make sure that peeking starts at the current scan position 00769 void ExScanner::ResetPeek() { 00770 pt = tokens; 00771 }