KiCad PCB EDA Suite
dsnlexer.cpp
Go to the documentation of this file.
1
2/*
3 * This program source code file is part of KiCad, a free EDA CAD application.
4 *
5 * Copyright (C) 2007-2013 SoftPLC Corporation, Dick Hollenbeck <[email protected]>
6 * Copyright (C) 2007-2020 KiCad Developers, see AUTHORS.txt for contributors.
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 2
11 * of the License, or (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, you may find one here:
20 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
21 * or you may search the http://www.gnu.org website for the version 2 license,
22 * or you may write to the Free Software Foundation, Inc.,
23 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
24 */
25
26#include <charconv>
27#include <cstdarg>
28#include <cstdio>
29#include <cstdlib> // bsearch()
30#include <cctype>
31
32#include <dsnlexer.h>
33#include <wx/translation.h>
34
35#define FMT_CLIPBOARD _( "clipboard" )
36
37
38//-----<DSNLEXER>-------------------------------------------------------------
39
41{
44
45 stringDelimiter = '"';
46
47 specctraMode = false;
49 commentsAreTokens = false;
50
51 curOffset = 0;
52}
53
54
55DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount,
56 const KEYWORD_MAP* aKeywordMap,
57 FILE* aFile, const wxString& aFilename ) :
58 iOwnReaders( true ),
59 start( nullptr ),
60 next( nullptr ),
61 limit( nullptr ),
62 reader( nullptr ),
63 keywords( aKeywordTable ),
64 keywordCount( aKeywordCount ),
65 keywordsLookup( aKeywordMap )
66{
67 FILE_LINE_READER* fileReader = new FILE_LINE_READER( aFile, aFilename );
68 PushReader( fileReader );
69 init();
70}
71
72
73DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount,
74 const KEYWORD_MAP* aKeywordMap,
75 const std::string& aClipboardTxt, const wxString& aSource ) :
76 iOwnReaders( true ),
77 start( nullptr ),
78 next( nullptr ),
79 limit( nullptr ),
80 reader( nullptr ),
81 keywords( aKeywordTable ),
82 keywordCount( aKeywordCount ),
83 keywordsLookup( aKeywordMap )
84{
85 STRING_LINE_READER* stringReader = new STRING_LINE_READER( aClipboardTxt, aSource.IsEmpty() ?
86 wxString( FMT_CLIPBOARD ) : aSource );
87 PushReader( stringReader );
88 init();
89}
90
91
92DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount,
93 const KEYWORD_MAP* aKeywordMap,
94 LINE_READER* aLineReader ) :
95 iOwnReaders( false ),
96 start( nullptr ),
97 next( nullptr ),
98 limit( nullptr ),
99 reader( nullptr ),
100 keywords( aKeywordTable ),
101 keywordCount( aKeywordCount ),
102 keywordsLookup( aKeywordMap )
103{
104 if( aLineReader )
105 PushReader( aLineReader );
106 init();
107}
108
109
110static const KEYWORD empty_keywords[1] = {};
111
112DSNLEXER::DSNLEXER( const std::string& aSExpression, const wxString& aSource ) :
113 iOwnReaders( true ),
114 start( nullptr ),
115 next( nullptr ),
116 limit( nullptr ),
117 reader( nullptr ),
118 keywords( empty_keywords ),
119 keywordCount( 0 ),
120 keywordsLookup( nullptr )
121{
122 STRING_LINE_READER* stringReader = new STRING_LINE_READER( aSExpression, aSource.IsEmpty() ?
123 wxString( FMT_CLIPBOARD ) : aSource );
124 PushReader( stringReader );
125 init();
126}
127
128
130{
131 if( iOwnReaders )
132 {
133 // delete the LINE_READERs from the stack, since I own them.
134 for( READER_STACK::iterator it = readerStack.begin(); it!=readerStack.end(); ++it )
135 delete *it;
136 }
137}
138
139
141{
142 specctraMode = aMode;
143 if( aMode )
144 {
145 // specctra mode defaults, some of which can still be changed in this mode.
147 }
148 else
149 {
151 stringDelimiter = '"';
152 }
153}
154
155
157{
160 commentsAreTokens = false;
161
162 curOffset = 0;
163}
164
165
167{
168 // Synchronize the pointers handling the data read by the LINE_READER
169 // only if aLexer shares the same LINE_READER, because only in this case
170 // the char buffer is be common
171
172 if( reader != aLexer.reader )
173 return false;
174
175 // We can synchronize the pointers which handle the data currently read
176 start = aLexer.start;
177 next = aLexer.next;
178 limit = aLexer.limit;
179
180 // Sync these parameters is not mandatory, but could help
181 // for instance in debug
182 curText = aLexer.curText;
183 curOffset = aLexer.curOffset;
184
185 return true;
186}
187
188
190{
191 readerStack.push_back( aLineReader );
192 reader = aLineReader;
193 start = (const char*) (*reader);
194
195 // force a new readLine() as first thing.
196 limit = start;
197 next = start;
198}
199
200
202{
203 LINE_READER* ret = nullptr;
204
205 if( readerStack.size() )
206 {
207 ret = reader;
208 readerStack.pop_back();
209
210 if( readerStack.size() )
211 {
212 reader = readerStack.back();
213 start = reader->Line();
214
215 // force a new readLine() as first thing.
216 limit = start;
217 next = start;
218 }
219 else
220 {
221 reader = nullptr;
222 start = dummy;
223 limit = dummy;
224 }
225 }
226 return ret;
227}
228
229
230int DSNLEXER::findToken( const std::string& tok ) const
231{
232 if( keywordsLookup != nullptr )
233 {
234 KEYWORD_MAP::const_iterator it = keywordsLookup->find( tok.c_str() );
235
236 if( it != keywordsLookup->end() )
237 return it->second;
238 }
239
240 return DSN_SYMBOL; // not a keyword, some arbitrary symbol.
241}
242
243
244const char* DSNLEXER::Syntax( int aTok )
245{
246 const char* ret;
247
248 switch( aTok )
249 {
250 case DSN_NONE:
251 ret = "NONE";
252 break;
253 case DSN_STRING_QUOTE:
254 ret = "string_quote"; // a special DSN syntax token, see specctra spec.
255 break;
256 case DSN_QUOTE_DEF:
257 ret = "quoted text delimiter";
258 break;
259 case DSN_DASH:
260 ret = "-";
261 break;
262 case DSN_SYMBOL:
263 ret = "symbol";
264 break;
265 case DSN_NUMBER:
266 ret = "number";
267 break;
268 case DSN_RIGHT:
269 ret = ")";
270 break;
271 case DSN_LEFT:
272 ret = "(";
273 break;
274 case DSN_STRING:
275 ret = "quoted string";
276 break;
277 case DSN_EOF:
278 ret = "end of input";
279 break;
280 default:
281 ret = "???";
282 }
283
284 return ret;
285}
286
287
288const char* DSNLEXER::GetTokenText( int aTok ) const
289{
290 const char* ret;
291
292 if( aTok < 0 )
293 {
294 return Syntax( aTok );
295 }
296 else if( (unsigned) aTok < keywordCount )
297 {
298 ret = keywords[aTok].name;
299 }
300 else
301 ret = "token too big";
302
303 return ret;
304}
305
306
307wxString DSNLEXER::GetTokenString( int aTok ) const
308{
309 wxString ret;
310
311 ret << wxT("'") << wxString::FromUTF8( GetTokenText(aTok) ) << wxT("'");
312
313 return ret;
314}
315
316
317bool DSNLEXER::IsSymbol( int aTok )
318{
319 // This is static and not inline to reduce code space.
320
321 // if aTok is >= 0, then it is a coincidental match to a keyword.
322 return aTok == DSN_SYMBOL || aTok == DSN_STRING || aTok >= 0;
323}
324
325
326void DSNLEXER::Expecting( int aTok ) const
327{
328 wxString errText = wxString::Format(
329 _( "Expecting %s" ), GetTokenString( aTok ) );
331}
332
333
334void DSNLEXER::Expecting( const char* text ) const
335{
336 wxString errText = wxString::Format(
337 _( "Expecting '%s'" ), wxString::FromUTF8( text ) );
339}
340
341
342void DSNLEXER::Unexpected( int aTok ) const
343{
344 wxString errText = wxString::Format(
345 _( "Unexpected %s" ), GetTokenString( aTok ) );
347}
348
349
350void DSNLEXER::Duplicate( int aTok )
351{
352 wxString errText = wxString::Format(
353 _("%s is a duplicate"), GetTokenString( aTok ).GetData() );
355}
356
357
358void DSNLEXER::Unexpected( const char* text ) const
359{
360 wxString errText = wxString::Format(
361 _( "Unexpected '%s'" ), wxString::FromUTF8( text ) );
363}
364
365
367{
368 int tok = NextTok();
369 if( tok != DSN_LEFT )
371}
372
373
375{
376 int tok = NextTok();
377
378 if( tok != DSN_RIGHT )
380}
381
382
384{
385 int tok = NextTok();
386
387 if( !IsSymbol( tok ) )
389
390 return tok;
391}
392
393
395{
396 int tok = NextTok();
397
398 if( !IsSymbol( tok ) && tok!=DSN_NUMBER )
399 Expecting( "a symbol or number" );
400
401 return tok;
402}
403
404
405int DSNLEXER::NeedNUMBER( const char* aExpectation )
406{
407 int tok = NextTok();
408
409 if( tok != DSN_NUMBER )
410 {
411 wxString errText = wxString::Format( _( "need a number for '%s'" ),
412 wxString::FromUTF8( aExpectation ).GetData() );
414 }
415
416 return tok;
417}
418
419
426static bool isSpace( char cc )
427{
428 // cc is signed, so it is often negative.
429 // Treat negative as large positive to exclude rapidly.
430 if( (unsigned char) cc <= ' ' )
431 {
432 switch( (unsigned char) cc )
433 {
434 case ' ':
435 case '\n':
436 case '\r':
437 case '\t':
438 case '\0': // PCAD s-expression files have this.
439 return true;
440 }
441 }
442
443 return false;
444}
445
446
447inline bool isDigit( char cc )
448{
449 return '0' <= cc && cc <= '9';
450}
451
452
454inline bool isSep( char cc )
455{
456 return isSpace( cc ) || cc=='(' || cc==')';
457}
458
459
469static bool isNumber( const char* cp, const char* limit )
470{
471 // regex for a float: "^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?" i.e. any number,
472 // code traversal manually here:
473
474 bool sawNumber = false;
475
476 if( cp < limit && ( *cp=='-' || *cp=='+' ) )
477 ++cp;
478
479 while( cp < limit && isDigit( *cp ) )
480 {
481 ++cp;
482 sawNumber = true;
483 }
484
485 if( cp < limit && *cp == '.' )
486 {
487 ++cp;
488
489 while( cp < limit && isDigit( *cp ) )
490 {
491 ++cp;
492 sawNumber = true;
493 }
494 }
495
496 if( sawNumber )
497 {
498 if( cp < limit && ( *cp=='E' || *cp=='e' ) )
499 {
500 ++cp;
501
502 sawNumber = false; // exponent mandates at least one digit thereafter.
503
504 if( cp < limit && ( *cp=='-' || *cp=='+' ) )
505 ++cp;
506
507 while( cp < limit && isDigit( *cp ) )
508 {
509 ++cp;
510 sawNumber = true;
511 }
512 }
513 }
514
515 return sawNumber && cp==limit;
516}
517
518
520{
521 const char* cur = next;
522 const char* head = cur;
523
524 prevTok = curTok;
525
526 if( curTok == DSN_EOF )
527 goto exit;
528
529 if( cur >= limit )
530 {
531L_read:
532 // blank lines are returned as "\n" and will have a len of 1.
533 // EOF will have a len of 0 and so is detectable.
534 int len = readLine();
535
536 if( len == 0 )
537 {
538 cur = start; // after readLine(), since start can change, set cur offset to start
539 curTok = DSN_EOF;
540 goto exit;
541 }
542
543 cur = start; // after readLine() since start can change.
544
545 // skip leading whitespace
546 while( cur < limit && isSpace( *cur ) )
547 ++cur;
548
549 // If the first non-blank character is #, this line is a comment.
550 // Comments cannot follow any other token on the same line.
551 if( cur<limit && *cur=='#' )
552 {
554 {
555 // Grab the entire current line [excluding end of line char(s)] as the
556 // current token. The '#' character may not be at offset zero.
557
558 while( limit[-1] == '\n' || limit[-1] == '\r' )
559 --limit;
560
561 curText.clear();
562 curText.append( start, limit );
563
564 cur = start; // ensure a good curOffset below
566 head = limit; // do a readLine() on next call in here.
567 goto exit;
568 }
569 else
570 {
571 goto L_read;
572 }
573 }
574 }
575 else
576 {
577 // skip leading whitespace
578 while( cur < limit && isSpace( *cur ) )
579 ++cur;
580 }
581
582 if( cur >= limit )
583 goto L_read;
584
585 if( *cur == '(' )
586 {
587 curText = *cur;
589 head = cur+1;
590 goto exit;
591 }
592
593 if( *cur == ')' )
594 {
595 curText = *cur;
597 head = cur+1;
598 goto exit;
599 }
600
601 // Non-specctraMode, understands and deciphers escaped \, \r, \n, and \".
602 // Strips off leading and trailing double quotes
603 if( !specctraMode )
604 {
605 // a quoted string, will return DSN_STRING
606 if( *cur == stringDelimiter )
607 {
608 // copy the token, character by character so we can remove doubled up quotes.
609 curText.clear();
610
611 ++cur; // skip over the leading delimiter, which is always " in non-specctraMode
612
613 head = cur;
614
615 while( head<limit )
616 {
617 // ESCAPE SEQUENCES:
618 if( *head =='\\' )
619 {
620 char tbuf[8];
621 char c;
622 int i;
623
624 if( ++head >= limit )
625 break; // throw exception at L_unterminated
626
627 switch( *head++ )
628 {
629 case '"':
630 case '\\': c = head[-1]; break;
631 case 'a': c = '\x07'; break;
632 case 'b': c = '\x08'; break;
633 case 'f': c = '\x0c'; break;
634 case 'n': c = '\n'; break;
635 case 'r': c = '\r'; break;
636 case 't': c = '\x09'; break;
637 case 'v': c = '\x0b'; break;
638
639 case 'x': // 1 or 2 byte hex escape sequence
640 for( i = 0; i < 2; ++i )
641 {
642 if( !isxdigit( head[i] ) )
643 break;
644
645 tbuf[i] = head[i];
646 }
647
648 tbuf[i] = '\0';
649
650 if( i > 0 )
651 c = (char) strtoul( tbuf, nullptr, 16 );
652 else
653 c = 'x'; // a goofed hex escape sequence, interpret as 'x'
654
655 head += i;
656 break;
657
658 default: // 1-3 byte octal escape sequence
659 --head;
660
661 for( i=0; i<3; ++i )
662 {
663 if( head[i] < '0' || head[i] > '7' )
664 break;
665
666 tbuf[i] = head[i];
667 }
668
669 tbuf[i] = '\0';
670
671 if( i > 0 )
672 c = (char) strtoul( tbuf, nullptr, 8 );
673 else
674 c = '\\'; // a goofed octal escape sequence, interpret as '\'
675
676 head += i;
677 break;
678 }
679
680 curText += c;
681 }
682
683 else if( *head == '"' ) // end of the non-specctraMode DSN_STRING
684 {
686 ++head; // omit this trailing double quote
687 goto exit;
688 }
689
690 else
691 curText += *head++;
692
693 } // while
694
695 // L_unterminated:
696 wxString errtxt( _( "Un-terminated delimited string" ) );
698 cur - start + curText.length() );
699 }
700 }
701 else // is specctraMode, tests in this block should not occur in KiCad mode.
702 {
703 /* get the dash out of a <pin_reference> which is embedded for example
704 like: U2-14 or "U2"-"14"
705 This is detectable by a non-space immediately preceding the dash.
706 */
707 if( *cur == '-' && cur>start && !isSpace( cur[-1] ) )
708 {
709 curText = '-';
711 head = cur+1;
712 goto exit;
713 }
714
715 // switching the string_quote character
717 {
718 static const wxString errtxt( _("String delimiter must be a single character of "
719 "', \", or $") );
720
721 char cc = *cur;
722 switch( cc )
723 {
724 case '\'':
725 case '$':
726 case '"':
727 break;
728 default:
730 }
731
732 curText = cc;
733
734 head = cur+1;
735
736 if( head<limit && !isSep( *head ) )
737 {
739 }
740
742 goto exit;
743 }
744
745 // specctraMode DSN_STRING
746 if( *cur == stringDelimiter )
747 {
748 ++cur; // skip over the leading delimiter: ",', or $
749
750 head = cur;
751
752 while( head<limit && !isStringTerminator( *head ) )
753 ++head;
754
755 if( head >= limit )
756 {
757 wxString errtxt( _( "Un-terminated delimited string" ) );
759 }
760
761 curText.clear();
762 curText.append( cur, head );
763
764 ++head; // skip over the trailing delimiter
765
767 goto exit;
768 }
769 } // specctraMode
770
771 // non-quoted token, read it into curText.
772 curText.clear();
773
774 head = cur;
775 while( head<limit && !isSep( *head ) )
776 curText += *head++;
777
778 if( isNumber( curText.c_str(), curText.c_str() + curText.size() ) )
779 {
781 goto exit;
782 }
783
784 if( specctraMode && curText == "string_quote" )
785 {
787 goto exit;
788 }
789
791
792exit: // single point of exit, no returns elsewhere please.
793
794 curOffset = cur - start;
795
796 next = head;
797
798 return curTok;
799}
800
801
803{
804 wxArrayString* ret = nullptr;
805 bool cmt_setting = SetCommentsAreTokens( true );
806 int tok = NextTok();
807
808 if( tok == DSN_COMMENT )
809 {
810 ret = new wxArrayString();
811
812 do
813 {
814 ret->Add( FromUTF8() );
815 }
816 while( ( tok = NextTok() ) == DSN_COMMENT );
817 }
818
819 SetCommentsAreTokens( cmt_setting );
820
821 return ret;
822}
823
824
826{
827#if ( defined( __GNUC__ ) && __GNUC__ < 11 ) || ( defined( __clang__ ) && __clang_major__ < 13 )
828 // GCC older than 11 "supports" C++17 without supporting the C++17 std::from_chars for doubles
829 // clang is similar
830
831 char* tmp;
832
833 errno = 0;
834
835 double fval = strtod( CurText(), &tmp );
836
837 if( errno )
838 {
839 wxString error;
840 error.Printf( _( "Invalid floating point number in\nfile: '%s'\nline: %d\noffset: %d" ),
842
843 THROW_IO_ERROR( error );
844 }
845
846 if( CurText() == tmp )
847 {
848 wxString error;
849 error.Printf( _( "Missing floating point number in\nfile: '%s'\nline: %d\noffset: %d" ),
851
852 THROW_IO_ERROR( error );
853 }
854
855 return fval;
856#else
857 // Use std::from_chars which is designed to be locale independent and performance oriented for data interchange
858
859 const std::string& str = CurStr();
860
861 // Offset any leading whitespace, this is one thing from_chars does not handle
862 size_t woff = 0;
863 while( std::isspace( str[woff] ) && woff < str.length() )
864 {
865 woff++;
866 }
867
868 double dval{};
869 std::from_chars_result res =
870 std::from_chars( str.data() + woff, str.data() + str.size(), dval );
871
872 if( res.ec != std::errc() )
873 {
874 THROW_PARSE_ERROR( _( "Invalid floating point number" ), CurSource(), CurLine(),
876 }
877
878 return dval;
879#endif
880}
Implement a lexical analyzer for the SPECCTRA DSN file format.
Definition: dsnlexer.h:79
int NeedSYMBOLorNUMBER()
Call NextTok() and then verifies that the token read in satisfies bool IsSymbol() or the next token i...
Definition: dsnlexer.cpp:394
virtual ~DSNLEXER()
Definition: dsnlexer.cpp:129
void NeedLEFT()
Call NextTok() and then verifies that the token read in is a DSN_LEFT.
Definition: dsnlexer.cpp:366
const char * next
Definition: dsnlexer.h:524
std::string curText
the text of the current token
Definition: dsnlexer.h:550
unsigned keywordCount
count of keywords table
Definition: dsnlexer.h:553
int NeedNUMBER(const char *aExpectation)
Call NextTok() and then verifies that the token read is type DSN_NUMBER.
Definition: dsnlexer.cpp:405
bool commentsAreTokens
true if should return comments as tokens
Definition: dsnlexer.h:544
int findToken(const std::string &aToken) const
Take aToken string and looks up the string in the keywords table.
Definition: dsnlexer.cpp:230
const char * CurText() const
Return a pointer to the current token's text.
Definition: dsnlexer.h:398
static bool IsSymbol(int aTok)
Test a token to see if it is a symbol.
Definition: dsnlexer.cpp:317
void init()
Definition: dsnlexer.cpp:40
const char * start
Definition: dsnlexer.h:523
const char * GetTokenText(int aTok) const
Return the C string representation of a DSN_T value.
Definition: dsnlexer.cpp:288
wxArrayString * ReadCommentLines()
Check the next sequence of tokens and reads them into a wxArrayString if they are comments.
Definition: dsnlexer.cpp:802
int curTok
the current token obtained on last NextTok()
Definition: dsnlexer.h:549
bool space_in_quoted_tokens
blank spaces within quoted strings
Definition: dsnlexer.h:542
bool specctraMode
if true, then: 1) stringDelimiter can be changed 2) Kicad quoting protocol is not in effect 3) space_...
Definition: dsnlexer.h:535
int NextTok()
Return the next token found in the input file or DSN_EOF when reaching the end of file.
Definition: dsnlexer.cpp:519
int NeedSYMBOL()
Call NextTok() and then verifies that the token read in satisfies IsSymbol().
Definition: dsnlexer.cpp:383
int curOffset
offset within current line of the current token
Definition: dsnlexer.h:547
char stringDelimiter
Definition: dsnlexer.h:541
LINE_READER * reader
Definition: dsnlexer.h:533
const char * limit
Definition: dsnlexer.h:525
bool isStringTerminator(char cc) const
Definition: dsnlexer.h:490
const KEYWORD * keywords
table sorted by CMake for bsearch()
Definition: dsnlexer.h:552
void NeedRIGHT()
Call NextTok() and then verifies that the token read in is a DSN_RIGHT.
Definition: dsnlexer.cpp:374
int readLine()
Definition: dsnlexer.h:461
READER_STACK readerStack
all the LINE_READERs by pointer.
Definition: dsnlexer.h:530
char dummy[1]
when there is no reader.
Definition: dsnlexer.h:526
double parseDouble()
Parse the current token as an ASCII numeric string with possible leading whitespace into a double pre...
Definition: dsnlexer.cpp:825
void SetSpecctraMode(bool aMode)
Change the behavior of this lexer into or out of "specctra mode".
Definition: dsnlexer.cpp:140
wxString FromUTF8() const
Return the current token text as a wxString, assuming that the input byte stream is UTF8 encoded.
Definition: dsnlexer.h:415
void Expecting(int aTok) const
Throw an IO_ERROR exception with an input file specific error message.
Definition: dsnlexer.cpp:326
const wxString & CurSource() const
Return the current LINE_READER source.
Definition: dsnlexer.h:441
LINE_READER * PopReader()
Delete the top most LINE_READER from an internal stack of LINE_READERs and in the case of FILE_LINE_R...
Definition: dsnlexer.cpp:201
void Duplicate(int aTok)
Throw an IO_ERROR exception with a message saying specifically that aTok is a duplicate of one alread...
Definition: dsnlexer.cpp:350
const std::string & CurStr() const
Return a reference to current token in std::string form.
Definition: dsnlexer.h:406
const char * CurLine() const
Return the current line of text from which the CurText() would return its token.
Definition: dsnlexer.h:431
bool SetCommentsAreTokens(bool val)
Change the handling of comments.
Definition: dsnlexer.h:296
wxString GetTokenString(int aTok) const
Return a quote wrapped wxString representation of a token value.
Definition: dsnlexer.cpp:307
int CurOffset() const
Return the byte offset within the current line, using a 1 based index.
Definition: dsnlexer.h:451
void InitParserState()
Reinit variables used during parsing, to ensure od states are not used in a new parsing must be calle...
Definition: dsnlexer.cpp:156
int prevTok
curTok from previous NextTok() call.
Definition: dsnlexer.h:546
const KEYWORD_MAP * keywordsLookup
fast, specialized "C string" hashtable
Definition: dsnlexer.h:554
static const char * Syntax(int aTok)
Definition: dsnlexer.cpp:244
DSNLEXER(const KEYWORD *aKeywordTable, unsigned aKeywordCount, const KEYWORD_MAP *aKeywordMap, FILE *aFile, const wxString &aFileName)
Initialize a DSN lexer and prepares to read from aFile which is already open and has aFilename.
Definition: dsnlexer.cpp:55
bool SyncLineReaderWith(DSNLEXER &aLexer)
Usable only for DSN lexers which share the same LINE_READER.
Definition: dsnlexer.cpp:166
void Unexpected(int aTok) const
Throw an IO_ERROR exception with an input file specific error message.
Definition: dsnlexer.cpp:342
int CurLineNumber() const
Return the current line number within my LINE_READER.
Definition: dsnlexer.h:423
bool iOwnReaders
on readerStack, should I delete them?
Definition: dsnlexer.h:522
void PushReader(LINE_READER *aLineReader)
Manage a stack of LINE_READERs in order to handle nested file inclusion.
Definition: dsnlexer.cpp:189
A LINE_READER that reads from an open file.
Definition: richio.h:173
An abstract class from which implementation specific LINE_READERs may be derived to read single lines...
Definition: richio.h:81
char * Line() const
Return a pointer to the last line that was read in.
Definition: richio.h:117
Is a LINE_READER that reads from a multiline 8 bit wide std::string.
Definition: richio.h:241
static bool isNumber(const char *cp, const char *limit)
Return true if the next sequence of text is a number: either an integer, fixed point,...
Definition: dsnlexer.cpp:469
bool isDigit(char cc)
Definition: dsnlexer.cpp:447
static bool isSpace(char cc)
Test for whitespace.
Definition: dsnlexer.cpp:426
static const KEYWORD empty_keywords[1]
Definition: dsnlexer.cpp:110
#define FMT_CLIPBOARD
Definition: dsnlexer.cpp:35
bool isSep(char cc)
Definition: dsnlexer.cpp:454
@ DSN_QUOTE_DEF
Definition: dsnlexer.h:62
@ DSN_STRING_QUOTE
Definition: dsnlexer.h:61
@ DSN_LEFT
Definition: dsnlexer.h:67
@ DSN_RIGHT
Definition: dsnlexer.h:66
@ DSN_NUMBER
Definition: dsnlexer.h:65
@ DSN_NONE
Definition: dsnlexer.h:59
@ DSN_DASH
Definition: dsnlexer.h:63
@ DSN_SYMBOL
Definition: dsnlexer.h:64
@ DSN_COMMENT
Definition: dsnlexer.h:60
@ DSN_STRING
Definition: dsnlexer.h:68
@ DSN_EOF
Definition: dsnlexer.h:69
#define _(s)
std::unordered_map< const char *, int, fnv_1a, iequal_to > KEYWORD_MAP
A hashtable made of a const char* and an int.
Definition: hashtables.h:95
#define THROW_IO_ERROR(msg)
Definition: ki_exception.h:38
#define THROW_PARSE_ERROR(aProblem, aSource, aInputLine, aLineNumber, aByteIndex)
Definition: ki_exception.h:164
void Format(OUTPUTFORMATTER *out, int aNestLevel, int aCtl, const CPTREE &aTree)
Output a PTREE into s-expression format via an OUTPUTFORMATTER derivative.
Definition: ptree.cpp:200
CITER next(CITER it)
Definition: ptree.cpp:126
Hold a keyword string and its unique integer token.
Definition: dsnlexer.h:40
const char * name
unique keyword.
Definition: dsnlexer.h:41
VECTOR3I res