dsnlexer.cpp

Go to the documentation of this file.

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright (C) 2007-2013 SoftPLC Corporation, Dick Hollenbeck <[email protected]>
 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
#include <fast_float/fast_float.h>
#include <cstdarg>
#include <cstdio>
#include <cstdlib>         // bsearch()
#include <cctype>
 
#include <dsnlexer.h>
#include <wx/translation.h>
 
#define FMT_CLIPBOARD       _( "clipboard" )
 
 
//-----<DSNLEXER>-------------------------------------------------------------
 
void DSNLEXER::init()
{
    curTok  = DSN_NONE;
    prevTok = DSN_NONE;
 
    stringDelimiter = '"';
 
    specctraMode = false;
    space_in_quoted_tokens = false;
    commentsAreTokens = false;
    SetKnowsBar( true );    // default since version 20240706
    curOffset = 0;
}
 
 
DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount, const KEYWORD_MAP* aKeywordMap,
                    FILE* aFile, const wxString& aFilename ) :
    iOwnReaders( true ),
    start( nullptr ),
    next( nullptr ),
    limit( nullptr ),
    reader( nullptr ),
    specctraMode( false ),
    m_knowsBar( false ),
    space_in_quoted_tokens( false ),
    commentsAreTokens( false ),
    keywords( aKeywordTable ),
    keywordCount( aKeywordCount ),
    keywordsLookup( aKeywordMap )
{
    PushReader( new FILE_LINE_READER( aFile, aFilename ) );
    init();
}
 
 
DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount, const KEYWORD_MAP* aKeywordMap,
                    const std::string& aClipboardTxt, const wxString& aSource ) :
        iOwnReaders( true ),
        start( nullptr ),
        next( nullptr ),
        limit( nullptr ),
        reader( nullptr ),
        specctraMode( false ),
        m_knowsBar( false ),
        space_in_quoted_tokens( false ),
        commentsAreTokens( false ),
        keywords( aKeywordTable ),
        keywordCount( aKeywordCount ),
        keywordsLookup( aKeywordMap )
{
    PushReader( new STRING_LINE_READER( aClipboardTxt, aSource.IsEmpty() ? wxString( FMT_CLIPBOARD )
                                                                         : aSource ) );
    init();
}
 
 
DSNLEXER::DSNLEXER( const KEYWORD* aKeywordTable, unsigned aKeywordCount, const KEYWORD_MAP* aKeywordMap,
                    LINE_READER* aLineReader ) :
        iOwnReaders( false ),
        start( nullptr ),
        next( nullptr ),
        limit( nullptr ),
        reader( nullptr ),
        specctraMode( false ),
        m_knowsBar( false ),
        space_in_quoted_tokens( false ),
        commentsAreTokens( false ),
        keywords( aKeywordTable ),
        keywordCount( aKeywordCount ),
        keywordsLookup( aKeywordMap )
{
    if( aLineReader )
        PushReader( aLineReader );
 
    init();
}
 
 
static const KEYWORD empty_keywords[1] = {};
 
DSNLEXER::DSNLEXER( const std::string& aSExpression, const wxString& aSource ) :
        iOwnReaders( true ),
        start( nullptr ),
        next( nullptr ),
        limit( nullptr ),
        reader( nullptr ),
        specctraMode( false ),
        m_knowsBar( false ),
        space_in_quoted_tokens( false ),
        commentsAreTokens( false ),
        keywords( empty_keywords ),
        keywordCount( 0 ),
        keywordsLookup( nullptr )
{
    PushReader( new STRING_LINE_READER( aSExpression, aSource.IsEmpty() ? wxString( FMT_CLIPBOARD )
                                                                        : aSource ) );
    init();
}
 
 
DSNLEXER::~DSNLEXER()
{
    if( iOwnReaders )
    {
        // delete the LINE_READERs from the stack, since I own them.
        for( READER_STACK::iterator it = readerStack.begin(); it!=readerStack.end();  ++it )
            delete *it;
    }
}
 
 
void DSNLEXER::SetSpecctraMode( bool aMode )
{
    specctraMode = aMode;
 
    if( aMode )
    {
        // specctra mode defaults, some of which can still be changed in this mode.
        space_in_quoted_tokens = true;
    }
    else
    {
        space_in_quoted_tokens = false;
        stringDelimiter = '"';
    }
}
 
 
void DSNLEXER::InitParserState()
{
    curTok  = DSN_NONE;
    prevTok = DSN_NONE;
    commentsAreTokens = false;
 
    curOffset = 0;
}
 
 
bool DSNLEXER::SyncLineReaderWith( DSNLEXER& aLexer )
{
    // Synchronize the pointers handling the data read by the LINE_READER
    // only if aLexer shares the same LINE_READER, because only in this case
    // the char buffer is be common
 
    if( reader != aLexer.reader )
        return false;
 
    // We can synchronize the pointers which handle the data currently read
    start = aLexer.start;
    next = aLexer.next;
    limit = aLexer.limit;
 
    // Sync these parameters is not mandatory, but could help
    // for instance in debug
    curText = aLexer.curText;
    curOffset = aLexer.curOffset;
 
    return true;
}
 
 
void DSNLEXER::PushReader( LINE_READER* aLineReader )
{
    readerStack.push_back( aLineReader );
    reader = aLineReader;
    start  = (const char*) (*reader);
 
    // force a new readLine() as first thing.
    limit = start;
    next  = start;
}
 
 
LINE_READER* DSNLEXER::PopReader()
{
    LINE_READER* ret = nullptr;
 
    if( readerStack.size() )
    {
        ret = reader;
        readerStack.pop_back();
 
        if( readerStack.size() )
        {
            reader = readerStack.back();
            start  = reader->Line();
 
            // force a new readLine() as first thing.
            limit = start;
            next  = start;
        }
        else
        {
            reader = nullptr;
            start  = dummy;
            limit  = dummy;
        }
    }
    return ret;
}
 
 
int DSNLEXER::findToken( const std::string& tok ) const
{
    if( keywordsLookup != nullptr )
    {
        KEYWORD_MAP::const_iterator it = keywordsLookup->find( tok.c_str() );
 
        if( it != keywordsLookup->end() )
            return it->second;
 
        // The Specctra DSN grammar is case-insensitive and the keyword table is lowercase
        if( specctraMode )
        {
            std::string lower;
 
            lower.reserve( tok.size() );
 
            for( char cc : tok )
                lower += std::tolower( cc );
 
            it = keywordsLookup->find( lower.c_str() );
 
            if( it != keywordsLookup->end() )
                return it->second;
        }
    }
 
    return DSN_SYMBOL;      // not a keyword, some arbitrary symbol.
}
 
 
const char* DSNLEXER::Syntax( int aTok )
{
    const char* ret;
 
    switch( aTok )
    {
    case DSN_NONE:
        ret = "NONE";
        break;
    case DSN_STRING_QUOTE:
        ret = "string_quote";   // a special DSN syntax token, see specctra spec.
        break;
    case DSN_QUOTE_DEF:
        ret = "quoted text delimiter";
        break;
    case DSN_DASH:
        ret = "-";
        break;
    case DSN_SYMBOL:
        ret = "symbol";
        break;
    case DSN_NUMBER:
        ret = "number";
        break;
    case DSN_RIGHT:
        ret = ")";
        break;
    case DSN_LEFT:
        ret = "(";
        break;
    case DSN_STRING:
        ret = "quoted string";
        break;
    case DSN_EOF:
        ret = "end of input";
        break;
    case DSN_BAR:
        ret = "|";
        break;
    default:
        ret = "???";
    }
 
    return ret;
}
 
 
const char* DSNLEXER::GetTokenText( int aTok ) const
{
    const char* ret;
 
    if( aTok < 0 )
        return Syntax( aTok );
    else if( (unsigned) aTok < keywordCount )
        ret = keywords[aTok].name;
    else
        ret = "token too big";
 
    return ret;
}
 
 
wxString DSNLEXER::GetTokenString( int aTok ) const
{
    wxString ret;
 
    ret << wxT("'") << wxString::FromUTF8( GetTokenText(aTok) ) << wxT("'");
 
    return ret;
}
 
 
bool DSNLEXER::IsSymbol( int aTok )
{
    // This is static and not inline to reduce code space.
 
    // if aTok is >= 0, then it is a coincidental match to a keyword.
    return aTok == DSN_SYMBOL || aTok == DSN_STRING || aTok >= 0;
}
 
 
bool DSNLEXER::IsNumber( int aTok )
{
    return aTok == DSN_NUMBER;
}
 
 
void DSNLEXER::Expecting( int aTok ) const
{
    wxString errText = wxString::Format( _( "Expecting '%s'" ), GetTokenString( aTok ) );
    THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
}
 
 
void DSNLEXER::Expecting( const char* text ) const
{
    wxString errText =
            wxString::Format( _( "Expecting %s. Got '%s'" ), wxString::FromUTF8( text ), GetTokenString( CurTok() ) );
    THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
}
 
 
void DSNLEXER::Unexpected( int aTok ) const
{
    wxString errText = wxString::Format( _( "Unexpected '%s'" ), GetTokenString( aTok ) );
    THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
}
 
 
void DSNLEXER::Duplicate( int aTok )
{
    wxString errText = wxString::Format( _( "%s is a duplicate" ), GetTokenString( aTok ).GetData() );
    THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
}
 
 
void DSNLEXER::Unexpected( const char* text ) const
{
    wxString errText = wxString::Format( _( "Unexpected %s" ), wxString::FromUTF8( text ) );
    THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
}
 
 
void DSNLEXER::NeedLEFT()
{
    int tok = NextTok();
 
    if( tok != DSN_LEFT )
        Expecting( DSN_LEFT );
}
 
 
void DSNLEXER::NeedRIGHT()
{
    int tok = NextTok();
 
    if( tok != DSN_RIGHT )
        Expecting( DSN_RIGHT );
}
 
 
void DSNLEXER::NeedBAR()
{
    int tok = NextTok();
 
    if( tok != DSN_BAR )
        Expecting( DSN_BAR );
}
 
 
int DSNLEXER::NeedSYMBOL()
{
    int tok = NextTok();
 
    if( !IsSymbol( tok ) )
        Expecting( DSN_SYMBOL );
 
    return tok;
}
 
 
int DSNLEXER::NeedSYMBOLorNUMBER()
{
    int  tok = NextTok();
 
    if( !IsSymbol( tok ) && !IsNumber( tok ) )
        Expecting( "a symbol or number" );
 
    return tok;
}
 
 
int DSNLEXER::NeedNUMBER( const char* aExpectation )
{
    int tok = NextTok();
 
    if( !IsNumber( tok ) )
    {
        wxString errText = wxString::Format( _( "need a number for '%s'" ),
                                             wxString::FromUTF8( aExpectation ).GetData() );
        THROW_PARSE_ERROR( errText, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
    }
 
    return tok;
}
 

static bool isSpace( char cc )
{
    // cc is signed, so it is often negative.
    // Treat negative as large positive to exclude rapidly.
    if( (unsigned char) cc <= ' ' )
    {
        switch( (unsigned char) cc )
        {
        case ' ':
        case '\n':
        case '\r':
        case '\t':
        case '\0':              // PCAD s-expression files have this.
            return true;
        }
    }
 
    return false;
}
 
 
inline bool isDigit( char cc )
{
    return '0' <= cc && cc <= '9';
}
 

inline bool DSNLEXER::isSep( char cc )
{
    return isSpace( cc ) || cc == '(' || cc == ')' || ( m_knowsBar && cc == '|' );
}
 
 
bool DSNLEXER::isNumber( const char* cp, const char* limit )
{
    // regex for a float: "^[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?" i.e. any number,
    // code traversal manually here:
 
    bool sawNumber = false;
 
    if( cp < limit && ( *cp=='-' || *cp=='+' ) )
        ++cp;
 
    while( cp < limit && isDigit( *cp ) )
    {
        ++cp;
        sawNumber = true;
    }
 
    if( cp < limit && *cp == '.' )
    {
        ++cp;
 
        while( cp < limit && isDigit( *cp ) )
        {
            ++cp;
            sawNumber = true;
        }
    }
 
    if( sawNumber )
    {
        if( cp < limit && ( *cp=='E' || *cp=='e' ) )
        {
            ++cp;
 
            sawNumber = false;  // exponent mandates at least one digit thereafter.
 
            if( cp < limit && ( *cp=='-' || *cp=='+' )  )
                ++cp;
 
            while( cp < limit && isDigit( *cp ) )
            {
                ++cp;
                sawNumber = true;
            }
        }
    }
 
    return sawNumber && cp==limit;
}
 
 
int DSNLEXER::NextTok()
{
    const char*   cur  = next;
    const char*   head = cur;
 
    prevTok = curTok;
    curSeparator.clear();
 
    if( curTok == DSN_EOF )
        goto exit;
 
    if( cur >= limit )
    {
L_read:
        // blank lines are returned as "\n" and will have a len of 1.
        // EOF will have a len of 0 and so is detectable.
        int len = readLine();
 
        if( len == 0 )
        {
            cur = start;        // after readLine(), since start can change, set cur offset to start
            curTok = DSN_EOF;
            goto exit;
        }
 
        cur = start;    // after readLine() since start can change.
 
        // skip leading whitespace
        while( cur < limit && isSpace( *cur ) )
        {
            curSeparator += *cur;
            ++cur;
        }
 
        // If the first non-blank character is #, this line is a comment.
        // Comments cannot follow any other token on the same line.
        if( cur<limit && *cur=='#' )
        {
            if( commentsAreTokens )
            {
                // Grab the entire current line [excluding end of line char(s)] as the
                // current token.  The '#' character may not be at offset zero.
 
                while( limit[-1] == '\n' || limit[-1] == '\r' )
                    --limit;
 
                curText.clear();
                curText.append( start, limit );
 
                cur     = start;        // ensure a good curOffset below
                curTok  = DSN_COMMENT;
                head    = limit;        // do a readLine() on next call in here.
                goto exit;
            }
            else
            {
                goto L_read;
            }
        }
    }
    else
    {
        // skip leading whitespace
        while( cur < limit && isSpace( *cur ) )
        {
            curSeparator += *cur;
            ++cur;
        }
    }
 
    if( cur >= limit )
        goto L_read;
 
    if( *cur == '(' )
    {
        curText = *cur;
        curTok = DSN_LEFT;
        head = cur+1;
        goto exit;
    }
 
    if( *cur == ')' )
    {
        curText = *cur;
        curTok = DSN_RIGHT;
        head = cur+1;
        goto exit;
    }
 
    if( m_knowsBar && *cur == '|' )
    {
        curText = *cur;
        curTok = DSN_BAR;
        head = cur+1;
        goto exit;
    }
 
    // Non-specctraMode, understands and deciphers escaped \, \r, \n, and \".
    // Strips off leading and trailing double quotes
    if( !specctraMode )
    {
        // a quoted string, will return DSN_STRING
        if( *cur == stringDelimiter )
        {
            // copy the token, character by character so we can remove doubled up quotes.
            curText.clear();
 
            ++cur;  // skip over the leading delimiter, which is always " in non-specctraMode
 
            head = cur;
 
            while( head<limit )
            {
                // ESCAPE SEQUENCES:
                if( *head =='\\' )
                {
                    char    tbuf[8];
                    char    c;
                    int     i;
 
                    if( ++head >= limit )
                        break;  // throw exception at L_unterminated
 
                    switch( *head++ )
                    {
                    case '"':
                    case '\\':  c = head[-1];   break;
                    case 'a':   c = '\x07';     break;
                    case 'b':   c = '\x08';     break;
                    case 'f':   c = '\x0c';     break;
                    case 'n':   c = '\n';       break;
                    case 'r':   c = '\r';       break;
                    case 't':   c = '\x09';     break;
                    case 'v':   c = '\x0b';     break;
 
                    case 'x':   // 1 or 2 byte hex escape sequence
                        for( i = 0; i < 2; ++i )
                        {
                            if( !isxdigit( head[i] ) )
                                break;
 
                            tbuf[i] = head[i];
                        }
 
                        tbuf[i] = '\0';
 
                        if( i > 0 )
                            c = (char) strtoul( tbuf, nullptr, 16 );
                        else
                            c = 'x';   // a goofed hex escape sequence, interpret as 'x'
 
                        head += i;
                        break;
 
                    default:    // 1-3 byte octal escape sequence
                        --head;
 
                        for( i=0; i<3; ++i )
                        {
                            if( head[i] < '0' || head[i] > '7' )
                                break;
 
                            tbuf[i] = head[i];
                        }
 
                        tbuf[i] = '\0';
 
                        if( i > 0 )
                            c = (char) strtoul( tbuf, nullptr, 8 );
                        else
                            c = '\\';   // a goofed octal escape sequence, interpret as '\'
 
                        head += i;
                        break;
                    }
 
                    curText += c;
                }
 
                else if( *head == '"' )     // end of the non-specctraMode DSN_STRING
                {
                    curTok = DSN_STRING;
                    ++head;                 // omit this trailing double quote
                    goto exit;
                }
 
                else
                    curText += *head++;
 
            }   // while
 
            // L_unterminated:
            wxString errtxt( _( "Unterminated delimited string" ) );
            THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(),
                               cur - start + curText.length() );
        }
    }
    else    // is specctraMode, tests in this block should not occur in KiCad mode.
    {
        /*  get the dash out of a <pin_reference> which is embedded for example
            like:  U2-14 or "U2"-"14"
            This is detectable by a non-space immediately preceding the dash.
        */
        if( *cur == '-' && cur>start && !isSpace( cur[-1] ) )
        {
            curText = '-';
            curTok = DSN_DASH;
            head = cur+1;
            goto exit;
        }
 
        // switching the string_quote character
        if( prevTok == DSN_STRING_QUOTE )
        {
            static const wxString errtxt( _("String delimiter must be a single character of "
                                            "', \", or $") );
 
            char cc = *cur;
            switch( cc )
            {
            case '\'':
            case '$':
            case '"':
                break;
            default:
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }
 
            curText = cc;
 
            head = cur+1;
 
            if( head<limit && !isSep( *head ) )
            {
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }
 
            curTok = DSN_QUOTE_DEF;
            goto exit;
        }
 
        // specctraMode DSN_STRING
        if( *cur == stringDelimiter )
        {
            ++cur;  // skip over the leading delimiter: ",', or $
 
            head = cur;
 
            while( head<limit  &&  !isStringTerminator( *head ) )
                ++head;
 
            if( head >= limit )
            {
                wxString errtxt( _( "Un-terminated delimited string" ) );
                THROW_PARSE_ERROR( errtxt, CurSource(), CurLine(), CurLineNumber(), CurOffset() );
            }
 
            curText.clear();
            curText.append( cur, head );
 
            ++head;     // skip over the trailing delimiter
 
            curTok  = DSN_STRING;
            goto exit;
        }
    }           // specctraMode
 
    // non-quoted token, read it into curText.
    curText.clear();
 
    head = cur;
    while( head<limit && !isSep( *head ) )
        curText += *head++;
 
    if( isNumber( curText.c_str(), curText.c_str() + curText.size() ) )
    {
        curTok = DSN_NUMBER;
        goto exit;
    }
 
    if( specctraMode && curText == "string_quote" )
    {
        curTok = DSN_STRING_QUOTE;
        goto exit;
    }
 
    curTok = findToken( curText );
 
exit:   // single point of exit, no returns elsewhere please.
 
    curOffset = cur - start;
 
    next = head;
 
    return curTok;
}
 
 
wxArrayString* DSNLEXER::ReadCommentLines()
{
    wxArrayString*  ret = nullptr;
    bool            cmt_setting = SetCommentsAreTokens( true );
    int             tok = NextTok();
 
    if( tok == DSN_COMMENT )
    {
        ret = new wxArrayString();
 
        do
        {
            ret->Add( FromUTF8() );
        }
        while( ( tok = NextTok() ) == DSN_COMMENT );
    }
 
    SetCommentsAreTokens( cmt_setting );
 
    return ret;
}
 
// Skip the white space and also accept both normal numbers and scientfic notation
#define FROM_CHARS_FLAGS ( fast_float::chars_format::skip_white_space | fast_float::chars_format::general )
 
double DSNLEXER::parseDouble()
{
    // Use fast_float::from_chars which is designed to be locale independent and significantly
    // faster than strtod and std::from_chars
    const std::string& str = CurStr();
 
    double                 dval{};
    fast_float::from_chars_result res = fast_float::from_chars( str.data(), str.data() + str.size(), dval,
                                                                FROM_CHARS_FLAGS );
 
    if( res.ec != std::errc() )
    {
        THROW_PARSE_ERROR( _( "Invalid floating point number" ), CurSource(), CurLine(),
                           CurLineNumber(), CurOffset() );
    }
 
    return dval;
}