embedded_files.cpp

Go to the documentation of this file.

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright (C) 2024 KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
#include "embedded_files.h"
#include "embedded_files_parser.h"
 
#include <wx/base64.h>
#include <wx/debug.h>
#include <wx/filename.h>
#include <wx/log.h>
#include <wx/mstream.h>
#include <wx/wfstream.h>
 
#include <map>
#include <memory>
#include <sstream>
 
#include <zstd.h>
 
#include <kiid.h>
#include <mmh3_hash.h>
#include <paths.h>
 
 
 
EMBEDDED_FILES::EMBEDDED_FILE* EMBEDDED_FILES::AddFile( const wxFileName& aName, bool aOverwrite )
{
    if( HasFile( aName.GetFullName() ) )
    {
        if( !aOverwrite )
            return m_files[aName.GetFullName()];
 
        m_files.erase( aName.GetFullName() );
    }
 
    wxFFileInputStream file( aName.GetFullPath() );
    wxMemoryBuffer     buffer;
 
    if( !file.IsOk() )
        return nullptr;
 
    wxFileOffset length = file.GetLength();
 
    std::unique_ptr<EMBEDDED_FILE> efile = std::make_unique<EMBEDDED_FILE>();
    efile->name = aName.GetFullName();
    efile->decompressedData.resize( length );
 
    wxString ext = aName.GetExt().Upper();
 
    // Handle some common file extensions
    if( ext == "STP" || ext == "STPZ" || ext == "STEP" || ext == "WRL" || ext == "WRZ" )
    {
        efile->type = EMBEDDED_FILE::FILE_TYPE::MODEL;
    }
    else if( ext == "WOFF" || ext == "WOFF2" || ext == "TTF" || ext == "OTF" )
    {
        efile->type = EMBEDDED_FILE::FILE_TYPE::FONT;
    }
    else if( ext == "PDF" )
    {
        efile->type = EMBEDDED_FILE::FILE_TYPE::DATASHEET;
    }
    else if( ext == "KICAD_WKS" )
    {
        efile->type = EMBEDDED_FILE::FILE_TYPE::WORKSHEET;
    }
 
    if( !efile->decompressedData.data() )
        return nullptr;
 
    char* data = efile->decompressedData.data();
    wxFileOffset total_read = 0;
 
    while( !file.Eof() && total_read < length )
    {
        file.Read( data, length - total_read );
 
        size_t read = file.LastRead();
        data += read;
        total_read += read;
    }
 
    if( CompressAndEncode( *efile ) != RETURN_CODE::OK )
        return nullptr;
 
    efile->is_valid = true;
 
    m_files[aName.GetFullName()] = efile.release();
 
    return m_files[aName.GetFullName()];
}
 
 
void EMBEDDED_FILES::AddFile( EMBEDDED_FILE* aFile )
{
    m_files.insert( { aFile->name, aFile } );
}
 
// Remove a file from the collection
void EMBEDDED_FILES::RemoveFile( const wxString& name, bool aErase )
{
    auto it = m_files.find( name );
 
    if( it != m_files.end() )
    {
        m_files.erase( it );
 
        if( aErase )
            delete it->second;
    }
}
 
 
void EMBEDDED_FILES::ClearEmbeddedFonts()
{
    for( auto it = m_files.begin(); it != m_files.end(); )
    {
        if( it->second->type == EMBEDDED_FILE::FILE_TYPE::FONT )
        {
            delete it->second;
            it = m_files.erase( it );
        }
        else
        {
            ++it;
        }
    }
}
 
 
// Write the collection of files to a disk file in the specified format
void EMBEDDED_FILES::WriteEmbeddedFiles( OUTPUTFORMATTER& aOut, bool aWriteData ) const
{
    ssize_t MIME_BASE64_LENGTH = 76;
    aOut.Print( "(embedded_files " );
 
    for( const auto& [name, entry] : m_files )
    {
        const EMBEDDED_FILE& file = *entry;
 
        aOut.Print( "(file " );
        aOut.Print( "(name %s)", aOut.Quotew( file.name ).c_str() );
 
        const char* type = nullptr;
 
        switch( file.type )
        {
        case EMBEDDED_FILE::FILE_TYPE::DATASHEET: type = "datasheet"; break;
        case EMBEDDED_FILE::FILE_TYPE::FONT:      type = "font";      break;
        case EMBEDDED_FILE::FILE_TYPE::MODEL:     type = "model";     break;
        case EMBEDDED_FILE::FILE_TYPE::WORKSHEET: type = "worksheet"; break;
        default:                                  type = "other";     break;
        }
 
        aOut.Print( "(type %s)", type );
 
        if( aWriteData )
        {
            aOut.Print( "(data" );
 
            size_t first = 0;
 
            while( first < file.compressedEncodedData.length() )
            {
                ssize_t remaining = file.compressedEncodedData.length() - first;
                int     length = std::min( remaining, MIME_BASE64_LENGTH );
 
                std::string_view view( file.compressedEncodedData.data() + first, length );
 
                aOut.Print( "\n%1s%.*s%s\n", first ? "" : "|", length, view.data(),
                            remaining == length ? "|" : "" );
                first += MIME_BASE64_LENGTH;
            }
            aOut.Print( ")" ); // Close data
        }
 
        aOut.Print( "(checksum %s)", aOut.Quotew( file.data_hash ).c_str() );
        aOut.Print( ")" ); // Close file
    }
 
    aOut.Print( ")" ); // Close embedded_files
}
 
// Compress and Base64 encode data
EMBEDDED_FILES::RETURN_CODE EMBEDDED_FILES::CompressAndEncode( EMBEDDED_FILE& aFile )
{
    std::vector<char> compressedData;
    size_t estCompressedSize = ZSTD_compressBound( aFile.decompressedData.size() );
    compressedData.resize( estCompressedSize );
    size_t compressedSize = ZSTD_compress( compressedData.data(), estCompressedSize,
                                           aFile.decompressedData.data(),
                                           aFile.decompressedData.size(), 15 );
 
    if( ZSTD_isError( compressedSize ) )
    {
        compressedData.clear();
        return RETURN_CODE::OUT_OF_MEMORY;
    }
 
    const size_t dstLen = wxBase64EncodedSize( compressedSize );
    aFile.compressedEncodedData.resize( dstLen );
    size_t retval = wxBase64Encode( aFile.compressedEncodedData.data(), dstLen,
                                          compressedData.data(), compressedSize );
    if( retval != dstLen )
    {
        aFile.compressedEncodedData.clear();
        return RETURN_CODE::OUT_OF_MEMORY;
    }
 
    MMH3_HASH hash( EMBEDDED_FILES::Seed() );
    hash.add( aFile.decompressedData );
    aFile.data_hash = hash.digest().ToString();
 
    return RETURN_CODE::OK;
}
 
// Decompress and Base64 decode data
EMBEDDED_FILES::RETURN_CODE EMBEDDED_FILES::DecompressAndDecode( EMBEDDED_FILE& aFile )
{
    std::vector<char> compressedData;
    size_t            compressedSize = wxBase64DecodedSize( aFile.compressedEncodedData.size() );
 
    if( compressedSize == 0 )
    {
        wxLogTrace( wxT( "KICAD_EMBED" ),
                    wxT( "%s:%s:%d\n * Base64DecodedSize failed for file '%s' with size %zu" ),
                    __FILE__, __FUNCTION__, __LINE__, aFile.name, aFile.compressedEncodedData.size() );
        return RETURN_CODE::OUT_OF_MEMORY;
    }
 
    compressedData.resize( compressedSize );
    void* compressed = compressedData.data();
 
    // The return value from wxBase64Decode is the actual size of the decoded data avoiding
    // the modulo 4 padding of the base64 encoding
    compressedSize = wxBase64Decode( compressed, compressedSize, aFile.compressedEncodedData );
 
    unsigned long long estDecompressedSize = ZSTD_getFrameContentSize( compressed, compressedSize );
 
    if( estDecompressedSize > 1e9 ) // Limit to 1GB
        return RETURN_CODE::OUT_OF_MEMORY;
 
    if( estDecompressedSize == ZSTD_CONTENTSIZE_ERROR
        || estDecompressedSize == ZSTD_CONTENTSIZE_UNKNOWN )
    {
        return RETURN_CODE::OUT_OF_MEMORY;
    }
 
    aFile.decompressedData.resize( estDecompressedSize );
    void* decompressed = aFile.decompressedData.data();
 
    size_t decompressedSize = ZSTD_decompress( decompressed, estDecompressedSize,
                                               compressed, compressedSize );
 
    if( ZSTD_isError( decompressedSize ) )
    {
        wxLogTrace( wxT( "KICAD_EMBED" ),
                    wxT( "%s:%s:%d\n * ZSTD_decompress failed with error '%s'" ),
                    __FILE__, __FUNCTION__, __LINE__, ZSTD_getErrorName( decompressedSize ) );
        aFile.decompressedData.clear();
        return RETURN_CODE::OUT_OF_MEMORY;
    }
 
    aFile.decompressedData.resize( decompressedSize );
    std::string test_hash;
    std::string new_hash;
 
    MMH3_HASH hash( EMBEDDED_FILES::Seed() );
    hash.add( aFile.decompressedData );
    new_hash = hash.digest().ToString();
 
    if( aFile.data_hash.length() == 64 )
        picosha2::hash256_hex_string( aFile.decompressedData, test_hash );
    else
        test_hash = new_hash;
 
    if( test_hash != aFile.data_hash )
    {
        wxLogTrace( wxT( "KICAD_EMBED" ),
                    wxT( "%s:%s:%d\n * Checksum error in embedded file '%s'" ),
                    __FILE__, __FUNCTION__, __LINE__, aFile.name );
        aFile.decompressedData.clear();
        return RETURN_CODE::CHECKSUM_ERROR;
    }
 
    aFile.data_hash = new_hash;
 
    return RETURN_CODE::OK;
}
 
// Parsing method
void EMBEDDED_FILES_PARSER::ParseEmbedded( EMBEDDED_FILES* aFiles )
{
    if( !aFiles )
        THROW_PARSE_ERROR( "No embedded files object provided", CurSource(), CurLine(),
                           CurLineNumber(), CurOffset() );
 
    using namespace EMBEDDED_FILES_T;
 
    std::unique_ptr<EMBEDDED_FILES::EMBEDDED_FILE> file( nullptr );
 
    for( T token = NextTok(); token != T_RIGHT; token = NextTok() )
    {
        if( token != T_LEFT )
            Expecting( T_LEFT );
 
        token = NextTok();
 
        if( token != T_file )
            Expecting( "file" );
 
        if( file )
        {
            if( !file->compressedEncodedData.empty() )
            {
                EMBEDDED_FILES::DecompressAndDecode( *file );
 
                if( !file->Validate() )
                    THROW_PARSE_ERROR( "Checksum error in embedded file " + file->name, CurSource(),
                                        CurLine(), CurLineNumber(), CurOffset() );
            }
 
            aFiles->AddFile( file.release() );
        }
 
        file = std::unique_ptr<EMBEDDED_FILES::EMBEDDED_FILE>( nullptr );
 
 
        for( token = NextTok(); token != T_RIGHT; token = NextTok() )
        {
            if( token != T_LEFT )
                Expecting( T_LEFT );
 
            token = NextTok();
 
            switch( token )
            {
 
            case T_checksum:
                NeedSYMBOLorNUMBER();
 
                if( !IsSymbol( token ) )
                    Expecting( "checksum data" );
 
                file->data_hash = CurStr();
                NeedRIGHT();
                break;
 
            case T_data:
                NeedBAR();
                token = NextTok();
 
                file->compressedEncodedData.reserve( 1 << 17 );
 
                while( token != T_BAR )
                {
                    if( !IsSymbol( token ) )
                        Expecting( "base64 file data" );
 
                    file->compressedEncodedData += CurStr();
                    token = NextTok();
                }
 
                file->compressedEncodedData.shrink_to_fit();
 
                NeedRIGHT();
                break;
 
            case T_name:
 
                if( file )
                {
                    wxLogTrace( wxT( "KICAD_EMBED" ),
                                wxT( "Duplicate 'name' tag in embedded file %s" ), file->name );
                }
 
                NeedSYMBOLorNUMBER();
 
                file = std::make_unique<EMBEDDED_FILES::EMBEDDED_FILE>();
                file->name = CurStr();
                NeedRIGHT();
 
                break;
 
            case T_type:
 
                token = NextTok();
 
                switch( token )
                {
                case T_datasheet:
                    file->type = EMBEDDED_FILES::EMBEDDED_FILE::FILE_TYPE::DATASHEET;
                    break;
                case T_font:
                    file->type = EMBEDDED_FILES::EMBEDDED_FILE::FILE_TYPE::FONT;
                    break;
                case T_model:
                    file->type = EMBEDDED_FILES::EMBEDDED_FILE::FILE_TYPE::MODEL;
                    break;
                case T_worksheet:
                    file->type = EMBEDDED_FILES::EMBEDDED_FILE::FILE_TYPE::WORKSHEET;
                    break;
                case T_other:
                    file->type = EMBEDDED_FILES::EMBEDDED_FILE::FILE_TYPE::OTHER;
                    break;
                default:
                    Expecting( "datasheet, font, model, worksheet or other" );
                    break;
                }
                NeedRIGHT();
                break;
 
            default:
                Expecting( "checksum, data or name" );
            }
        }
    }
 
    // Add the last file in the collection
    if( file )
    {
        if( !file->compressedEncodedData.empty() )
        {
            if( EMBEDDED_FILES::DecompressAndDecode( *file ) == EMBEDDED_FILES::RETURN_CODE::CHECKSUM_ERROR )
                THROW_PARSE_ERROR( "Checksum error in embedded file " + file->name, CurSource(),
                                    CurLine(), CurLineNumber(), CurOffset() );
        }
 
        aFiles->AddFile( file.release() );
    }
}
 
 
wxFileName EMBEDDED_FILES::GetTemporaryFileName( const wxString& aName ) const
{
    wxFileName cacheFile;
 
    auto it = m_files.find( aName );
 
    if( it == m_files.end() )
        return cacheFile;
 
    cacheFile.AssignDir( PATHS::GetUserCachePath() );
    cacheFile.AppendDir( wxT( "embed" ) );
 
    if( !PATHS::EnsurePathExists( cacheFile.GetFullPath() ) )
    {
        wxLogTrace( wxT( "KICAD_EMBED" ),
                    wxT( "%s:%s:%d\n * failed to create embed cache directory '%s'" ),
                    __FILE__, __FUNCTION__, __LINE__, cacheFile.GetPath() );
 
        cacheFile.SetPath( wxFileName::GetTempDir() );
    }
 
    wxFileName inputName( aName );
 
    // Store the cache file name using the data hash to allow for shared data between
    // multiple projects using the same files as well as deconflicting files with the same name
    cacheFile.SetName( "kicad_embedded_" + it->second->data_hash );
    cacheFile.SetExt( inputName.GetExt() );
 
    if( cacheFile.FileExists() && cacheFile.IsFileReadable() )
        return cacheFile;
 
    wxFFileOutputStream out( cacheFile.GetFullPath() );
 
    if( !out.IsOk() )
    {
        cacheFile.Clear();
        return cacheFile;
    }
 
    out.Write( it->second->decompressedData.data(), it->second->decompressedData.size() );
 
    return cacheFile;
}
 
 
const std::vector<wxString>* EMBEDDED_FILES::GetFontFiles() const
{
    return &m_fontFiles;
}
 
 
const std::vector<wxString>* EMBEDDED_FILES::UpdateFontFiles()
{
    m_fontFiles.clear();
 
    for( const auto& [name, entry] : m_files )
    {
        if( entry->type == EMBEDDED_FILE::FILE_TYPE::FONT )
            m_fontFiles.push_back( GetTemporaryFileName( name ).GetFullPath() );
    }
 
    return &m_fontFiles;
}