http_lib_connection.cpp

Go to the documentation of this file.

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright (C) 2023 Andre F. K. Iwers <[email protected]>
 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
#include <wx/log.h>
#include <fmt/format.h>
#include <wx/translation.h>
#include <ctime>
 
#include <boost/algorithm/string.hpp>
#include <json_common.h>
#include <wx/base64.h>
 
#include <kicad_curl/kicad_curl_easy.h>
#include <curl/curl.h>
 
#include <http_lib/http_lib_connection.h>
#include <lib_id.h>
 
const char* const traceHTTPLib = "KICAD_HTTP_LIB";
 
 
static HTTP_LIB_CONNECTION::RETRIEVER makeDefaultRetriever( const HTTP_LIB_SOURCE& aSource )
{
    auto curl = std::make_shared<KICAD_CURL_EASY>();
    curl->SetHeader( "Accept", "application/json" );
    curl->SetHeader( "Authorization", "Token " + aSource.token );
    curl->SetFollowRedirects( true );
 
    return  [curl]( const std::string& aUrl, int& aStatusCode, std::string& aBody, std::string& aError )
            {
                if( !curl->SetURL( aUrl ) )
                {
                    aError = "Unable to set request URL.";
                    return false;
                }
 
                if( const int result = curl->Perform(); result != CURLE_OK )
                {
                    aError = curl->GetErrorText( result );
                    return false;
                }
 
                aStatusCode = curl->GetResponseStatusCode();
                aBody = curl->GetBuffer();
                return true;
            };
}
 
 
HTTP_LIB_CONNECTION::HTTP_LIB_CONNECTION( const HTTP_LIB_SOURCE& aSource, bool aTestConnectionNow ) :
        HTTP_LIB_CONNECTION( aSource, aTestConnectionNow, makeDefaultRetriever( aSource ) )
{
}
 
 
HTTP_LIB_CONNECTION::HTTP_LIB_CONNECTION( const HTTP_LIB_SOURCE& aSource, bool aTestConnectionNow,
                                          RETRIEVER aRetriever ) :
        m_source( aSource ),
        m_retriever( std::move( aRetriever ) )
{
    if( aTestConnectionNow )
        validateHttpLibraryEndpoints();
}
 
 
bool HTTP_LIB_CONNECTION::validateHttpLibraryEndpoints()
{
    std::lock_guard lock( m_queryMutex );
 
    m_endpointValid = false;
    std::string res = "";
 
    try
    {
        int statusCode = 0;
 
        if( !retrieve( m_source.root_url, statusCode, res ) )
            return false;
 
        if( !checkServerResponse( statusCode ) )
            return false;
 
        if( res.length() == 0 )
        {
            m_lastError += wxString::Format( _( "KiCad received an empty response!" ) + "\n" );
        }
        else
        {
            nlohmann::json response = nlohmann::json::parse( res );
 
            // Check that the endpoints exist, if not fail.
            if( !response.at( "categories" ).empty() && !response.at( "parts" ).empty() )
                m_endpointValid = true;
        }
    }
    catch( const std::exception& e )
    {
        m_lastError += wxString::Format( _( "Error: %s" ) + "\n" + _( "API Response:  %s" ) + "\n",
                                         e.what(), res );
 
        wxLogTrace( traceHTTPLib, wxT( "validateHttpLibraryEndpoints: Exception while testing API connection: %s" ),
                    m_lastError );
 
        m_endpointValid = false;
    }
 
    if( m_endpointValid )
        syncCategories();
 
    return m_endpointValid;
}
 
 
bool HTTP_LIB_CONNECTION::syncCategories()
{
    // Caller should already hold m_queryMutex.
    if( !IsValidEndpoint() )
    {
        wxLogTrace( traceHTTPLib, wxT( "syncCategories: without valid connection!" ) );
        return false;
    }
 
    std::string res = "";
 
    try
    {
        int statusCode = 0;
 
        if( !retrieve( m_source.root_url + "categories.json", statusCode, res ) )
            return false;
 
        if( !checkServerResponse( statusCode ) )
            return false;
 
        nlohmann::json response = nlohmann::json::parse( res );
 
        // collect the categories in vector
        for( const auto& item : response.items() )
        {
            HTTP_LIB_CATEGORY category;
 
            auto& value = item.value();
            category.id = value["id"].get<std::string>();
            category.name = value["name"].get<std::string>();
 
            if( value.contains( "description" ) )
            {
                category.description = value["description"].get<std::string>();
                m_categoryDescriptions[category.name] = category.description;
            }
 
            m_categories.push_back( category );
        }
    }
    catch( const std::exception& e )
    {
        m_lastError += wxString::Format( _( "Error: %s" ) + "\n" + _( "API Response:  %s" ) + "\n",
                                         e.what(), res );
 
        wxLogTrace( traceHTTPLib, wxT( "syncCategories: Exception while syncing categories: %s" ), m_lastError );
 
        m_categories.clear();
 
        return false;
    }
 
    return true;
}
 
 
bool boolFromString( const std::any& aVal, bool aDefaultValue )
{
    try
    {
        wxString strval( std::any_cast<std::string>( aVal ).c_str(), wxConvUTF8 );
 
        if( strval.IsEmpty() )
            return aDefaultValue;
 
        strval.MakeLower();
 
        for( const auto& trueVal : { wxS( "true" ), wxS( "yes" ), wxS( "y" ), wxS( "1" ) } )
        {
            if( strval.Matches( trueVal ) )
                return true;
        }
 
        for( const auto& falseVal : { wxS( "false" ), wxS( "no" ), wxS( "n" ), wxS( "0" ) } )
        {
            if( strval.Matches( falseVal ) )
                return false;
        }
    }
    catch( const std::bad_any_cast& )
    {
    }
 
    return aDefaultValue;
}
 
 
void setPartIdNameAndMetadata( const nlohmann::json& aPart_json, HTTP_LIB_PART& aPart )
{
    // the id used to identify the part, the name is needed to show a human-readable
    // part description to the user inside the symbol chooser dialog
    aPart.id = aPart_json.at( "id" );
 
    // API might not want to return an optional name.
    if( aPart_json.contains( "name" ) )
        aPart.name = aPart_json.at( "name" );
    else
        aPart.name = aPart.id;
 
    aPart.name = LIB_ID::FixIllegalChars( aPart.name, false ).c_str();
 
    if( aPart_json.contains( "description" ) )
        aPart.desc = aPart_json.at( "description" );
 
    if( aPart_json.contains( "keywords" ) )
        aPart.keywords = aPart_json.at( "keywords" );
 
    if( aPart_json.contains( "footprint_filters" ) )
    {
        nlohmann::json filters_json = aPart_json.at( "footprint_filters" );
 
        if( filters_json.is_array() )
        {
            for( const auto& val : filters_json )
                aPart.fp_filters.push_back( val );
        }
        else
        {
            aPart.fp_filters.push_back( filters_json );
        }
    }
}
 
 
// The API is loosely specified and some servers return the exclusion flags and field
// visibility as native JSON booleans rather than strings, so accept either form.
static bool jsonBoolField( const nlohmann::json& aValue, bool aDefault )
{
    if( aValue.is_boolean() )
        return aValue.get<bool>();
 
    if( aValue.is_string() )
        return boolFromString( aValue.get<std::string>(), aDefault );
 
    return aDefault;
}
 
 
bool setPartExtendedData( const nlohmann::json& aPartJson, HTTP_LIB_PART& aPart )
{
    if( aPartJson.contains( "symbolIdStr" ) && aPartJson.at( "symbolIdStr" ).is_string() )
        aPart.symbolIdStr = aPartJson.at( "symbolIdStr" ).get<std::string>();
 
    if( aPartJson.contains( "exclude_from_bom" ) )
        aPart.exclude_from_bom = jsonBoolField( aPartJson.at( "exclude_from_bom" ), false );
 
    if( aPartJson.contains( "exclude_from_board" ) )
        aPart.exclude_from_board = jsonBoolField( aPartJson.at( "exclude_from_board" ), false );
 
    if( aPartJson.contains( "exclude_from_sim" ) )
        aPart.exclude_from_sim = jsonBoolField( aPartJson.at( "exclude_from_sim" ), false );
 
    if( !aPartJson.contains( "fields" ) || !aPartJson.at( "fields" ).is_object() )
        return false;
 
    aPart.fields.clear();
 
    for( const auto& field : aPartJson.at( "fields" ).items() )
    {
        const nlohmann::json& properties = field.value();
 
        if( !properties.is_object() || !properties.contains( "value" )
            || !properties.at( "value" ).is_string() )
        {
            continue;
        }
 
        std::string value = properties.at( "value" ).get<std::string>();
        bool        visible = true;
 
        if( properties.contains( "visible" ) )
            visible = jsonBoolField( properties.at( "visible" ), true );
 
        aPart.fields.emplace_back( field.key(), std::make_tuple( value, visible ) );
    }
 
    return true;
}
 
 
bool HTTP_LIB_CONNECTION::SelectOne( const std::string& aPartID, HTTP_LIB_PART& aFetchedPart )
{
    std::lock_guard lock( m_queryMutex );
 
    if( !IsValidEndpoint() )
    {
        wxLogTrace( traceHTTPLib, wxT( "SelectOne: without valid connection!" ) );
        return false;
    }
 
    // Check if there is already a part in our cache, if not fetch it
    if( m_cachedParts.find( aPartID ) != m_cachedParts.end() )
    {
        // check if it's outdated, if so re-fetch
        if( std::difftime( std::time( nullptr ), m_cachedParts[aPartID].lastCached ) < m_source.timeout_parts )
        {
            aFetchedPart = m_cachedParts[aPartID];
            return true;
        }
    }
 
    std::string res = "";
    std::string url = m_source.root_url + fmt::format( "parts/{}.json", aPartID );
 
    try
    {
        int statusCode = 0;
 
        if( !retrieve( url, statusCode, res ) )
            return false;
 
        if( !checkServerResponse( statusCode ) )
            return false;
 
        nlohmann::ordered_json response = nlohmann::ordered_json::parse( res );
 
        // get a timestamp for caching
        aFetchedPart.lastCached = std::time( nullptr );
 
        setPartIdNameAndMetadata( response, aFetchedPart );
        setPartExtendedData( response, aFetchedPart );
 
        // parse optional pin assignments (legacy flat form; issue #2282)
        aFetchedPart.pin_map.clear();
 
        if( response.contains( "pin_map" ) )
            aFetchedPart.pin_map = ParseLegacyPinAssignments( response["pin_map"] );
 
        // parse the spec-form named pin maps + footprint associations (issue #2282)
        aFetchedPart.named_pin_maps = ParsePinMapSet( response );
        aFetchedPart.associated_footprints = ParseAssociatedFootprints( response );
 
        // Reaching the per-part endpoint means we have the full record, even if the
        // server returned no fields for this part; otherwise it would be re-fetched forever.
        aFetchedPart.detailsLoaded = true;
    }
    catch( const std::exception& e )
    {
        m_lastError += wxString::Format( _( "Error: %s" ) + "\n" + _( "API Response: %s" ) + "\n",
                                         e.what(), res );
 
        wxLogTrace( traceHTTPLib, wxT( "SelectOne: Exception while fetching part: %s" ), m_lastError );
 
        return false;
    }
 
    m_cachedParts[aFetchedPart.id] = aFetchedPart;
 
    return true;
}
 
 
bool HTTP_LIB_CONNECTION::SelectAll( const HTTP_LIB_CATEGORY& aCategory, std::vector<HTTP_LIB_PART>& aParts )
{
    std::lock_guard lock( m_queryMutex );
 
    if( !IsValidEndpoint() )
    {
        wxLogTrace( traceHTTPLib, wxT( "SelectAll: without valid connection!" ) );
        return false;
    }
 
    std::string res = "";
    std::string url = m_source.root_url + fmt::format( "parts/category/{}.json", aCategory.id );
 
    try
    {
        int statusCode = 0;
 
        if( !retrieve( url, statusCode, res ) )
            return false;
 
        nlohmann::json response = nlohmann::json::parse( res );
 
        for( nlohmann::json& item : response )
        {
            HTTP_LIB_PART part;
 
            setPartIdNameAndMetadata( item, part );
 
            // Some servers include the full field set in the category listing; when they do,
            // the chooser can show field content without a per-part fetch.
            part.detailsLoaded = setPartExtendedData( item, part );
 
            m_cache[part.name] = std::make_tuple( part.id, aCategory.id );
 
            if( part.detailsLoaded )
            {
                part.lastCached = std::time( nullptr );
                m_cachedParts[part.id] = part;
            }
 
            aParts.emplace_back( std::move( part ) );
        }
    }
    catch( const std::exception& e )
    {
        m_lastError += wxString::Format( _( "Error: %s" ) + "\n" + _( "API Response: %s" ) + "\n",
                                         e.what(), res );
 
        wxLogTrace( traceHTTPLib, wxT( "Exception occurred while syncing parts: %s" ), m_lastError );
 
        return false;
    }
 
    return true;
}
 
 
bool HTTP_LIB_CONNECTION::retrieve( const std::string& aUrl, int& aStatusCode, std::string& aBody )
{
    std::string error;
 
    if( !m_retriever( aUrl, aStatusCode, aBody, error ) )
    {
        m_lastError += error;
        return false;
    }
 
    return true;
}
 
 
bool HTTP_LIB_CONNECTION::checkServerResponse( int aStatusCode )
{
    if( aStatusCode != 200 )
    {
        m_lastError += wxString::Format( _( "API responded with error code: %s" ) + "\n",
                                         httpErrorCodeDescription( aStatusCode ) );
        return false;
    }
 
    return true;
}
 
 
wxString HTTP_LIB_CONNECTION::httpErrorCodeDescription( uint16_t aHttpCode )
{
    auto codeDescription =
            []( uint16_t aCode ) -> wxString
            {
                switch( aCode )
                {
                case 100: return wxS( "Continue" );
                case 101: return wxS( "Switching Protocols" );
                case 102: return wxS( "Processing" );
                case 103: return wxS( "Early Hints" );
 
                case 200: return wxS( "OK" );
                case 201: return wxS( "Created" );
                case 203: return wxS( "Non-Authoritative Information" );
                case 204: return wxS( "No Content" );
                case 205: return wxS( "Reset Content" );
                case 206: return wxS( "Partial Content" );
                case 207: return wxS( "Multi-Status" );
                case 208: return wxS( "Already Reported" );
                case 226: return wxS( "IM Used" );
 
                case 300: return wxS( "Multiple Choices" );
                case 301: return wxS( "Moved Permanently" );
                case 302: return wxS( "Found" );
                case 303: return wxS( "See Other" );
                case 304: return wxS( "Not Modified" );
                case 305: return wxS( "Use Proxy (Deprecated)" );
                case 306: return wxS( "Unused" );
                case 307: return wxS( "Temporary Redirect" );
                case 308: return wxS( "Permanent Redirect" );
 
                case 400: return wxS( "Bad Request" );
                case 401: return wxS( "Unauthorized" );
                case 402: return wxS( "Payment Required (Experimental)" );
                case 403: return wxS( "Forbidden" );
                case 404: return wxS( "Not Found" );
                case 405: return wxS( "Method Not Allowed" );
                case 406: return wxS( "Not Acceptable" );
                case 407: return wxS( "Proxy Authentication Required" );
                case 408: return wxS( "Request Timeout" );
                case 409: return wxS( "Conflict" );
                case 410: return wxS( "Gone" );
                case 411: return wxS( "Length Required" );
                case 412: return wxS( "Payload Too Large" );
                case 414: return wxS( "URI Too Long" );
                case 415: return wxS( "Unsupported Media Type" );
                case 416: return wxS( "Range Not Satisfiable" );
                case 417: return wxS( "Expectation Failed" );
                case 418: return wxS( "I'm a teapot" );
                case 421: return wxS( "Misdirected Request" );
                case 422: return wxS( "Unprocessable Content" );
                case 423: return wxS( "Locked" );
                case 424: return wxS( "Failed Dependency" );
                case 425: return wxS( "Too Early (Experimental)" );
                case 426: return wxS( "Upgrade Required" );
                case 428: return wxS( "Precondition Required" );
                case 429: return wxS( "Too Many Requests" );
                case 431: return wxS( "Request Header Fields Too Large" );
                case 451: return wxS( "Unavailable For Legal Reasons" );
 
                case 500: return wxS( "Internal Server Error" );
                case 501: return wxS( "Not Implemented" );
                case 502: return wxS( "Bad Gateway" );
                case 503: return wxS( "Service Unavailable" );
                case 504: return wxS( "Gateway Timeout" );
                case 505: return wxS( "HTTP Version Not Supported" );
                case 506: return wxS( "Variant Also Negotiates" );
                case 507: return wxS( "Insufficient Storage" );
                case 508: return wxS( "Loop Detected" );
                case 510: return wxS( "Not Extended" );
                case 511: return wxS( "Network Authentication Required" );
                default:  return wxS( "Unknown" );
                }
            };
 
    return wxString::Format( wxS( "%d: %s" ), aHttpCode, codeDescription( aHttpCode ) );
}