43#include <wx/mstream.h>
44#include <wx/tokenzr.h>
45#include <wx/wfstream.h>
46#include <wx/zipstrm.h>
54static const wxChar
tracePcm[] = wxT(
"KICAD_PCM" );
63 void error(
const json::json_pointer& ptr,
const json& instance,
64 const std::string& message )
override
66 throw std::invalid_argument( std::string(
"At " ) + ptr.to_string() +
", value:\n"
67 + instance.dump() +
"\n" + message +
"\n" );
73 std::function<
void(
int )> aAvailableUpdateCallback ) :
75 m_availableUpdateCallback( aAvailableUpdateCallback )
82 schema_file.AppendDir( wxS(
"schemas" ) );
84 std::ifstream schema_stream( schema_file.GetFullPath().fn_str() );
85 nlohmann::json schema;
89 schema_stream >> schema;
92 catch( std::exception& e )
94 if( !schema_file.FileExists() )
96 wxLogError( wxString::Format(
_(
"schema file '%s' not found" ),
97 schema_file.GetFullPath() ) );
101 wxLogError( wxString::Format(
_(
"Error loading schema: %s" ), e.what() ) );
110 std::ifstream installed_stream( f.GetFullPath().fn_str() );
111 nlohmann::json installed;
115 installed_stream >> installed;
117 if( installed.contains(
"packages" ) && installed[
"packages"].is_array() )
119 for(
const auto& js_entry : installed[
"packages"] )
126 catch( std::exception& e )
128 wxLogError( wxString::Format(
_(
"Error loading installed packages list: %s" ),
142 wxDir package_dir( d.GetPath() );
144 if( !package_dir.IsOpened() )
148 bool more = package_dir.GetFirst( &subdir,
"", wxDIR_DIRS | wxDIR_HIDDEN );
152 wxString actual_package_id = subdir;
153 actual_package_id.Replace(
'_',
'.' );
158 wxFileName subdir_file( d.GetPath(), subdir );
162 int stat_code = wxStat( subdir_file.GetFullPath(), &stat );
182 more = package_dir.GetNext( &subdir );
189 [&]( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
191 PreparePackage( entry.second.package );
210 const size_t aSizeLimit )
212 bool size_exceeded =
false;
214 TRANSFER_CALLBACK callback = [&](
size_t dltotal,
size_t dlnow,
size_t ultotal,
size_t ulnow )
216 if( aSizeLimit > 0 && ( dltotal > aSizeLimit || dlnow > aSizeLimit ) )
218 size_exceeded =
true;
227 aReporter->
Report( wxString::Format(
_(
"Downloading %lld/%lld kB" ), dlnow / 1000,
240 curl.
SetURL( aUrl.ToUTF8().data() );
249 if( code != CURLE_OK )
253 if( code == CURLE_ABORTED_BY_CALLBACK && size_exceeded )
254 wxMessageBox(
_(
"Download is too large." ) );
255 else if( code != CURLE_ABORTED_BY_CALLBACK )
269 std::stringstream repository_stream;
271 aReporter->
SetTitle(
_(
"Fetching repository" ) );
276 wxLogError(
_(
"Unable to load repository url" ) );
281 nlohmann::json repository_json;
285 repository_stream >> repository_json;
287 ValidateJson( repository_json, nlohmann::json_uri(
"#/definitions/Repository" ) );
291 catch(
const std::exception& e )
295 wxLogError( wxString::Format(
_(
"Unable to parse repository: %s" ), e.what() ) );
296 wxLogError(
_(
"The given repository URL does not look like a valid KiCad package "
297 "repository. Please double check the URL." ) );
308 const nlohmann::json_uri& aUri )
const
316 const std::optional<wxString>& aHash,
317 std::vector<PCM_PACKAGE>& aPackages,
320 std::stringstream packages_stream;
322 aReporter->
SetTitle(
_(
"Fetching repository packages" ) );
327 wxLogError(
_(
"Unable to load repository packages url." ) );
332 std::istringstream isstream( packages_stream.str() );
334 if( aHash && !
VerifyHash( isstream, *aHash ) )
337 wxLogError(
_(
"Packages hash doesn't match. Repository may be corrupted." ) );
344 nlohmann::json packages_json = nlohmann::json::parse( packages_stream.str() );
345 ValidateJson( packages_json, nlohmann::json_uri(
"#/definitions/PackageArray" ) );
347 aPackages = packages_json[
"packages"].get<std::vector<PCM_PACKAGE>>();
349 catch( std::exception& e )
353 wxLogError( wxString::Format(
_(
"Unable to parse packages metadata:\n\n%s" ),
366 std::vector<unsigned char> bytes( picosha2::k_digest_size );
368 picosha2::hash256( std::istreambuf_iterator<char>( aStream ), std::istreambuf_iterator<char>(),
369 bytes.begin(), bytes.end() );
370 std::string hex_str = picosha2::bytes_to_hex_string( bytes.begin(), bytes.end() );
372 return aHash.compare( hex_str ) == 0;
380 wxT(
"Repository is not cached." ) );
391 const auto repository_tuple =
393 [&aRepositoryId](
const std::tuple<wxString, wxString, wxString>& t )
395 return std::get<0>( t ) == aRepositoryId;
401 wxString url = std::get<2>( *repository_tuple );
407 std::shared_ptr<PROGRESS_REPORTER> reporter;
410 reporter = std::make_shared<WX_PROGRESS_REPORTER>(
m_dialog, wxEmptyString, 1 );
417 bool packages_cache_exists =
false;
421 repo_cache.AppendDir( wxT(
"pcm" ) );
422 repo_cache.AppendDir( aRepositoryId );
423 wxFileName packages_cache( repo_cache.GetPath(), wxT(
"packages.json" ) );
425 if( repo_cache.FileExists() && packages_cache.FileExists() )
427 std::ifstream repo_stream( repo_cache.GetFullPath().fn_str() );
437 wxLogError(
_(
"Failed to parse locally stored repository.json." ) );
444 std::ifstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
448 packages_cache_stream >> js;
449 saved_repo.
package_list = js[
"packages"].get<std::vector<PCM_PACKAGE>>();
451 for(
size_t i = 0; i < saved_repo.
package_list.size(); i++ )
459 packages_cache_exists =
true;
465 wxLogError(
_(
"Packages cache for current repository is corrupted, it will "
466 "be redownloaded." ) );
472 if( !packages_cache_exists )
481 for(
size_t i = 0; i < current_repo.
package_list.size(); i++ )
487 repo_cache.Mkdir( wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL );
489 std::ofstream repo_cache_stream( repo_cache.GetFullPath().fn_str() );
490 repo_cache_stream << std::setw( 4 ) << nlohmann::json( current_repo ) << std::endl;
492 std::ofstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
494 js[
"packages"] = nlohmann::json( current_repo.
package_list );
495 packages_cache_stream << std::setw( 4 ) << js << std::endl;
506 wxFileName resource_file( repo_cache.GetPath(), wxT(
"resources.zip" ) );
510 if( resource_file.FileExists() )
511 mtime = wxFileModificationTime( resource_file.GetFullPath() );
515 std::ofstream resources_stream( resource_file.GetFullPath().fn_str(),
516 std::ios_base::binary );
518 reporter->SetTitle(
_(
"Downloading resources" ) );
524 resources_stream.close();
528 std::ifstream read_stream( resource_file.GetFullPath().fn_str(),
529 std::ios_base::binary );
538 wxLogError(
_(
"Resources file hash doesn't match and will not be used. "
539 "Repository may be corrupted." ) );
542 wxRemoveFile( resource_file.GetFullPath() );
548 wxRemoveFile( resource_file.GetFullPath() );
569 wxLogTrace(
tracePcm, wxS(
"Invalid/Missing repository " ) + aRepositoryId );
573 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
585 std::optional<PACKAGE_VERSION> current_version;
587 auto current_version_it =
591 return version.version == entry.current_version;
595 current_version = *current_version_it;
605 return version.version == entry.current_version;
616 return a.parsed_version > b.parsed_version;
628 int epoch = 0, major = 0, minor = 0, patch = 0;
633 wxStringTokenizer version_tokenizer( ver.
version, wxT(
"." ) );
635 major = wxAtoi( version_tokenizer.GetNextToken() );
637 if( version_tokenizer.HasMoreTokens() )
638 minor = wxAtoi( version_tokenizer.GetNextToken() );
640 if( version_tokenizer.HasMoreTokens() )
641 patch = wxAtoi( version_tokenizer.GetNextToken() );
643 ver.
parsed_version = std::make_tuple( epoch, major, minor, patch );
648 auto parse_version_tuple =
649 [](
const wxString& version,
int deflt )
651 int ver_major = deflt;
652 int ver_minor = deflt;
653 int ver_patch = deflt;
655 wxStringTokenizer tokenizer( version, wxT(
"." ) );
657 ver_major = wxAtoi( tokenizer.GetNextToken() );
659 if( tokenizer.HasMoreTokens() )
660 ver_minor = wxAtoi( tokenizer.GetNextToken() );
662 if( tokenizer.HasMoreTokens() )
663 ver_patch = wxAtoi( tokenizer.GetNextToken() );
665 return std::tuple<int, int, int>( ver_major, ver_minor, ver_patch );
676 wxString platform = wxT(
"windows" );
677#elif defined( __APPLE__ )
678 wxString platform = wxT(
"macos" );
680 wxString platform = wxT(
"linux" );
695 return a.parsed_version > b.parsed_version;
700const std::vector<PCM_PACKAGE>&
703 static std::vector<PCM_PACKAGE>
empty{};
721 auto it = std::find_if( aRepositories.begin(), aRepositories.end(),
722 [&](
const auto& new_entry )
724 return new_entry.first == std::get<1>( entry );
727 if( it == aRepositories.end() )
736 for(
const std::pair<wxString, wxString>& repo : aRepositories )
738 std::string url_sha = picosha2::hash256_hex_string( repo.second );
739 m_repository_list.push_back( std::make_tuple( url_sha.substr( 0, 16 ), repo.first,
751 repo_cache.AppendDir( wxT(
"pcm" ) );
752 repo_cache.AppendDir( aRepositoryId );
754 if( repo_cache.DirExists() )
755 repo_cache.Rmdir( wxPATH_RMDIR_RECURSIVE );
760 const wxString& aRepositoryId )
778 if( !aRepositoryId.IsEmpty() )
802 const wxString& aPackageId )
839 return ver.compatible;
850 wxT(
"GetPackageUpdateVersion called on a not installed package" ) );
854 auto installed_ver_it = std::find_if(
858 return ver.version == entry.current_version;
862 wxT(
"Installed package version not found" ) );
864 auto ver_it = std::find_if( aPackage.
versions.begin(), aPackage.
versions.end(),
867 return ver.compatible
868 && installed_ver_it->status >= ver.status
869 && installed_ver_it->parsed_version < ver.parsed_version;
872 return ver_it == aPackage.
versions.end() ? wxString( wxT(
"" ) ) : ver_it->version;
877 return std::chrono::duration_cast<std::chrono::seconds>(
878 std::chrono::system_clock::now().time_since_epoch() ).count();
887 js[
"packages"] = nlohmann::json::array();
889 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
891 js[
"packages"].emplace_back( pair.second );
895 std::ofstream stream( f.GetFullPath().fn_str() );
897 stream << std::setw( 4 ) << js << std::endl;
899 catch( nlohmann::detail::exception& )
908 std::vector<PCM_INSTALLATION_ENTRY> v;
911 [&v](
const std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
913 v.push_back( entry.second );
916 std::sort( v.begin(), v.end(),
919 return ( a.install_timestamp < b.install_timestamp )
920 || ( a.install_timestamp == b.install_timestamp
921 && a.package.identifier < b.package.identifier );
932 wxT(
"Installed package not found." ) );
934 return m_installed.at( aPackageId ).current_version;
957 const wxString& aSearchTerm )
959 wxArrayString terms = wxStringTokenize( aSearchTerm.Lower(), wxS(
" " ), wxTOKEN_STRTOK );
962 const auto find_term_matches =
963 [&](
const wxString& str )
966 wxString lower = str.Lower();
968 for(
const wxString& term : terms )
970 if( lower.Find( term ) != wxNOT_FOUND )
978 if( terms.size() == 1 && terms[0] == aPackage.
identifier )
981 if( terms.size() == 1 && find_term_matches( aPackage.
identifier ) )
985 rank += 500 * find_term_matches( aPackage.
name );
988 for(
const std::string& tag : aPackage.
tags )
989 rank += 100 * find_term_matches( wxString( tag ) );
992 rank += 10 * find_term_matches( aPackage.
description );
996 rank += find_term_matches( aPackage.
author.
name );
999 rank += 3 * find_term_matches( aPackage.
maintainer->name );
1002 for(
const std::pair<const std::string, wxString>& entry : aPackage.
resources )
1004 rank += find_term_matches( entry.first );
1005 rank += find_term_matches( entry.second );
1009 if( terms.size() == 1 && terms[0] == aPackage.
license )
1016std::unordered_map<wxString, wxBitmap>
1019 std::unordered_map<wxString, wxBitmap> bitmaps;
1022 resources_file.AppendDir( wxT(
"pcm" ) );
1023 resources_file.AppendDir( aRepositoryId );
1025 if( !resources_file.FileExists() )
1028 wxFFileInputStream stream( resources_file.GetFullPath() );
1029 wxZipInputStream
zip( stream );
1031 if( !
zip.IsOk() ||
zip.GetTotalEntries() == 0 )
1034 for( wxArchiveEntry* entry =
zip.GetNextEntry(); entry; entry =
zip.GetNextEntry() )
1036 wxArrayString path_parts = wxSplit( entry->GetName(), wxFileName::GetPathSeparator(),
1039 if( path_parts.size() != 2 || path_parts[1] != wxT(
"icon.png" ) )
1044 wxMemoryInputStream image_stream(
zip, entry->GetSize() );
1045 wxImage
image( image_stream, wxBITMAP_TYPE_PNG );
1046 bitmaps.emplace( path_parts[0], wxBitmap(
image ) );
1051 wxLogTrace( wxT(
"Error loading png bitmap for entry %s from %s" ), entry->GetName(),
1052 resources_file.GetFullPath() );
1062 std::unordered_map<wxString, wxBitmap> bitmaps;
1065 resources_dir_fn.AppendDir( wxT(
"resources" ) );
1066 wxDir resources_dir( resources_dir_fn.GetPath() );
1068 if( !resources_dir.IsOpened() )
1072 bool more = resources_dir.GetFirst( &subdir, wxEmptyString, wxDIR_DIRS | wxDIR_HIDDEN );
1076 wxFileName icon( resources_dir_fn.GetPath(), wxT(
"icon.png" ) );
1077 icon.AppendDir( subdir );
1079 if( icon.FileExists() )
1081 wxString actual_package_id = subdir;
1082 actual_package_id.Replace(
'_',
'.' );
1086 wxBitmap bitmap( icon.GetFullPath(), wxBITMAP_TYPE_PNG );
1087 bitmaps.emplace( actual_package_id, bitmap );
1092 wxLogTrace( wxT(
"Error loading png bitmap from %s" ), icon.GetFullPath() );
1096 more = resources_dir.GetNext( &subdir );
1122 std::unordered_set<wxString> repo_ids;
1124 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1126 if( !pair.second.pinned )
1127 repo_ids.insert( pair.second.repository_id );
1133 if( repo_ids.count( repository_id ) == 0 )
1137 _(
"Fetching repository..." ) );
1148 int availableUpdateCount = 0;
1151 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1163 availableUpdateCount++;
wxString GetMajorMinorVersion()
Get only the major and minor version in a string major.minor.
const std::tuple< int, int, int > & GetMajorMinorPatchTuple()
Get the build version numbers as a tuple.
std::shared_ptr< BACKGROUND_JOB > Create(const wxString &aName)
Creates a background job with the given name.
void Remove(std::shared_ptr< BACKGROUND_JOB > job)
Removes the given background job from any lists and frees it.
int Perform()
Equivalent to curl_easy_perform.
bool SetTransferCallback(const TRANSFER_CALLBACK &aCallback, size_t aInterval)
bool SetURL(const std::string &aURL)
Set the request URL.
bool SetFollowRedirects(bool aFollow)
Enable the following of HTTP(s) and other redirects, by default curl does not follow redirects.
bool SetOutputStream(const std::ostream *aOutput)
const std::string GetErrorText(int aCode)
Fetch CURL's "friendly" error string for a given error code.
static wxString GetDefault3rdPartyPath()
Gets the default path for PCM packages.
static wxString GetStockDataPath(bool aRespectRunFromBuildDir=true)
Gets the stock (install) data path, which is the base path for things like scripting,...
static wxString GetUserCachePath()
Gets the stock (install) 3d viewer plugins path.
static wxString GetUserSettingsPath()
Return the user configuration path used to store KiCad's configuration files.
virtual ENV_VAR_MAP & GetLocalEnvVariables() const
virtual BACKGROUND_JOBS_MONITOR & GetBackgroundJobMonitor() const
time_t getCurrentTimestamp() const
const std::vector< PCM_PACKAGE > & GetRepositoryPackages(const wxString &aRepositoryId) const
Get the packages metadata from a previously cached repository.
void SetRepositoryList(const STRING_PAIR_LIST &aRepositories)
Set list of repositories.
void ValidateJson(const nlohmann::json &aJson, const nlohmann::json_uri &aUri=nlohmann::json_uri("#")) const
Validates json against a specific definition in the PCM schema.
std::unordered_map< wxString, PCM_REPOSITORY > m_repository_cache
const PCM_REPOSITORY & getCachedRepository(const wxString &aRepositoryId) const
Get the cached repository metadata.
int GetPackageSearchRank(const PCM_PACKAGE &aPackage, const wxString &aSearchTerm)
Get the approximate measure of how much given package matches the search term.
void MarkUninstalled(const PCM_PACKAGE &aPackage)
Mark package as uninstalled.
bool CacheRepository(const wxString &aRepositoryId)
Cache specified repository packages and other metadata.
void SaveInstalledPackages()
Saves metadata of installed packages to disk.
~PLUGIN_CONTENT_MANAGER()
const std::vector< PCM_INSTALLATION_ENTRY > GetInstalledPackages() const
Get list of installed packages.
void SetPinned(const wxString &aPackageId, const bool aPinned)
Set the pinned status of a package.
static void PreparePackage(PCM_PACKAGE &aPackage)
Parses version strings and calculates compatibility.
PLUGIN_CONTENT_MANAGER(std::function< void(int)> aAvailableUpdateCallbac)
bool DownloadToStream(const wxString &aUrl, std::ostream *aOutput, PROGRESS_REPORTER *aReporter, const size_t aSizeLimit=DEFAULT_DOWNLOAD_MEM_LIMIT)
Downloads url to an output stream.
std::thread m_updateThread
PCM_PACKAGE_STATE GetPackageState(const wxString &aRepositoryId, const wxString &aPackageId)
Get current state of the package.
std::map< wxString, PCM_INSTALLATION_ENTRY > m_installed
void RunBackgroundUpdate()
Runs a background update thread that checks for new package versions.
std::unordered_map< wxString, wxBitmap > GetRepositoryPackageBitmaps(const wxString &aRepositoryId)
Get the icon bitmaps for repository packages.
const wxString GetPackageUpdateVersion(const PCM_PACKAGE &aPackage)
Get the preferred package update version or empty string if there is none.
void MarkInstalled(const PCM_PACKAGE &aPackage, const wxString &aVersion, const wxString &aRepositoryId)
Mark package as installed.
nlohmann::json_schema::json_validator m_schema_validator
std::unordered_map< wxString, wxBitmap > GetInstalledPackageBitmaps()
Get the icon bitmaps for installed packages.
const wxString & GetInstalledPackageVersion(const wxString &aPackageId) const
Get the current version of an installed package.
void updateInstalledPackagesMetadata(const wxString &aRepositoryId)
Updates metadata of installed packages from freshly fetched repo.
bool IsPackagePinned(const wxString &aPackageId) const
Returns pinned status of a package.
static const std::tuple< int, int, int > m_kicad_version
std::shared_ptr< BACKGROUND_JOB > m_updateBackgroundJob
void StopBackgroundUpdate()
Interrupts and joins() the update thread.
STRING_TUPLE_LIST m_repository_list
bool fetchPackages(const wxString &aUrl, const std::optional< wxString > &aHash, std::vector< PCM_PACKAGE > &aPackages, PROGRESS_REPORTER *aReporter)
Downloads packages metadata to in memory stream, verifies hash and attempts to parse it.
std::function< void(int)> m_availableUpdateCallback
bool FetchRepository(const wxString &aUrl, PCM_REPOSITORY &aRepository, PROGRESS_REPORTER *aReporter)
Fetches repository metadata from given url.
void DiscardRepositoryCache(const wxString &aRepositoryId)
Discard in-memory and on-disk cache of a repository.
bool VerifyHash(std::istream &aStream, const wxString &aHash) const
Verifies SHA256 hash of a binary stream.
void ReadEnvVar()
Stores 3rdparty path from environment variables.
A progress reporter interface for use in multi-threaded environments.
virtual bool IsCancelled() const =0
virtual bool KeepRefreshing(bool aWait=false)=0
Update the UI (if any).
virtual void Report(const wxString &aMessage)=0
Display aMessage in the progress bar dialog.
virtual void SetTitle(const wxString &aTitle)=0
Change the title displayed on the window caption.
virtual void SetCurrentProgress(double aProgress)=0
Set the progress value to aProgress (0..1).
void error(const json::json_pointer &ptr, const json &instance, const std::string &message) override
static bool empty(const wxTextEntryBase *aCtrl)
Functions related to environment variables, including help functions.
static const wxChar tracePcm[]
Flag to enable PCM debugging output.
std::map< wxString, ENV_VAR_ITEM > ENV_VAR_MAP
std::function< int(size_t, size_t, size_t, size_t)> TRANSFER_CALLBACK
Wrapper interface around the curl_easy API/.
KICOMMON_API std::optional< wxString > GetVersionedEnvVarValue(const std::map< wxString, ENV_VAR_ITEM > &aMap, const wxString &aBaseName)
Attempts to retrieve the value of a versioned environment variable, such as KICAD8_TEMPLATE_DIR.
std::vector< std::pair< wxString, wxString > > STRING_PAIR_LIST
const std::unordered_set< wxString > PCM_PACKAGE_DIRECTORIES({ "plugins", "footprints", "3dmodels", "symbols", "resources", "colors", "templates", "scripts" })
< Contains list of all valid directories that get extracted from a package archive
PGM_BASE & Pgm()
The global Program "get" accessor.
< Package version metadata Package metadata
PCM_PACKAGE_VERSION_STATUS status
std::optional< wxString > kicad_version_max
std::optional< int > version_epoch
std::vector< std::string > platforms
std::tuple< int, int, int, int > parsed_version
uint64_t install_timestamp
Repository reference to a resource.
wxString description_full
std::vector< std::string > tags
std::optional< PCM_CONTACT > maintainer
std::vector< PACKAGE_VERSION > versions
Package installation entry.
PCM_RESOURCE_REFERENCE packages
std::vector< PCM_PACKAGE > package_list
std::optional< PCM_RESOURCE_REFERENCE > resources
std::unordered_map< wxString, size_t > package_map
std::optional< wxString > sha256
uint64_t update_timestamp
#define FN_NORMALIZE_FLAGS
Default flags to pass to wxFileName::Normalize().