42#include <wx/mstream.h>
43#include <wx/tokenzr.h>
44#include <wx/wfstream.h>
45#include <wx/zipstrm.h>
54 void error(
const json::json_pointer& ptr,
const json& instance,
55 const std::string& message )
override
57 throw std::invalid_argument( std::string(
"At " ) + ptr.to_string() +
", value:\n"
58 + instance.dump() +
"\n" + message +
"\n" );
64 std::function<
void(
int )> aAvailableUpdateCallback ) :
66 m_availableUpdateCallback( aAvailableUpdateCallback )
73 schema_file.AppendDir( wxS(
"schemas" ) );
75 std::ifstream schema_stream( schema_file.GetFullPath().fn_str() );
76 nlohmann::json schema;
80 schema_stream >> schema;
83 catch( std::exception& e )
85 if( !schema_file.FileExists() )
87 wxLogError( wxString::Format(
_(
"schema file '%s' not found" ),
88 schema_file.GetFullPath() ) );
92 wxLogError( wxString::Format(
_(
"Error loading schema: %s" ), e.what() ) );
101 std::ifstream installed_stream( f.GetFullPath().fn_str() );
102 nlohmann::json installed;
106 installed_stream >> installed;
108 if( installed.contains(
"packages" ) && installed[
"packages"].is_array() )
110 for(
const auto& js_entry : installed[
"packages"] )
117 catch( std::exception& e )
119 wxLogError( wxString::Format(
_(
"Error loading installed packages list: %s" ),
133 wxDir package_dir( d.GetPath() );
135 if( !package_dir.IsOpened() )
139 bool more = package_dir.GetFirst( &subdir,
"", wxDIR_DIRS | wxDIR_HIDDEN );
143 wxString actual_package_id = subdir;
144 actual_package_id.Replace(
'_',
'.' );
149 wxFileName subdir_file( d.GetPath(), subdir );
153 int stat_code = wxStat( subdir_file.GetFullPath(), &stat );
173 more = package_dir.GetNext( &subdir );
180 [&]( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
182 PreparePackage( entry.second.package );
191 auto it = env.find( wxT(
"KICAD7_3RD_PARTY" ) );
193 if( it != env.end() && !it->second.GetValue().IsEmpty() )
202 const size_t aSizeLimit )
204 bool size_exceeded =
false;
206 TRANSFER_CALLBACK callback = [&](
size_t dltotal,
size_t dlnow,
size_t ultotal,
size_t ulnow )
208 if( aSizeLimit > 0 && ( dltotal > aSizeLimit || dlnow > aSizeLimit ) )
210 size_exceeded =
true;
219 aReporter->
Report( wxString::Format(
_(
"Downloading %lld/%lld kB" ), dlnow / 1000,
232 curl.
SetURL( aUrl.ToUTF8().data() );
241 if( code != CURLE_OK )
245 if( code == CURLE_ABORTED_BY_CALLBACK && size_exceeded )
246 wxMessageBox(
_(
"Download is too large." ) );
247 else if( code != CURLE_ABORTED_BY_CALLBACK )
261 std::stringstream repository_stream;
263 aReporter->
SetTitle(
_(
"Fetching repository" ) );
268 wxLogError(
_(
"Unable to load repository url" ) );
273 nlohmann::json repository_json;
277 repository_stream >> repository_json;
279 ValidateJson( repository_json, nlohmann::json_uri(
"#/definitions/Repository" ) );
283 catch(
const std::exception& e )
287 wxLogError( wxString::Format(
_(
"Unable to parse repository: %s" ), e.what() ) );
288 wxLogError(
_(
"The given repository URL does not look like a valid KiCad package "
289 "repository. Please double check the URL." ) );
300 const nlohmann::json_uri& aUri )
const
308 const std::optional<wxString>& aHash,
309 std::vector<PCM_PACKAGE>& aPackages,
312 std::stringstream packages_stream;
314 aReporter->
SetTitle(
_(
"Fetching repository packages" ) );
319 wxLogError(
_(
"Unable to load repository packages url." ) );
324 std::istringstream isstream( packages_stream.str() );
326 if( aHash && !
VerifyHash( isstream, *aHash ) )
329 wxLogError(
_(
"Packages hash doesn't match. Repository may be corrupted." ) );
336 nlohmann::json packages_json = nlohmann::json::parse( packages_stream.str() );
337 ValidateJson( packages_json, nlohmann::json_uri(
"#/definitions/PackageArray" ) );
339 aPackages = packages_json[
"packages"].get<std::vector<PCM_PACKAGE>>();
341 catch( std::exception& e )
345 wxLogError( wxString::Format(
_(
"Unable to parse packages metadata:\n\n%s" ),
358 std::vector<unsigned char> bytes( picosha2::k_digest_size );
360 picosha2::hash256( std::istreambuf_iterator<char>( aStream ), std::istreambuf_iterator<char>(),
361 bytes.begin(), bytes.end() );
362 std::string hex_str = picosha2::bytes_to_hex_string( bytes.begin(), bytes.end() );
364 return aHash.compare( hex_str ) == 0;
372 wxT(
"Repository is not cached." ) );
383 const auto repository_tuple =
385 [&aRepositoryId](
const std::tuple<wxString, wxString, wxString>& t )
387 return std::get<0>( t ) == aRepositoryId;
393 wxString url = std::get<2>( *repository_tuple );
399 std::shared_ptr<PROGRESS_REPORTER> reporter;
402 reporter = std::make_shared<WX_PROGRESS_REPORTER>(
m_dialog, wxEmptyString, 1 );
409 bool packages_cache_exists =
false;
413 repo_cache.AppendDir( wxT(
"pcm" ) );
414 repo_cache.AppendDir( aRepositoryId );
415 wxFileName packages_cache( repo_cache.GetPath(), wxT(
"packages.json" ) );
417 if( repo_cache.FileExists() && packages_cache.FileExists() )
419 std::ifstream repo_stream( repo_cache.GetFullPath().fn_str() );
429 wxLogError(
_(
"Failed to parse locally stored repository.json." ) );
436 std::ifstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
440 packages_cache_stream >> js;
441 saved_repo.
package_list = js[
"packages"].get<std::vector<PCM_PACKAGE>>();
443 for(
size_t i = 0; i < saved_repo.
package_list.size(); i++ )
451 packages_cache_exists =
true;
457 wxLogError(
_(
"Packages cache for current repository is corrupted, it will "
458 "be redownloaded." ) );
464 if( !packages_cache_exists )
473 for(
size_t i = 0; i < current_repo.
package_list.size(); i++ )
479 repo_cache.Mkdir( wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL );
481 std::ofstream repo_cache_stream( repo_cache.GetFullPath().fn_str() );
482 repo_cache_stream << std::setw( 4 ) << nlohmann::json( current_repo ) << std::endl;
484 std::ofstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
486 js[
"packages"] = nlohmann::json( current_repo.
package_list );
487 packages_cache_stream << std::setw( 4 ) << js << std::endl;
498 wxFileName resource_file( repo_cache.GetPath(), wxT(
"resources.zip" ) );
502 if( resource_file.FileExists() )
503 mtime = wxFileModificationTime( resource_file.GetFullPath() );
507 std::ofstream resources_stream( resource_file.GetFullPath().fn_str(),
508 std::ios_base::binary );
510 reporter->SetTitle(
_(
"Downloading resources" ) );
516 resources_stream.close();
520 std::ifstream read_stream( resource_file.GetFullPath().fn_str(),
521 std::ios_base::binary );
530 wxLogError(
_(
"Resources file hash doesn't match and will not be used. "
531 "Repository may be corrupted." ) );
534 wxRemoveFile( resource_file.GetFullPath() );
540 wxRemoveFile( resource_file.GetFullPath() );
561 wxLogDebug(
"Invalid/Missing repository " + aRepositoryId );
565 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
577 std::optional<PACKAGE_VERSION> current_version;
579 auto current_version_it =
583 return version.version == entry.current_version;
587 current_version = *current_version_it;
597 return version.version == entry.current_version;
608 return a.parsed_version > b.parsed_version;
620 int epoch = 0, major = 0, minor = 0, patch = 0;
625 wxStringTokenizer version_tokenizer( ver.
version, wxT(
"." ) );
627 major = wxAtoi( version_tokenizer.GetNextToken() );
629 if( version_tokenizer.HasMoreTokens() )
630 minor = wxAtoi( version_tokenizer.GetNextToken() );
632 if( version_tokenizer.HasMoreTokens() )
633 patch = wxAtoi( version_tokenizer.GetNextToken() );
635 ver.
parsed_version = std::make_tuple( epoch, major, minor, patch );
640 auto parse_version_tuple =
641 [](
const wxString& version,
int deflt )
643 int ver_major = deflt;
644 int ver_minor = deflt;
645 int ver_patch = deflt;
647 wxStringTokenizer tokenizer( version, wxT(
"." ) );
649 ver_major = wxAtoi( tokenizer.GetNextToken() );
651 if( tokenizer.HasMoreTokens() )
652 ver_minor = wxAtoi( tokenizer.GetNextToken() );
654 if( tokenizer.HasMoreTokens() )
655 ver_patch = wxAtoi( tokenizer.GetNextToken() );
657 return std::tuple<int, int, int>( ver_major, ver_minor, ver_patch );
668 wxString platform = wxT(
"windows" );
671 wxString platform = wxT(
"macos" );
674 wxString platform = wxT(
"linux" );
689 return a.parsed_version > b.parsed_version;
694const std::vector<PCM_PACKAGE>&
697 static std::vector<PCM_PACKAGE>
empty{};
715 auto it = std::find_if( aRepositories.begin(), aRepositories.end(),
716 [&](
const auto& new_entry )
718 return new_entry.first == std::get<1>( entry );
721 if( it == aRepositories.end() )
730 for(
const std::pair<wxString, wxString>& repo : aRepositories )
732 std::string url_sha = picosha2::hash256_hex_string( repo.second );
733 m_repository_list.push_back( std::make_tuple( url_sha.substr( 0, 16 ), repo.first,
745 repo_cache.AppendDir( wxT(
"pcm" ) );
746 repo_cache.AppendDir( aRepositoryId );
748 if( repo_cache.DirExists() )
749 repo_cache.Rmdir( wxPATH_RMDIR_RECURSIVE );
754 const wxString& aRepositoryId )
772 if( !aRepositoryId.IsEmpty() )
796 const wxString& aPackageId )
833 return ver.compatible;
844 wxT(
"GetPackageUpdateVersion called on a not installed package" ) );
848 auto installed_ver_it = std::find_if(
852 return ver.version == entry.current_version;
856 wxT(
"Installed package version not found" ) );
858 auto ver_it = std::find_if( aPackage.
versions.begin(), aPackage.
versions.end(),
861 return ver.compatible
862 && installed_ver_it->status >= ver.status
863 && installed_ver_it->parsed_version < ver.parsed_version;
866 return ver_it == aPackage.
versions.end() ? wxString( wxT(
"" ) ) : ver_it->version;
871 return std::chrono::duration_cast<std::chrono::seconds>(
872 std::chrono::system_clock::now().time_since_epoch() ).count();
881 js[
"packages"] = nlohmann::json::array();
883 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
885 js[
"packages"].emplace_back( pair.second );
889 std::ofstream stream( f.GetFullPath().fn_str() );
891 stream << std::setw( 4 ) << js << std::endl;
893 catch( nlohmann::detail::exception& )
902 std::vector<PCM_INSTALLATION_ENTRY> v;
905 [&v](
const std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
907 v.push_back( entry.second );
910 std::sort( v.begin(), v.end(),
913 return ( a.install_timestamp < b.install_timestamp )
914 || ( a.install_timestamp == b.install_timestamp
915 && a.package.identifier < b.package.identifier );
926 wxT(
"Installed package not found." ) );
928 return m_installed.at( aPackageId ).current_version;
951 const wxString& aSearchTerm )
953 wxArrayString terms = wxStringTokenize( aSearchTerm.Lower(), wxS(
" " ), wxTOKEN_STRTOK );
956 const auto find_term_matches =
957 [&](
const wxString& str )
960 wxString lower = str.Lower();
962 for(
const wxString& term : terms )
964 if( lower.Find( term ) != wxNOT_FOUND )
972 if( terms.size() == 1 && terms[0] == aPackage.
identifier )
975 if( terms.size() == 1 && find_term_matches( aPackage.
identifier ) )
979 rank += 500 * find_term_matches( aPackage.
name );
982 for(
const std::string& tag : aPackage.
tags )
983 rank += 100 * find_term_matches( wxString( tag ) );
986 rank += 10 * find_term_matches( aPackage.
description );
990 rank += find_term_matches( aPackage.
author.
name );
993 rank += 3 * find_term_matches( aPackage.
maintainer->name );
996 for(
const std::pair<const std::string, wxString>& entry : aPackage.
resources )
998 rank += find_term_matches( entry.first );
999 rank += find_term_matches( entry.second );
1003 if( terms.size() == 1 && terms[0] == aPackage.
license )
1010std::unordered_map<wxString, wxBitmap>
1013 std::unordered_map<wxString, wxBitmap> bitmaps;
1016 resources_file.AppendDir( wxT(
"pcm" ) );
1017 resources_file.AppendDir( aRepositoryId );
1019 if( !resources_file.FileExists() )
1022 wxFFileInputStream stream( resources_file.GetFullPath() );
1023 wxZipInputStream
zip( stream );
1025 if( !
zip.IsOk() ||
zip.GetTotalEntries() == 0 )
1028 for( wxArchiveEntry* entry =
zip.GetNextEntry(); entry; entry =
zip.GetNextEntry() )
1030 wxArrayString path_parts = wxSplit( entry->GetName(), wxFileName::GetPathSeparator(),
1033 if( path_parts.size() != 2 || path_parts[1] != wxT(
"icon.png" ) )
1038 wxMemoryInputStream image_stream(
zip, entry->GetSize() );
1039 wxImage
image( image_stream, wxBITMAP_TYPE_PNG );
1040 bitmaps.emplace( path_parts[0], wxBitmap(
image ) );
1045 wxLogTrace( wxT(
"Error loading png bitmap for entry %s from %s" ), entry->GetName(),
1046 resources_file.GetFullPath() );
1056 std::unordered_map<wxString, wxBitmap> bitmaps;
1059 resources_dir_fn.AppendDir( wxT(
"resources" ) );
1060 wxDir resources_dir( resources_dir_fn.GetPath() );
1062 if( !resources_dir.IsOpened() )
1066 bool more = resources_dir.GetFirst( &subdir, wxEmptyString, wxDIR_DIRS | wxDIR_HIDDEN );
1070 wxFileName icon( resources_dir_fn.GetPath(), wxT(
"icon.png" ) );
1071 icon.AppendDir( subdir );
1073 if( icon.FileExists() )
1075 wxString actual_package_id = subdir;
1076 actual_package_id.Replace(
'_',
'.' );
1080 wxBitmap bitmap( icon.GetFullPath(), wxBITMAP_TYPE_PNG );
1081 bitmaps.emplace( actual_package_id, bitmap );
1086 wxLogTrace( wxT(
"Error loading png bitmap from %s" ), icon.GetFullPath() );
1090 more = resources_dir.GetNext( &subdir );
1117 std::unordered_set<wxString> repo_ids;
1119 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1121 if( !pair.second.pinned )
1122 repo_ids.insert( pair.second.repository_id );
1128 if( repo_ids.count( repository_id ) == 0 )
1132 _(
"Fetching repository..." ) );
1143 int availableUpdateCount = 0;
1146 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1158 availableUpdateCount++;
wxString GetMajorMinorVersion()
Get only the major and minor version in a string major.minor.
const std::tuple< int, int, int > & GetMajorMinorPatchTuple()
Get the build version numbers as a tuple.
int Perform()
Equivalent to curl_easy_perform.
bool SetTransferCallback(const TRANSFER_CALLBACK &aCallback, size_t aInterval)
bool SetURL(const std::string &aURL)
Set the request URL.
bool SetFollowRedirects(bool aFollow)
Enable the following of HTTP(s) and other redirects, by default curl does not follow redirects.
bool SetOutputStream(const std::ostream *aOutput)
const std::string GetErrorText(int aCode)
Fetch CURL's "friendly" error string for a given error code.
static wxString GetDefault3rdPartyPath()
Gets the default path for PCM packages.
static wxString GetStockDataPath(bool aRespectRunFromBuildDir=true)
Gets the stock (install) data path, which is the base path for things like scripting,...
static wxString GetUserCachePath()
Gets the stock (install) 3d viewer plugins path.
static wxString GetUserSettingsPath()
Return the user configuration path used to store KiCad's configuration files.
time_t getCurrentTimestamp() const
const std::vector< PCM_PACKAGE > & GetRepositoryPackages(const wxString &aRepositoryId) const
Get the packages metadata from a previously cached repository.
void SetRepositoryList(const STRING_PAIR_LIST &aRepositories)
Set list of repositories.
void ValidateJson(const nlohmann::json &aJson, const nlohmann::json_uri &aUri=nlohmann::json_uri("#")) const
Validates json against a specific definition in the PCM schema.
std::unordered_map< wxString, PCM_REPOSITORY > m_repository_cache
const PCM_REPOSITORY & getCachedRepository(const wxString &aRepositoryId) const
Get the cached repository metadata.
int GetPackageSearchRank(const PCM_PACKAGE &aPackage, const wxString &aSearchTerm)
Get the approximate measure of how much given package matches the search term.
void MarkUninstalled(const PCM_PACKAGE &aPackage)
Mark package as uninstalled.
bool CacheRepository(const wxString &aRepositoryId)
Cache specified repository packages and other metadata.
void SaveInstalledPackages()
Saves metadata of installed packages to disk.
~PLUGIN_CONTENT_MANAGER()
const std::vector< PCM_INSTALLATION_ENTRY > GetInstalledPackages() const
Get list of installed packages.
void SetPinned(const wxString &aPackageId, const bool aPinned)
Set the pinned status of a package.
static void PreparePackage(PCM_PACKAGE &aPackage)
Parses version strings and calculates compatibility.
PLUGIN_CONTENT_MANAGER(std::function< void(int)> aAvailableUpdateCallbac)
bool DownloadToStream(const wxString &aUrl, std::ostream *aOutput, PROGRESS_REPORTER *aReporter, const size_t aSizeLimit=DEFAULT_DOWNLOAD_MEM_LIMIT)
Downloads url to an output stream.
std::thread m_updateThread
PCM_PACKAGE_STATE GetPackageState(const wxString &aRepositoryId, const wxString &aPackageId)
Get current state of the package.
std::map< wxString, PCM_INSTALLATION_ENTRY > m_installed
void RunBackgroundUpdate()
Runs a background update thread that checks for new package versions.
std::unordered_map< wxString, wxBitmap > GetRepositoryPackageBitmaps(const wxString &aRepositoryId)
Get the icon bitmaps for repository packages.
const wxString GetPackageUpdateVersion(const PCM_PACKAGE &aPackage)
Get the preferred package update version or empty string if there is none.
void MarkInstalled(const PCM_PACKAGE &aPackage, const wxString &aVersion, const wxString &aRepositoryId)
Mark package as installed.
nlohmann::json_schema::json_validator m_schema_validator
std::unordered_map< wxString, wxBitmap > GetInstalledPackageBitmaps()
Get the icon bitmaps for installed packages.
const wxString & GetInstalledPackageVersion(const wxString &aPackageId) const
Get the current version of an installed package.
void updateInstalledPackagesMetadata(const wxString &aRepositoryId)
Updates metadata of installed packages from freshly fetched repo.
bool IsPackagePinned(const wxString &aPackageId) const
Returns pinned status of a package.
static const std::tuple< int, int, int > m_kicad_version
std::shared_ptr< BACKGROUND_JOB > m_updateBackgroundJob
void StopBackgroundUpdate()
Interrupts and joins() the update thread.
STRING_TUPLE_LIST m_repository_list
bool fetchPackages(const wxString &aUrl, const std::optional< wxString > &aHash, std::vector< PCM_PACKAGE > &aPackages, PROGRESS_REPORTER *aReporter)
Downloads packages metadata to in memory stream, verifies hash and attempts to parse it.
std::function< void(int)> m_availableUpdateCallback
bool FetchRepository(const wxString &aUrl, PCM_REPOSITORY &aRepository, PROGRESS_REPORTER *aReporter)
Fetches repository metadata from given url.
void DiscardRepositoryCache(const wxString &aRepositoryId)
Discard in-memory and on-disk cache of a repository.
bool VerifyHash(std::istream &aStream, const wxString &aHash) const
Verifies SHA256 hash of a binary stream.
void ReadEnvVar()
Stores 3rdparty path from environment variables.
A progress reporter interface for use in multi-threaded environments.
virtual bool IsCancelled() const =0
virtual bool KeepRefreshing(bool aWait=false)=0
Update the UI (if any).
virtual void Report(const wxString &aMessage)=0
Display aMessage in the progress bar dialog.
virtual void SetTitle(const wxString &aTitle)=0
Change the title displayed on the window caption.
virtual void SetCurrentProgress(double aProgress)=0
Set the progress value to aProgress (0..1).
void error(const json::json_pointer &ptr, const json &instance, const std::string &message) override
static bool empty(const wxTextEntryBase *aCtrl)
std::map< wxString, ENV_VAR_ITEM > ENV_VAR_MAP
std::function< int(size_t, size_t, size_t, size_t)> TRANSFER_CALLBACK
Wrapper interface around the curl_easy API/.
std::vector< std::pair< wxString, wxString > > STRING_PAIR_LIST
const std::unordered_set< wxString > PCM_PACKAGE_DIRECTORIES({ "plugins", "footprints", "3dmodels", "symbols", "resources", "colors", "templates", "scripts" })
< Contains list of all valid directories that get extracted from a package archive
KIWAY Kiway & Pgm(), KFCTL_STANDALONE
The global Program "get" accessor.
< Package version metadata Package metadata
PCM_PACKAGE_VERSION_STATUS status
std::optional< wxString > kicad_version_max
std::optional< int > version_epoch
std::vector< std::string > platforms
std::tuple< int, int, int, int > parsed_version
uint64_t install_timestamp
Repository reference to a resource.
wxString description_full
std::vector< std::string > tags
std::optional< PCM_CONTACT > maintainer
std::vector< PACKAGE_VERSION > versions
Package installation entry.
PCM_RESOURCE_REFERENCE packages
std::vector< PCM_PACKAGE > package_list
std::optional< PCM_RESOURCE_REFERENCE > resources
std::unordered_map< wxString, size_t > package_map
std::optional< wxString > sha256
uint64_t update_timestamp
#define FN_NORMALIZE_FLAGS
Default flags to pass to wxFileName::Normalize().