44#include <wx/mstream.h>
45#include <wx/tokenzr.h>
46#include <wx/wfstream.h>
47#include <wx/zipstrm.h>
55static const wxChar
tracePcm[] = wxT(
"KICAD_PCM" );
64 void error(
const json::json_pointer& ptr,
const json& instance,
65 const std::string& message )
override
67 throw std::invalid_argument( std::string(
"At " ) + ptr.to_string() +
", value:\n"
68 + instance.dump() +
"\n" + message +
"\n" );
74 std::function<
void(
int )> aAvailableUpdateCallback ) :
76 m_availableUpdateCallback( aAvailableUpdateCallback )
83 schema_file.AppendDir( wxS(
"schemas" ) );
92 std::ifstream installed_stream( f.GetFullPath().fn_str() );
93 nlohmann::json installed;
97 installed_stream >> installed;
99 if( installed.contains(
"packages" ) && installed[
"packages"].is_array() )
101 for(
const auto& js_entry : installed[
"packages"] )
108 catch( std::exception& e )
110 wxLogError( wxString::Format(
_(
"Error loading installed packages list: %s" ),
124 wxDir package_dir( d.GetPath() );
126 if( !package_dir.IsOpened() )
130 bool more = package_dir.GetFirst( &subdir,
"", wxDIR_DIRS | wxDIR_HIDDEN );
134 wxString actual_package_id = subdir;
135 actual_package_id.Replace(
'_',
'.' );
140 wxFileName subdir_file( d.GetPath(), subdir );
144 int stat_code = wxStat( subdir_file.GetFullPath(), &stat );
164 more = package_dir.GetNext( &subdir );
171 [&]( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
173 PreparePackage( entry.second.package );
192 const size_t aSizeLimit )
194 bool size_exceeded =
false;
196 TRANSFER_CALLBACK callback = [&](
size_t dltotal,
size_t dlnow,
size_t ultotal,
size_t ulnow )
198 if( aSizeLimit > 0 && ( dltotal > aSizeLimit || dlnow > aSizeLimit ) )
200 size_exceeded =
true;
209 aReporter->
Report( wxString::Format(
_(
"Downloading %lld/%lld kB" ), dlnow / 1000,
222 curl.
SetURL( aUrl.ToUTF8().data() );
231 if( code != CURLE_OK )
235 if( code == CURLE_ABORTED_BY_CALLBACK && size_exceeded )
236 wxMessageBox(
_(
"Download is too large." ) );
237 else if( code != CURLE_ABORTED_BY_CALLBACK )
251 std::stringstream repository_stream;
253 aReporter->
SetTitle(
_(
"Fetching repository" ) );
258 wxLogError(
_(
"Unable to load repository url" ) );
263 nlohmann::json repository_json;
267 repository_stream >> repository_json;
269 ValidateJson( repository_json, nlohmann::json_uri(
"#/definitions/Repository" ) );
273 catch(
const std::exception& e )
277 wxLogError( wxString::Format(
_(
"Unable to parse repository: %s" ), e.what() ) );
278 wxLogError(
_(
"The given repository URL does not look like a valid KiCad package "
279 "repository. Please double check the URL." ) );
290 const nlohmann::json_uri& aUri )
const
298 const std::optional<wxString>& aHash,
299 std::vector<PCM_PACKAGE>& aPackages,
302 std::stringstream packages_stream;
304 aReporter->
SetTitle(
_(
"Fetching repository packages" ) );
309 wxLogError(
_(
"Unable to load repository packages url." ) );
314 std::istringstream isstream( packages_stream.str() );
316 if( aHash && !
VerifyHash( isstream, *aHash ) )
319 wxLogError(
_(
"Packages hash doesn't match. Repository may be corrupted." ) );
326 nlohmann::json packages_json = nlohmann::json::parse( packages_stream.str() );
327 ValidateJson( packages_json, nlohmann::json_uri(
"#/definitions/PackageArray" ) );
329 aPackages = packages_json[
"packages"].get<std::vector<PCM_PACKAGE>>();
331 catch( std::exception& e )
335 wxLogError( wxString::Format(
_(
"Unable to parse packages metadata:\n\n%s" ),
348 std::vector<unsigned char> bytes( picosha2::k_digest_size );
350 picosha2::hash256( std::istreambuf_iterator<char>( aStream ), std::istreambuf_iterator<char>(),
351 bytes.begin(), bytes.end() );
352 std::string hex_str = picosha2::bytes_to_hex_string( bytes.begin(), bytes.end() );
354 return aHash.compare( hex_str ) == 0;
362 wxT(
"Repository is not cached." ) );
373 const auto repository_tuple =
375 [&aRepositoryId](
const std::tuple<wxString, wxString, wxString>& t )
377 return std::get<0>( t ) == aRepositoryId;
383 wxString url = std::get<2>( *repository_tuple );
389 std::shared_ptr<PROGRESS_REPORTER> reporter;
392 reporter = std::make_shared<WX_PROGRESS_REPORTER>(
m_dialog, wxEmptyString, 1 );
399 bool packages_cache_exists =
false;
403 repo_cache.AppendDir( wxT(
"pcm" ) );
404 repo_cache.AppendDir( aRepositoryId );
405 wxFileName packages_cache( repo_cache.GetPath(), wxT(
"packages.json" ) );
407 if( repo_cache.FileExists() && packages_cache.FileExists() )
409 std::ifstream repo_stream( repo_cache.GetFullPath().fn_str() );
419 wxLogError(
_(
"Failed to parse locally stored repository.json." ) );
426 std::ifstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
430 packages_cache_stream >> js;
431 saved_repo.
package_list = js[
"packages"].get<std::vector<PCM_PACKAGE>>();
433 for(
size_t i = 0; i < saved_repo.
package_list.size(); i++ )
441 packages_cache_exists =
true;
447 wxLogError(
_(
"Packages cache for current repository is corrupted, it will "
448 "be redownloaded." ) );
454 if( !packages_cache_exists )
463 for(
size_t i = 0; i < current_repo.
package_list.size(); i++ )
469 repo_cache.Mkdir( wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL );
471 std::ofstream repo_cache_stream( repo_cache.GetFullPath().fn_str() );
472 repo_cache_stream << std::setw( 4 ) << nlohmann::json( current_repo ) << std::endl;
474 std::ofstream packages_cache_stream( packages_cache.GetFullPath().fn_str() );
476 js[
"packages"] = nlohmann::json( current_repo.
package_list );
477 packages_cache_stream << std::setw( 4 ) << js << std::endl;
488 wxFileName resource_file( repo_cache.GetPath(), wxT(
"resources.zip" ) );
492 if( resource_file.FileExists() )
493 mtime = wxFileModificationTime( resource_file.GetFullPath() );
497 std::ofstream resources_stream( resource_file.GetFullPath().fn_str(),
498 std::ios_base::binary );
500 reporter->SetTitle(
_(
"Downloading resources" ) );
506 resources_stream.close();
510 std::ifstream read_stream( resource_file.GetFullPath().fn_str(),
511 std::ios_base::binary );
520 wxLogError(
_(
"Resources file hash doesn't match and will not be used. "
521 "Repository may be corrupted." ) );
524 wxRemoveFile( resource_file.GetFullPath() );
530 wxRemoveFile( resource_file.GetFullPath() );
551 wxLogTrace(
tracePcm, wxS(
"Invalid/Missing repository " ) + aRepositoryId );
555 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
567 std::optional<PACKAGE_VERSION> current_version;
569 auto current_version_it =
573 return version.version == entry.current_version;
577 current_version = *current_version_it;
587 return version.version == entry.current_version;
598 return a.parsed_version > b.parsed_version;
610 int epoch = 0, major = 0, minor = 0, patch = 0;
615 wxStringTokenizer version_tokenizer( ver.
version, wxT(
"." ) );
617 major = wxAtoi( version_tokenizer.GetNextToken() );
619 if( version_tokenizer.HasMoreTokens() )
620 minor = wxAtoi( version_tokenizer.GetNextToken() );
622 if( version_tokenizer.HasMoreTokens() )
623 patch = wxAtoi( version_tokenizer.GetNextToken() );
625 ver.
parsed_version = std::make_tuple( epoch, major, minor, patch );
630 auto parse_version_tuple =
631 [](
const wxString& version,
int deflt )
633 int ver_major = deflt;
634 int ver_minor = deflt;
635 int ver_patch = deflt;
637 wxStringTokenizer tokenizer( version, wxT(
"." ) );
639 ver_major = wxAtoi( tokenizer.GetNextToken() );
641 if( tokenizer.HasMoreTokens() )
642 ver_minor = wxAtoi( tokenizer.GetNextToken() );
644 if( tokenizer.HasMoreTokens() )
645 ver_patch = wxAtoi( tokenizer.GetNextToken() );
647 return std::tuple<int, int, int>( ver_major, ver_minor, ver_patch );
658 wxString platform = wxT(
"windows" );
659#elif defined( __APPLE__ )
660 wxString platform = wxT(
"macos" );
662 wxString platform = wxT(
"linux" );
677 return a.parsed_version > b.parsed_version;
682const std::vector<PCM_PACKAGE>&
685 static std::vector<PCM_PACKAGE>
empty{};
703 auto it = std::find_if( aRepositories.begin(), aRepositories.end(),
704 [&](
const auto& new_entry )
706 return new_entry.first == std::get<1>( entry );
709 if( it == aRepositories.end() )
718 for(
const std::pair<wxString, wxString>& repo : aRepositories )
720 std::string url_sha = picosha2::hash256_hex_string( repo.second );
721 m_repository_list.push_back( std::make_tuple( url_sha.substr( 0, 16 ), repo.first,
733 repo_cache.AppendDir( wxT(
"pcm" ) );
734 repo_cache.AppendDir( aRepositoryId );
736 if( repo_cache.DirExists() )
737 repo_cache.Rmdir( wxPATH_RMDIR_RECURSIVE );
742 const wxString& aRepositoryId )
760 if( !aRepositoryId.IsEmpty() )
784 const wxString& aPackageId )
821 return ver.compatible;
832 wxT(
"GetPackageUpdateVersion called on a not installed package" ) );
836 auto installed_ver_it = std::find_if(
840 return ver.version == entry.current_version;
844 wxT(
"Installed package version not found" ) );
846 auto ver_it = std::find_if( aPackage.
versions.begin(), aPackage.
versions.end(),
849 return ver.compatible
850 && installed_ver_it->status >= ver.status
851 && installed_ver_it->parsed_version < ver.parsed_version;
854 return ver_it == aPackage.
versions.end() ? wxString( wxT(
"" ) ) : ver_it->version;
859 return std::chrono::duration_cast<std::chrono::seconds>(
860 std::chrono::system_clock::now().time_since_epoch() ).count();
869 js[
"packages"] = nlohmann::json::array();
871 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
873 js[
"packages"].emplace_back( pair.second );
877 std::ofstream stream( f.GetFullPath().fn_str() );
879 stream << std::setw( 4 ) << js << std::endl;
881 catch( nlohmann::detail::exception& )
890 std::vector<PCM_INSTALLATION_ENTRY> v;
893 [&v](
const std::pair<const wxString, PCM_INSTALLATION_ENTRY>& entry )
895 v.push_back( entry.second );
898 std::sort( v.begin(), v.end(),
901 return ( a.install_timestamp < b.install_timestamp )
902 || ( a.install_timestamp == b.install_timestamp
903 && a.package.identifier < b.package.identifier );
914 wxT(
"Installed package not found." ) );
916 return m_installed.at( aPackageId ).current_version;
939 const wxString& aSearchTerm )
941 wxArrayString terms = wxStringTokenize( aSearchTerm.Lower(), wxS(
" " ), wxTOKEN_STRTOK );
944 const auto find_term_matches =
945 [&](
const wxString& str )
948 wxString lower = str.Lower();
950 for(
const wxString& term : terms )
952 if( lower.Find( term ) != wxNOT_FOUND )
960 if( terms.size() == 1 && terms[0] == aPackage.
identifier )
963 if( terms.size() == 1 && find_term_matches( aPackage.
identifier ) )
967 rank += 500 * find_term_matches( aPackage.
name );
970 for(
const std::string& tag : aPackage.
tags )
971 rank += 100 * find_term_matches( wxString( tag ) );
974 rank += 10 * find_term_matches( aPackage.
description );
978 rank += find_term_matches( aPackage.
author.
name );
981 rank += 3 * find_term_matches( aPackage.
maintainer->name );
984 for(
const std::pair<const std::string, wxString>& entry : aPackage.
resources )
986 rank += find_term_matches( entry.first );
987 rank += find_term_matches( entry.second );
991 if( terms.size() == 1 && terms[0] == aPackage.
license )
998std::unordered_map<wxString, wxBitmap>
1001 std::unordered_map<wxString, wxBitmap> bitmaps;
1004 resources_file.AppendDir( wxT(
"pcm" ) );
1005 resources_file.AppendDir( aRepositoryId );
1007 if( !resources_file.FileExists() )
1010 wxFFileInputStream stream( resources_file.GetFullPath() );
1011 wxZipInputStream
zip( stream );
1013 if( !
zip.IsOk() ||
zip.GetTotalEntries() == 0 )
1016 for( wxArchiveEntry* entry =
zip.GetNextEntry(); entry; entry =
zip.GetNextEntry() )
1018 wxArrayString path_parts = wxSplit( entry->GetName(), wxFileName::GetPathSeparator(),
1021 if( path_parts.size() != 2 || path_parts[1] != wxT(
"icon.png" ) )
1026 wxMemoryInputStream image_stream(
zip, entry->GetSize() );
1027 wxImage
image( image_stream, wxBITMAP_TYPE_PNG );
1028 bitmaps.emplace( path_parts[0], wxBitmap(
image ) );
1033 wxLogTrace( wxT(
"Error loading png bitmap for entry %s from %s" ), entry->GetName(),
1034 resources_file.GetFullPath() );
1044 std::unordered_map<wxString, wxBitmap> bitmaps;
1047 resources_dir_fn.AppendDir( wxT(
"resources" ) );
1048 wxDir resources_dir( resources_dir_fn.GetPath() );
1050 if( !resources_dir.IsOpened() )
1054 bool more = resources_dir.GetFirst( &subdir, wxEmptyString, wxDIR_DIRS | wxDIR_HIDDEN );
1058 wxFileName icon( resources_dir_fn.GetPath(), wxT(
"icon.png" ) );
1059 icon.AppendDir( subdir );
1061 if( icon.FileExists() )
1063 wxString actual_package_id = subdir;
1064 actual_package_id.Replace(
'_',
'.' );
1068 wxBitmap bitmap( icon.GetFullPath(), wxBITMAP_TYPE_PNG );
1069 bitmaps.emplace( actual_package_id, bitmap );
1074 wxLogTrace( wxT(
"Error loading png bitmap from %s" ), icon.GetFullPath() );
1078 more = resources_dir.GetNext( &subdir );
1122 std::unordered_set<wxString> repo_ids;
1124 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1126 if( !pair.second.pinned )
1127 repo_ids.insert( pair.second.repository_id );
1133 if( repo_ids.count( repository_id ) == 0 )
1137 _(
"Fetching repository..." ) );
1148 int availableUpdateCount = 0;
1151 for( std::pair<const wxString, PCM_INSTALLATION_ENTRY>& pair :
m_installed )
1163 availableUpdateCount++;
wxString GetMajorMinorVersion()
Get only the major and minor version in a string major.minor.
const std::tuple< int, int, int > & GetMajorMinorPatchTuple()
Get the build version numbers as a tuple.
std::shared_ptr< BACKGROUND_JOB > Create(const wxString &aName)
Creates a background job with the given name.
void Remove(std::shared_ptr< BACKGROUND_JOB > job)
Removes the given background job from any lists and frees it.
int Perform()
Equivalent to curl_easy_perform.
bool SetTransferCallback(const TRANSFER_CALLBACK &aCallback, size_t aInterval)
bool SetURL(const std::string &aURL)
Set the request URL.
bool SetFollowRedirects(bool aFollow)
Enable the following of HTTP(s) and other redirects, by default curl does not follow redirects.
bool SetOutputStream(const std::ostream *aOutput)
const std::string GetErrorText(int aCode)
Fetch CURL's "friendly" error string for a given error code.
static wxString GetDefault3rdPartyPath()
Gets the default path for PCM packages.
static wxString GetStockDataPath(bool aRespectRunFromBuildDir=true)
Gets the stock (install) data path, which is the base path for things like scripting,...
static wxString GetUserCachePath()
Gets the stock (install) 3d viewer plugins path.
static wxString GetUserSettingsPath()
Return the user configuration path used to store KiCad's configuration files.
virtual ENV_VAR_MAP & GetLocalEnvVariables() const
virtual BACKGROUND_JOBS_MONITOR & GetBackgroundJobMonitor() const
time_t getCurrentTimestamp() const
const std::vector< PCM_PACKAGE > & GetRepositoryPackages(const wxString &aRepositoryId) const
Get the packages metadata from a previously cached repository.
void SetRepositoryList(const STRING_PAIR_LIST &aRepositories)
Set list of repositories.
std::unique_ptr< JSON_SCHEMA_VALIDATOR > m_schema_validator
void ValidateJson(const nlohmann::json &aJson, const nlohmann::json_uri &aUri=nlohmann::json_uri("#")) const
Validates json against a specific definition in the PCM schema.
std::unordered_map< wxString, PCM_REPOSITORY > m_repository_cache
const PCM_REPOSITORY & getCachedRepository(const wxString &aRepositoryId) const
Get the cached repository metadata.
int GetPackageSearchRank(const PCM_PACKAGE &aPackage, const wxString &aSearchTerm)
Get the approximate measure of how much given package matches the search term.
void MarkUninstalled(const PCM_PACKAGE &aPackage)
Mark package as uninstalled.
bool CacheRepository(const wxString &aRepositoryId)
Cache specified repository packages and other metadata.
void SaveInstalledPackages()
Saves metadata of installed packages to disk.
~PLUGIN_CONTENT_MANAGER()
const std::vector< PCM_INSTALLATION_ENTRY > GetInstalledPackages() const
Get list of installed packages.
void SetPinned(const wxString &aPackageId, const bool aPinned)
Set the pinned status of a package.
static void PreparePackage(PCM_PACKAGE &aPackage)
Parses version strings and calculates compatibility.
PLUGIN_CONTENT_MANAGER(std::function< void(int)> aAvailableUpdateCallbac)
bool DownloadToStream(const wxString &aUrl, std::ostream *aOutput, PROGRESS_REPORTER *aReporter, const size_t aSizeLimit=DEFAULT_DOWNLOAD_MEM_LIMIT)
Downloads url to an output stream.
std::thread m_updateThread
PCM_PACKAGE_STATE GetPackageState(const wxString &aRepositoryId, const wxString &aPackageId)
Get current state of the package.
std::map< wxString, PCM_INSTALLATION_ENTRY > m_installed
void RunBackgroundUpdate()
Runs a background update thread that checks for new package versions.
std::unordered_map< wxString, wxBitmap > GetRepositoryPackageBitmaps(const wxString &aRepositoryId)
Get the icon bitmaps for repository packages.
const wxString GetPackageUpdateVersion(const PCM_PACKAGE &aPackage)
Get the preferred package update version or empty string if there is none.
void MarkInstalled(const PCM_PACKAGE &aPackage, const wxString &aVersion, const wxString &aRepositoryId)
Mark package as installed.
std::unordered_map< wxString, wxBitmap > GetInstalledPackageBitmaps()
Get the icon bitmaps for installed packages.
const wxString & GetInstalledPackageVersion(const wxString &aPackageId) const
Get the current version of an installed package.
void updateInstalledPackagesMetadata(const wxString &aRepositoryId)
Updates metadata of installed packages from freshly fetched repo.
bool IsPackagePinned(const wxString &aPackageId) const
Returns pinned status of a package.
static const std::tuple< int, int, int > m_kicad_version
std::shared_ptr< BACKGROUND_JOB > m_updateBackgroundJob
void StopBackgroundUpdate()
Interrupts and joins() the update thread.
STRING_TUPLE_LIST m_repository_list
bool fetchPackages(const wxString &aUrl, const std::optional< wxString > &aHash, std::vector< PCM_PACKAGE > &aPackages, PROGRESS_REPORTER *aReporter)
Downloads packages metadata to in memory stream, verifies hash and attempts to parse it.
std::function< void(int)> m_availableUpdateCallback
bool FetchRepository(const wxString &aUrl, PCM_REPOSITORY &aRepository, PROGRESS_REPORTER *aReporter)
Fetches repository metadata from given url.
void DiscardRepositoryCache(const wxString &aRepositoryId)
Discard in-memory and on-disk cache of a repository.
bool VerifyHash(std::istream &aStream, const wxString &aHash) const
Verifies SHA256 hash of a binary stream.
void ReadEnvVar()
Stores 3rdparty path from environment variables.
A progress reporter interface for use in multi-threaded environments.
virtual bool IsCancelled() const =0
virtual bool KeepRefreshing(bool aWait=false)=0
Update the UI (if any).
virtual void Report(const wxString &aMessage)=0
Display aMessage in the progress bar dialog.
virtual void SetTitle(const wxString &aTitle)=0
Change the title displayed on the window caption.
virtual void SetCurrentProgress(double aProgress)=0
Set the progress value to aProgress (0..1).
void error(const json::json_pointer &ptr, const json &instance, const std::string &message) override
static bool empty(const wxTextEntryBase *aCtrl)
Functions related to environment variables, including help functions.
static const wxChar tracePcm[]
Flag to enable PCM debugging output.
std::map< wxString, ENV_VAR_ITEM > ENV_VAR_MAP
std::function< int(size_t, size_t, size_t, size_t)> TRANSFER_CALLBACK
Wrapper interface around the curl_easy API/.
KICOMMON_API std::optional< wxString > GetVersionedEnvVarValue(const std::map< wxString, ENV_VAR_ITEM > &aMap, const wxString &aBaseName)
Attempt to retrieve the value of a versioned environment variable, such as KICAD8_TEMPLATE_DIR.
std::vector< std::pair< wxString, wxString > > STRING_PAIR_LIST
const std::unordered_set< wxString > PCM_PACKAGE_DIRECTORIES({ "plugins", "footprints", "3dmodels", "symbols", "resources", "colors", "templates", "scripts" })
< Contains list of all valid directories that get extracted from a package archive
PGM_BASE & Pgm()
The global program "get" accessor.
< Package version metadata Package metadata
PCM_PACKAGE_VERSION_STATUS status
std::optional< wxString > kicad_version_max
std::optional< int > version_epoch
std::vector< std::string > platforms
std::tuple< int, int, int, int > parsed_version
uint64_t install_timestamp
Repository reference to a resource.
wxString description_full
std::vector< std::string > tags
std::optional< PCM_CONTACT > maintainer
std::vector< PACKAGE_VERSION > versions
Package installation entry.
PCM_RESOURCE_REFERENCE packages
std::vector< PCM_PACKAGE > package_list
std::optional< PCM_RESOURCE_REFERENCE > resources
std::unordered_map< wxString, size_t > package_map
std::optional< wxString > sha256
uint64_t update_timestamp
UPDATE_CANCELLER(std::shared_ptr< BACKGROUND_JOB > &aJob)
std::shared_ptr< BACKGROUND_JOB > & m_jobToCancel
#define FN_NORMALIZE_FLAGS
Default flags to pass to wxFileName::Normalize().