KiCad PCB EDA Suite
pcm_task_manager.cpp
Go to the documentation of this file.
1 /*
2  * This program source code file is part of KiCad, a free EDA CAD application.
3  *
4  * Copyright (C) 2021 Andrew Lutsenko, anlutsenko at gmail dot com
5  * Copyright (C) 1992-2021 KiCad Developers, see AUTHORS.txt for contributors.
6  *
7  * This program is free software: you can redistribute it and/or modify it
8  * under the terms of the GNU General Public License as published by the
9  * Free Software Foundation, either version 3 of the License, or (at your
10  * option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License along
18  * with this program. If not, see <http://www.gnu.org/licenses/>.
19  */
20 // kicad_curl_easy.h **must be** included before any wxWidgets header to avoid conflicts
21 // at least on Windows/msys2
22 #include <kicad_curl/kicad_curl.h>
24 
25 #include "pcm_task_manager.h"
26 #include "reporter.h"
27 #include "wxstream_helper.h"
28 
29 #include <fstream>
30 #include <thread>
31 #include <unordered_set>
32 #include <wx/dir.h>
33 #include <wx/filename.h>
34 #include <wx/msgdlg.h>
35 #include <wx/sstream.h>
36 #include <wx/wfstream.h>
37 #include <wx/zipstrm.h>
38 
39 
40 void PCM_TASK_MANAGER::DownloadAndInstall( const PCM_PACKAGE& aPackage, const wxString& aVersion,
41  const wxString& aRepositoryId )
42 {
43  PCM_TASK download_task = [aPackage, aVersion, aRepositoryId, this]()
44  {
45  wxFileName file_path( m_pcm->Get3rdPartyPath(), "" );
46  file_path.AppendDir( "cache" );
47  file_path.SetFullName( wxString::Format( "%s_v%s.zip", aPackage.identifier, aVersion ) );
48 
49  auto find_pkgver = std::find_if( aPackage.versions.begin(), aPackage.versions.end(),
50  [&aVersion]( const PACKAGE_VERSION& pv )
51  {
52  return pv.version == aVersion;
53  } );
54 
55  wxASSERT_MSG( find_pkgver != aPackage.versions.end(), "Package version not found" );
56 
57  if( !wxDirExists( file_path.GetPath() )
58  && !wxFileName::Mkdir( file_path.GetPath(), wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL ) )
59  {
60  m_reporter->Report( _( "Unable to create download directory!" ), RPT_SEVERITY_ERROR );
61  return;
62  }
63 
64  int code = downloadFile( file_path.GetFullPath(), find_pkgver->download_url.get() );
65 
66  if( code == CURLE_OK )
67  {
68  PCM_TASK install_task = [aPackage, aVersion, aRepositoryId, file_path, this]()
69  {
70  auto get_pkgver = std::find_if( aPackage.versions.begin(), aPackage.versions.end(),
71  [&aVersion]( const PACKAGE_VERSION& pv )
72  {
73  return pv.version == aVersion;
74  } );
75 
76  const boost::optional<wxString>& hash = get_pkgver->download_sha256;
77  bool hash_match = true;
78 
79  if( hash )
80  {
81  std::ifstream stream( file_path.GetFullPath().ToUTF8(), std::ios::binary );
82  hash_match = m_pcm->VerifyHash( stream, hash.get() );
83  }
84 
85  if( !hash_match )
86  {
87  m_reporter->Report( wxString::Format( _( "Downloaded archive hash for package "
88  "%s does not match repository entry. "
89  "This may indicate a problem with the "
90  "package, if the issue persists "
91  "report this to repository maintainers." ),
92  aPackage.identifier ),
94  }
95  else
96  {
97  m_reporter->Report( wxString::Format( _( "Extracting package '%s'." ),
98  aPackage.identifier ) );
99 
100  if( extract( file_path.GetFullPath(), aPackage.identifier, true ) )
101  {
102  m_pcm->MarkInstalled( aPackage, get_pkgver->version, aRepositoryId );
103  // TODO register libraries.
104  }
105  else
106  {
107  // Cleanup possibly partially extracted package
109  }
110  }
111 
112  m_reporter->Report( wxString::Format( _( "Removing downloaded archive '%s'." ),
113  file_path.GetFullName() ) );
114  wxRemoveFile( file_path.GetFullPath() );
115  };
116 
117  m_install_queue.push( install_task );
118  }
119  else
120  {
121  // Cleanup after ourselves.
122  wxRemoveFile( file_path.GetFullPath() );
123  }
124  };
125 
126  m_download_queue.push( download_task );
127 }
128 
129 
130 int PCM_TASK_MANAGER::downloadFile( const wxString& aFilePath, const wxString& url )
131 {
132  TRANSFER_CALLBACK callback = [&]( size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow )
133  {
134  if( dltotal > 1024 )
135  m_reporter->SetDownloadProgress( dlnow, dltotal );
136  else
137  m_reporter->SetDownloadProgress( 0.0, 0.0 );
138 
139  return m_reporter->IsCancelled();
140  };
141 
142  std::ofstream out( aFilePath.ToUTF8(), std::ofstream::binary );
143 
144  KICAD_CURL_EASY curl;
145  curl.SetOutputStream( &out );
146  curl.SetURL( url.ToUTF8().data() );
147  curl.SetFollowRedirects( true );
148  curl.SetTransferCallback( callback, 250000L );
149 
150  m_reporter->Report( wxString::Format( _( "Downloading package url: '%s'" ), url ) );
151 
152  int code = curl.Perform();
153 
154  out.close();
155 
156  uint64_t download_total;
157 
158  if( CURLE_OK == curl.GetTransferTotal( download_total ) )
159  m_reporter->SetDownloadProgress( download_total, download_total );
160 
161  if( code != CURLE_OK && code != CURLE_ABORTED_BY_CALLBACK )
162  {
163  m_reporter->Report( wxString::Format( _( "Failed to download url %s\n%s" ), url,
164  curl.GetErrorText( code ) ),
166  }
167 
168  return code;
169 }
170 
171 
172 bool PCM_TASK_MANAGER::extract( const wxString& aFilePath, const wxString& aPackageId,
173  bool isMultiThreaded )
174 {
175  wxFFileInputStream stream( aFilePath );
176  wxZipInputStream zip( stream );
177 
178  wxLogNull no_wx_logging;
179 
180  int entries = zip.GetTotalEntries();
181  int extracted = 0;
182 
183  wxArchiveEntry* entry = zip.GetNextEntry();
184 
185  if( !zip.IsOk() )
186  {
187  m_reporter->Report( _( "Error extracting file!" ), RPT_SEVERITY_ERROR );
188  return false;
189  }
190 
191  // Namespace delimiter changed on disk to allow flat loading of Python modules
192  wxString clean_package_id = aPackageId;
193  clean_package_id.Replace( '.', '_' );
194 
195  for( ; entry; entry = zip.GetNextEntry() )
196  {
197  wxArrayString path_parts =
198  wxSplit( entry->GetName(), wxFileName::GetPathSeparator(), (wxChar) NULL );
199 
200  if( path_parts.size() < 2
201  || PCM_PACKAGE_DIRECTORIES.find( path_parts[0] ) == PCM_PACKAGE_DIRECTORIES.end()
202  || path_parts[path_parts.size() - 1].IsEmpty() )
203  {
204  // Ignore files in the root of the archive and files outside of package dirs.
205  continue;
206  }
207 
208  // m_reporter->Report( wxString::Format( _( "Extracting file '%s'\n" ), entry->GetName() ),
209  // RPT_SEVERITY_INFO );
210 
211  // Transform paths from
212  // <PackageRoot>/$folder/$contents
213  // To
214  // $KICAD6_3RD_PARTY/$folder/$package_id/$contents
215  path_parts.Insert( clean_package_id, 1 );
216  path_parts.Insert( m_pcm->Get3rdPartyPath(), 0 );
217 
218  wxString fullname = wxJoin( path_parts, wxFileName::GetPathSeparator(), (wxChar) NULL );
219 
220  // Ensure the target directory exists and create it if not.
221  wxString t_path = wxPathOnly( fullname );
222 
223  if( !wxDirExists( t_path ) )
224  {
225  wxFileName::Mkdir( t_path, wxS_DIR_DEFAULT, wxPATH_MKDIR_FULL );
226  }
227 
228  wxTempFileOutputStream out( fullname );
229 
230  if( !( CopyStreamData( zip, out, entry->GetSize() ) && out.Commit() ) )
231  {
232  m_reporter->Report( _( "Error extracting file!" ), RPT_SEVERITY_ERROR );
233  return false;
234  }
235 
236  extracted++;
237  m_reporter->SetPackageProgress( extracted, entries );
238 
239  if( !isMultiThreaded )
240  m_reporter->KeepRefreshing( false );
241 
242  if( m_reporter->IsCancelled() )
243  break;
244  }
245 
246  zip.CloseEntry();
247 
248  if( m_reporter->IsCancelled() )
249  {
250  m_reporter->Report( _( "Aborting package installation." ) );
251  return false;
252  }
253 
254  m_reporter->Report( _( "Extracted package\n" ), RPT_SEVERITY_INFO );
255  m_reporter->SetPackageProgress( entries, entries );
256 
257  return true;
258 }
259 
260 
261 void PCM_TASK_MANAGER::InstallFromFile( wxWindow* aParent, const wxString& aFilePath )
262 {
263  wxFFileInputStream stream( aFilePath );
264 
265  if( !stream.IsOk() )
266  {
267  wxLogError( _( "Could not open archive file." ) );
268  return;
269  }
270 
271  wxZipInputStream zip( stream );
272 
273  if( !zip.IsOk() )
274  {
275  wxLogError( _( "Invalid archive file format." ) );
276  return;
277  }
278 
279  nlohmann::json metadata;
280 
281  for( wxArchiveEntry* entry = zip.GetNextEntry(); entry != nullptr; entry = zip.GetNextEntry() )
282  {
283  // Find and load metadata.json
284  if( entry->GetName() != "metadata.json" )
285  continue;
286 
287  wxStringOutputStream strStream;
288 
289  if( CopyStreamData( zip, strStream, entry->GetSize() ) )
290  {
291  try
292  {
293  metadata = nlohmann::json::parse( strStream.GetString().ToUTF8().data() );
294  m_pcm->ValidateJson( metadata );
295  }
296  catch( const std::exception& e )
297  {
298  wxLogError( wxString::Format( _( "Unable to parse package metadata:\n\n%s" ),
299  e.what() ) );
300  break;
301  }
302  }
303  }
304 
305  if( metadata.empty() )
306  {
307  wxLogError( _( "Archive does not contain a valid metadata.json file" ) );
308  return;
309  }
310 
311  PCM_PACKAGE package = metadata.get<PCM_PACKAGE>();
312 
313  if( package.versions.size() != 1 )
314  {
315  wxLogError( _( "Archive metadata must have a single version defined" ) );
316  return;
317  }
318 
319  const auto installed_packages = m_pcm->GetInstalledPackages();
320  if( std::find_if( installed_packages.begin(), installed_packages.end(),
321  [&]( const PCM_INSTALLATION_ENTRY& entry )
322  {
323  return entry.package.identifier == package.identifier;
324  } )
325  != installed_packages.end() )
326  {
327  wxLogError( wxString::Format( _( "Package with identifier %s is already installed, you "
328  "must first uninstall this package." ),
329  package.identifier ) );
330  return;
331  }
332 
333  m_reporter = std::make_unique<DIALOG_PCM_PROGRESS>( aParent, false );
334  m_reporter->Show();
335 
336  if( extract( aFilePath, package.identifier, false ) )
337  m_pcm->MarkInstalled( package, package.versions[0].version, "" );
338 
339  m_reporter->SetFinished();
340  m_reporter->KeepRefreshing( true );
341  m_reporter->Destroy();
342  m_reporter.reset();
343 }
344 
345 
346 void PCM_TASK_MANAGER::deletePackageDirectories( const wxString& aPackageId )
347 {
348  // Namespace delimiter changed on disk to allow flat loading of Python modules
349  wxString clean_package_id = aPackageId;
350  clean_package_id.Replace( '.', '_' );
351 
352  for( const wxString& dir : PCM_PACKAGE_DIRECTORIES )
353  {
354  wxFileName d( m_pcm->Get3rdPartyPath(), "" );
355  d.AppendDir( dir );
356  d.AppendDir( clean_package_id );
357 
358  if( d.DirExists() )
359  {
360  m_reporter->Report( wxString::Format( _( "Removing directory %s" ), d.GetPath() ) );
361 
362  if( !d.Rmdir( wxPATH_RMDIR_RECURSIVE ) )
363  {
364  m_reporter->Report(
365  wxString::Format( _( "Failed to remove directory %s" ), d.GetPath() ),
367  }
368  }
369  }
370 }
371 
372 
374 {
375  PCM_TASK task = [aPackage, this]
376  {
378 
379  m_pcm->MarkUninstalled( aPackage );
380 
381  m_reporter->Report( wxString::Format( _( "Package %s uninstalled" ),
382  aPackage.identifier ) );
383  };
384 
385  m_install_queue.push( task );
386 }
387 
388 
389 void PCM_TASK_MANAGER::RunQueue( wxWindow* aParent )
390 {
391  m_reporter = std::make_unique<DIALOG_PCM_PROGRESS>( aParent );
392 
393  m_reporter->SetNumPhases( m_download_queue.size() + m_install_queue.size() );
394  m_reporter->Show();
395 
396  wxSafeYield();
397 
398  std::mutex mutex;
399  std::condition_variable condvar;
400  bool download_complete = false;
401 
402  std::thread download_thread(
403  [&]()
404  {
405  while( !m_download_queue.empty() && !m_reporter->IsCancelled() )
406  {
407  PCM_TASK task;
408  m_download_queue.pop( task );
409  task();
410  condvar.notify_all();
411  }
412 
413  std::unique_lock<std::mutex> lock( mutex );
414  download_complete = true;
415  condvar.notify_all();
416  } );
417 
418  std::thread install_thread(
419  [&]()
420  {
421  std::unique_lock<std::mutex> lock( mutex );
422 
423  do
424  {
425  condvar.wait( lock,
426  [&]()
427  {
428  return download_complete || !m_install_queue.empty()
429  || m_reporter->IsCancelled();
430  } );
431 
432  lock.unlock();
433 
434  while( !m_install_queue.empty() && !m_reporter->IsCancelled() )
435  {
436  PCM_TASK task;
437  m_install_queue.pop( task );
438  task();
439  m_reporter->AdvancePhase();
440  }
441 
442  lock.lock();
443 
444  } while( ( !m_install_queue.empty() || !download_complete )
445  && !m_reporter->IsCancelled() );
446 
447  m_reporter->Report( _( "Done." ) );
448 
449  m_reporter->SetFinished();
450  } );
451 
452  m_reporter->KeepRefreshing( true );
453  m_reporter->Destroy();
454  m_reporter.reset();
455 
456  download_thread.join();
457  install_thread.join();
458 }
void push(T const &aValue)
Push a value onto the queue.
Definition: sync_queue.h:41
< Package version metadata Package metadata
Definition: pcm_data.h:73
bool pop(T &aReceiver)
Pop a value if the queue into the provided variable.
Definition: sync_queue.h:63
std::shared_ptr< PLUGIN_CONTENT_MANAGER > m_pcm
void Uninstall(const PCM_PACKAGE &aPackage)
Enqueue package uninstallation.
const std::unordered_set< wxString > PCM_PACKAGE_DIRECTORIES({ "plugins", "footprints", "3dmodels", "symbols", "resources", "colors", })
< Contains list of all valid directories that get extracted from a package archive
bool parse(std::istream &aStream, bool aVerbose)
Parse a PCB or footprint file from the given input stream.
nlohmann::json json
Definition: gerbview.cpp:41
SYNC_QUEUE< PCM_TASK > m_download_queue
static bool CopyStreamData(wxInputStream &inputStream, wxOutputStream &outputStream, wxFileOffset size)
void InstallFromFile(wxWindow *aParent, const wxString &aFilePath)
Installs package from an archive file on disk.
bool empty() const
Return true if the queue is empty.
Definition: sync_queue.h:82
std::unique_ptr< DIALOG_PCM_PROGRESS > m_reporter
void deletePackageDirectories(const wxString &aPackageId)
Delete all package files.
Repository reference to a resource.
Definition: pcm_data.h:93
std::function< int(size_t, size_t, size_t, size_t)> TRANSFER_CALLBACK
Wrapper interface around the curl_easy API/.
#define _(s)
std::function< void()> PCM_TASK
void DownloadAndInstall(const PCM_PACKAGE &aPackage, const wxString &aVersion, const wxString &aRepositoryId)
Enqueue package download and installation.
std::vector< PACKAGE_VERSION > versions
Definition: pcm_data.h:105
void Format(OUTPUTFORMATTER *out, int aNestLevel, int aCtl, const CPTREE &aTree)
Output a PTREE into s-expression format via an OUTPUTFORMATTER derivative.
Definition: ptree.cpp:200
size_t size() const
Return the size of the queue.
Definition: sync_queue.h:91
void RunQueue(wxWindow *aParent)
Run queue of pending actions.
wxString identifier
Definition: pcm_data.h:98
bool extract(const wxString &aFilePath, const wxString &aPackageId, bool isMultiThreaded)
Extract package archive.
int downloadFile(const wxString &aFilePath, const wxString &aUrl)
Download URL to a file.
SYNC_QUEUE< PCM_TASK > m_install_queue
bool SetOutputStream(const std::ostream *aOutput)
Definition: pcm_data.h:133