KiCad PCB EDA Suite
Loading...
Searching...
No Matches
jobs_runner.cpp
Go to the documentation of this file.
1/*
2 * This program source code file is part of KiCad, a free EDA CAD application.
3 *
4 * Copyright (C) 2024 Mark Roszko <[email protected]>
5 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
6 *
7 * This program is free software: you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation, either version 3 of the License, or (at your
10 * option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <https://www.gnu.org/licenses/>.
19 */
20
21#include <common.h>
22#include <cli/exit_codes.h>
23#include <jobs_runner.h>
24#include <jobs/job_registry.h>
25#include <jobs/jobset.h>
29#include <kiway.h>
30#include <kiway_mail.h>
31#include <paths.h>
33#include <reporter.h>
34#include <optional>
35#include <wx/process.h>
36#include <wx/txtstrm.h>
37#include <wx/sstream.h>
38#include <wx/wfstream.h>
39#include <wx/mstream.h>
40#include <wx/tokenzr.h>
41#include <gestfich.h>
42
43JOBS_RUNNER::JOBS_RUNNER( KIWAY* aKiway, JOBSET* aJobsFile, PROJECT* aProject,
44 REPORTER& aReporter, JOBS_PROGRESS_REPORTER* aProgressReporter ) :
45 m_kiway( aKiway ),
46 m_jobsFile( aJobsFile ),
47 m_reporter( aReporter ),
48 m_progressReporter( aProgressReporter ),
49 m_project( aProject )
50{
51}
52
53
55{
56 bool success = true;
57
58 for( JOBSET_DESTINATION& destination : m_jobsFile->GetDestinations() )
59 success &= RunJobsForDestination( &destination, aBail );
60
61 return success;
62}
63
64
65int JOBS_RUNNER::runSpecialExecute( const JOBSET_JOB* aJob, REPORTER* aReporter, PROJECT* aProject )
66{
67 JOB_SPECIAL_EXECUTE* specialJob = static_cast<JOB_SPECIAL_EXECUTE*>( aJob->m_job.get() );
68 wxString cmd = ExpandExecuteCommand( specialJob->m_command, m_project );
69
70 aReporter->Report( cmd, RPT_SEVERITY_INFO );
71 aReporter->Report( wxEmptyString, RPT_SEVERITY_INFO );
72
73 wxProcess process;
74 process.Redirect();
75
77
78 wxInputStream* inputStream = process.GetInputStream();
79 wxInputStream* errorStream = process.GetErrorStream();
80
81 // Reads wxInputStream into a wxMemoryBuffer
82 auto streamToBuf = []( wxInputStream& aIs )
83 {
84 wxMemoryOutputStream memOut;
85 aIs >> memOut;
86
87 wxMemoryBuffer buf;
88 buf.AppendData( memOut.GetOutputStreamBuffer()->GetBufferStart(),
89 memOut.GetOutputStreamBuffer()->GetIntPosition() );
90
91 return buf;
92 };
93
94 if( inputStream && errorStream )
95 {
96 wxMemoryBuffer memInBuf = streamToBuf( *inputStream );
97 wxMemoryBuffer memErrBuf = streamToBuf( *errorStream );
98
99 if( !memInBuf.IsEmpty() )
100 {
101 wxString str = wxString::FromUTF8( memInBuf, memInBuf.GetDataLen() );
102 wxStringTokenizer tokenizer( str, "\r\n" );
103
104 while( tokenizer.HasMoreTokens() )
105 aReporter->Report( tokenizer.GetNextToken(), RPT_SEVERITY_INFO );
106 }
107
108 if( !memErrBuf.IsEmpty() )
109 {
110 wxString str = wxString::FromUTF8( memErrBuf, memErrBuf.GetDataLen() );
111 wxStringTokenizer tokenizer( str, "\r\n" );
112
113 while( tokenizer.HasMoreTokens() )
114 aReporter->Report( tokenizer.GetNextToken(), RPT_SEVERITY_ERROR );
115 }
116
117 if( specialJob->m_recordOutput )
118 {
119 wxFileName defaultLog( aJob->m_id );
120 defaultLog.SetExt( wxT( "log" ) );
121
122 // The default name is not stored, a saved jobset must not carry it
123 // A folder gets the default log name inside it
124 wxString logPath =
125 specialJob->GetConfiguredOutputPath().IsEmpty()
126 ? specialJob->ResolveOutputPath( defaultLog.GetFullPath(), false, aProject )
127 : JOB::AppendDefaultFileName( specialJob->GetFullOutputPath( aProject ),
128 defaultLog.GetFullName() );
129
130 bool logWritten = false;
131
132 if( PATHS::EnsurePathExists( logPath, true ) )
133 {
134 wxFileOutputStream procOutput( logPath );
135
136 // A command that fails to launch reports on stderr only, leaving an empty log
137 logWritten = procOutput.IsOk() && procOutput.WriteAll( memInBuf, memInBuf.GetDataLen() )
138 && procOutput.WriteAll( memErrBuf, memErrBuf.GetDataLen() ) && procOutput.Close();
139 }
140
141 if( !logWritten )
142 {
143 wxString msg = wxString::Format( _( "Command finished but its output could not be written to '%s'" ),
144 logPath );
145 aReporter->Report( msg, RPT_SEVERITY_ERROR );
147 }
148 }
149 }
150
151 if( specialJob->m_ignoreExitcode )
152 return CLI::EXIT_CODES::OK;
153
155 {
156 aReporter->Report( wxString::Format( _( "Command exited with code %d" ), result ),
158 }
159
160 return result;
161}
162
163
165 std::vector<wxString>& aPathsWritten )
166{
167 wxString source = ResolveProjectPath( aJob->m_source, aProject );
168
169 if( source.IsEmpty() )
171
172 wxFileName sourceFn( source );
173
174 wxFileName destFn( aJob->GetFullOutputPath( aProject ) );
175
176 if( !aJob->m_dest.IsEmpty() )
177 destFn.AppendDir( ExpandPathVariables( aJob->m_dest, aProject ) );
178
179 wxString errors;
180 bool success = CopyFilesOrDirectory( sourceFn.GetFullPath(), destFn.GetFullPath(), aJob->m_overwriteDest,
181 errors, aPathsWritten );
182
183 if( !success )
185
186 if( aJob->m_generateErrorOnNoCopy && aPathsWritten.empty() )
188
189 return CLI::EXIT_CODES::OK;
190}
191
192
193int JOBS_RUNNER::runSpecialArchive( const JOBSET_JOB* aJob, REPORTER* aReporter, PROJECT* aProject )
194{
195 JOB_SPECIAL_ARCHIVE* archiveJob = static_cast<JOB_SPECIAL_ARCHIVE*>( aJob->m_job.get() );
196
197 if( archiveJob->GetConfiguredOutputPath().IsEmpty() )
198 archiveJob->SetConfiguredOutputPath( wxT( "${PROJECTNAME}.zip" ) );
199
200 wxString zipFile = archiveJob->GetFullOutputPath( aProject );
201
202 if( !PATHS::EnsurePathExists( zipFile, true ) )
203 {
204 aReporter->Report( _( "Failed to create output directory\n" ), RPT_SEVERITY_ERROR );
206 }
207
208 if( !PROJECT_ARCHIVER::Archive( aProject->GetProjectPath(), zipFile, *aReporter, true,
209 archiveJob->m_includeExtraFiles ) )
210 {
212 }
213
214 return CLI::EXIT_CODES::OK;
215}
216
217
219{
220 bool genOutputs = true;
221 bool success = true;
222 std::vector<JOBSET_JOB> jobsForDestination = m_jobsFile->GetJobsForDestination( aDestination );
223 wxString msg;
224
225 wxFileName tmp;
226 tmp.AssignDir( wxFileName::GetTempDir() );
227 tmp.AppendDir( KIID().AsString() );
228
229 aDestination->m_lastRunSuccessMap.clear();
230 aDestination->m_lastRunReporters.clear();
231 aDestination->m_lastResolvedOutputPath.reset();
232
233 wxString tempDirPath = tmp.GetFullPath();
234
235 if( !wxFileName::Mkdir( tempDirPath, wxS_DIR_DEFAULT ) )
236 {
237 msg = wxString::Format( wxT( "Failed to create temporary directory %s" ), tempDirPath );
238 m_reporter.Report( msg, RPT_SEVERITY_ERROR );
239
240 aDestination->m_lastRunSuccess = false;
241
242 return false;
243 }
244
245 bool continueOuput = aDestination->m_outputHandler->OutputPrecheck();
246
247 if( !continueOuput )
248 {
249 msg = wxString::Format( wxT( "Destination precheck failed for destination %s" ),
250 aDestination->m_id );
251 m_reporter.Report( msg, RPT_SEVERITY_ERROR );
252
253 aDestination->m_lastRunSuccess = false;
254 return false;
255 }
256
257 msg += wxT( "|--------------------------------\n" );
258 msg += wxT( "| " );
259 msg += wxString::Format( wxT( "Running jobs for destination %s" ), aDestination->m_id );
260 msg += wxT( "\n" );
261 msg += wxT( "|--------------------------------\n" );
262
263 msg += wxString::Format( wxT( "|%-5s | %-50s\n" ), wxT( "No." ), wxT( "Description" ) );
264
265 int jobNum = 1;
266
267 for( const JOBSET_JOB& job : jobsForDestination )
268 {
269 msg += wxString::Format( wxT( "|%-5d | %-50s\n" ), jobNum, job.GetDescription() );
270 jobNum++;
271 }
272
273 msg += wxT( "|--------------------------------\n" );
274 msg += wxT( "\n" );
275 msg += wxT( "\n" );
276
277 m_reporter.Report( msg, RPT_SEVERITY_INFO );
278
279 std::vector<wxString> pathsWithOverwriteDisallowed;
280 std::vector<JOB_OUTPUT> outputs;
281
282 jobNum = 1;
283 int failCount = 0;
284 int successCount = 0;
285
286 wxSetEnv( OUTPUT_TMP_PATH_VAR_NAME, tempDirPath );
287
288 for( const JOBSET_JOB& job : jobsForDestination )
289 {
290 msg = wxT( "|--------------------------------\n" );
291
292 msg += wxString::Format( wxT( "| Running job %d: %s" ), jobNum, job.GetDescription() );
293
294 msg += wxT( "\n" );
295 msg += wxT( "|--------------------------------\n" );
296
297 m_reporter.Report( msg, RPT_SEVERITY_INFO );
298
300 {
301 msg.Printf( _( "Running job %d: %s" ), jobNum, job.GetDescription() );
302 m_progressReporter->AdvanceJob( msg );
303 m_progressReporter->KeepRefreshing();
304 }
305
306 jobNum++;
307
308 KIWAY::FACE_T iface = JOB_REGISTRY::GetKifaceType( job.m_type );
309
310 job.m_job->SetTempOutputDirectory( tempDirPath );
311
312 REPORTER* targetReporter = &m_reporter;
313
314 if( targetReporter == &NULL_REPORTER::GetInstance() )
315 {
316 aDestination->m_lastRunReporters[job.m_id] =
317 std::make_shared<JOBSET_OUTPUT_REPORTER>( tempDirPath, m_progressReporter );
318
319 targetReporter = aDestination->m_lastRunReporters[job.m_id].get();
320 }
321
322 // Use a redirect reporter so we don't have error flags set after running previous jobs
323 REDIRECT_REPORTER isolatedReporter( targetReporter );
325
326 if( iface < KIWAY::KIWAY_FACE_COUNT )
327 {
328 result = m_kiway->ProcessJob( iface, job.m_job.get(), &isolatedReporter, m_progressReporter );
329 }
330 else
331 {
332 // special jobs
333 if( job.m_job->GetType() == "special_execute" )
334 {
335 result = runSpecialExecute( &job, &isolatedReporter, m_project );
336 }
337 else if( job.m_job->GetType() == "special_copyfiles" )
338 {
339 JOB_SPECIAL_COPYFILES* copyJob = static_cast<JOB_SPECIAL_COPYFILES*>( job.m_job.get() );
340 std::vector<wxString> pathsWritten;
341
342 result = runSpecialCopyFiles( copyJob, m_project, pathsWritten );
343
344 if( !copyJob->m_overwriteDest )
345 {
346 pathsWithOverwriteDisallowed.insert( pathsWithOverwriteDisallowed.end(), pathsWritten.begin(),
347 pathsWritten.end() );
348 }
349 }
350 else if( job.m_job->GetType() == "special_archive" )
351 {
352 result = runSpecialArchive( &job, &isolatedReporter, m_project );
353 }
354 else
355 {
356 msg = wxString::Format( wxT( "Unsupported job type '%s'" ), job.m_type );
357 isolatedReporter.Report( msg, RPT_SEVERITY_ERROR );
359 }
360 }
361
362 aDestination->m_lastRunSuccessMap[job.m_id] = ( result == CLI::EXIT_CODES::SUCCESS );
363
365 {
366 wxString msg_fmt = wxT( "\033[32;1m%s\033[0m\n" );
367 msg = wxString::Format( msg_fmt, _( "Job successful" ) );
368
369 successCount++;
370 }
371 else
372 {
373 wxString msg_fmt = wxT( "\033[31;1m%s\033[0m\n" );
374 msg = wxString::Format( msg_fmt, _( "Job failed" ) );
375
376 failCount++;
377 }
378
379 msg += wxT( "\n\n" );
380 m_reporter.Report( msg, RPT_SEVERITY_INFO );
381
383 {
384 success = false;
385
386 if( aBail )
387 break;
388 }
389 else if( result != CLI::EXIT_CODES::SUCCESS )
390 {
391 genOutputs = false;
392 success = false;
393
394 if( aBail )
395 break;
396 }
397 }
398
399 wxUnsetEnv( OUTPUT_TMP_PATH_VAR_NAME );
400
401 if( genOutputs )
402 {
403 success &= aDestination->m_outputHandler->HandleOutputs( tempDirPath, m_project, pathsWithOverwriteDisallowed,
404 outputs, aDestination->m_lastResolvedOutputPath );
405 }
406
407 aDestination->m_lastRunSuccess = success;
408
409 msg = wxString::Format( wxT( "\n\n\033[33;1m%d %s, %d %s\033[0m\n" ),
410 successCount,
411 wxT( "jobs succeeded" ),
412 failCount,
413 wxT( "job failed" ) );
414
415 m_reporter.Report( msg, RPT_SEVERITY_INFO );
416
417 return success;
418}
virtual bool OutputPrecheck()
Checks if the output process can proceed before doing anything else This can include user prompts.
Definition jobs_output.h:45
virtual bool HandleOutputs(const wxString &aBaseTempPath, PROJECT *aProject, const std::vector< wxString > &aPathsWithOverwriteDisallowed, const std::vector< JOB_OUTPUT > &aOutputsToHandle, std::optional< wxString > &aResolvedOutputPath)=0
bool RunJobsAllDestinations(bool aBail=false)
REPORTER & m_reporter
Definition jobs_runner.h:79
KIWAY * m_kiway
Definition jobs_runner.h:77
JOBSET * m_jobsFile
Definition jobs_runner.h:78
JOBS_PROGRESS_REPORTER * m_progressReporter
Definition jobs_runner.h:80
bool RunJobsForDestination(JOBSET_DESTINATION *aDestination, bool aBail=false)
int runSpecialCopyFiles(const JOB_SPECIAL_COPYFILES *aJob, PROJECT *aProject, std::vector< wxString > &aPathsWritten)
int runSpecialArchive(const JOBSET_JOB *aJob, REPORTER *aReporter, PROJECT *aProject)
PROJECT * m_project
Definition jobs_runner.h:81
int runSpecialExecute(const JOBSET_JOB *aJob, REPORTER *aReporter, PROJECT *aProject)
JOBS_RUNNER(KIWAY *aKiway, JOBSET *aJobsFile, PROJECT *aProject, REPORTER &aReporter, JOBS_PROGRESS_REPORTER *aProgressReporter)
static KIWAY::FACE_T GetKifaceType(const wxString &aName)
void SetConfiguredOutputPath(const wxString &aPath)
Sets the configured output path for the job, this path is always saved to file.
Definition job.cpp:181
wxString ResolveOutputPath(const wxString &aPath, bool aPathIsDirectory, PROJECT *aProject) const
Definition job.cpp:100
wxString GetFullOutputPath(PROJECT *aProject) const
Returns the full output path for the job, taking into account the configured output path,...
Definition job.cpp:174
static wxString AppendDefaultFileName(const wxString &aPath, const wxString &aDefaultName)
Complete a resolved file output path that names a folder, either by a trailing separator or by being ...
Definition job.cpp:152
wxString GetConfiguredOutputPath() const
Returns the configured output path for the job.
Definition job.h:235
Definition kiid.h:46
A minimalistic software bus for communications between various DLLs/DSOs (DSOs) within the same KiCad...
Definition kiway.h:348
FACE_T
Known KIFACE implementations.
Definition kiway.h:354
@ KIWAY_FACE_COUNT
Definition kiway.h:361
static REPORTER & GetInstance()
Definition reporter.cpp:207
static bool EnsurePathExists(const wxString &aPath, bool aPathToFile=false)
Attempts to create a given path if it does not exist.
Definition paths.cpp:470
static bool Archive(const wxString &aSrcDir, const wxString &aDestFile, REPORTER &aReporter, bool aVerbose=true, bool aIncludeExtraFiles=false)
Create an archive of the project.
Container for project specific data.
Definition project.h:63
virtual const wxString GetProjectPath() const
Return the full path of the project.
Definition project.cpp:277
REPORTER & Report(const wxString &aMsg, SEVERITY aSeverity=RPT_SEVERITY_UNDEFINED) override
Report a string with a given severity.
Definition reporter.cpp:401
A pure virtual class used to derive REPORTER objects from.
Definition reporter.h:73
virtual REPORTER & Report(const wxString &aText, SEVERITY aSeverity=RPT_SEVERITY_UNDEFINED)
Report a string with a given severity.
Definition reporter.h:102
wxString ExpandPathVariables(const wxString &aPath, const PROJECT *aProject, const std::function< bool(wxString *)> *aTextResolver)
Expand text variables, environment variables and a leading '~' in a user-entered file path.
Definition common.cpp:798
wxString ResolveProjectPath(const wxString &aPath, const PROJECT *aProject, const std::function< bool(wxString *)> *aTextResolver)
Expand a user-entered file path as ExpandPathVariables() does, then anchor a relative result at the p...
Definition common.cpp:817
#define _(s)
int ExecuteCommandThroughShell(const wxString &aCommand, wxProcess *aProcess)
Run a user-supplied command line through the platform shell so glob expansion, pipes,...
Definition gestfich.cpp:334
bool CopyFilesOrDirectory(const wxString &aSourcePath, const wxString &aDestDir, bool aAllowOverwrites, wxString &aErrors, std::vector< wxString > &aPathsWritten)
Definition gestfich.cpp:614
std::string source
wxString ExpandExecuteCommand(const wxString &aCommand, const PROJECT *aProject)
Expand an Execute Command string, including the built-in variables.
#define OUTPUT_TMP_PATH_VAR_NAME
static const int ERR_ARGS
Definition exit_codes.h:31
static const int OK
Definition exit_codes.h:30
static const int ERR_RC_VIOLATIONS
Rules check violation count was greater than 0.
Definition exit_codes.h:37
static const int SUCCESS
Definition exit_codes.h:29
static const int ERR_INVALID_OUTPUT_CONFLICT
Definition exit_codes.h:34
static const int ERR_UNKNOWN
Definition exit_codes.h:32
static PGM_BASE * process
@ RPT_SEVERITY_ERROR
@ RPT_SEVERITY_INFO
std::unordered_map< wxString, std::shared_ptr< JOBSET_OUTPUT_REPORTER > > m_lastRunReporters
Definition jobset.h:143
std::shared_ptr< JOBS_OUTPUT_HANDLER > m_outputHandler
Definition jobset.h:137
std::optional< wxString > m_lastResolvedOutputPath
Definition jobset.h:144
std::optional< bool > m_lastRunSuccess
Definition jobset.h:141
std::unordered_map< wxString, std::optional< bool > > m_lastRunSuccessMap
Definition jobset.h:142
wxString m_id
Definition jobset.h:134
wxString m_id
Definition jobset.h:88
std::shared_ptr< JOB > m_job
Definition jobset.h:91
wxString result
Test unit parsing edge cases and error handling.