KiCad PCB EDA Suite
Loading...
Searching...
No Matches
drc_benchmark.cpp
Go to the documentation of this file.
1/*
2 * This program source code file is part of KiCad, a free EDA CAD application.
3 *
4 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <https://www.gnu.org/licenses/>.
18 */
19
32
33#include <algorithm>
34#include <atomic>
35#include <chrono>
36#include <clocale>
37#include <cmath>
38#include <cstdio>
39#include <fstream>
40#include <map>
41#include <memory>
42#include <optional>
43#include <set>
44#include <sstream>
45#include <string>
46#include <thread>
47#include <vector>
48
49#include <wx/cmdline.h>
50#include <wx/filename.h>
51#include <wx/init.h>
52#include <wx/log.h>
53
54#include <core/profile.h>
55#include <pgm_base.h>
58#include <string_utils.h>
59#include <thread_pool.h>
61
62#include <board.h>
65#include <drc/drc_engine.h>
66#include <drc/drc_item.h>
68#include <pcb_marker.h>
69#include <project.h>
71#include <wx_log_utils.h>
72
74
75#include "corpus.h"
76#include "trace_capture.h"
77
78
79namespace
80{
81
87struct BENCH_PGM : public PGM_BASE
88{
89 void MacOpenFile( const wxString& aFileName ) override {}
90};
91
92
93BENCH_PGM g_program;
94
95
97enum class RULES_VARIANT
98{
99 NONE,
100 DEFAULT,
101 HEAVY
102};
103
104
106enum class CACHE_MODE
107{
108 COLD,
109 WARM
110};
111
112
113struct BENCH_CONFIG
114{
115 RULES_VARIANT rulesVariant = RULES_VARIANT::DEFAULT;
116 CACHE_MODE cache = CACHE_MODE::COLD;
117 int threads = 0;
118 int repeat = 5;
119};
120
121
123struct RUN_SAMPLE
124{
125 double compileMs = 0.0;
126 double checkMs = 0.0;
127 double cacheGenMs = 0.0;
128 int violations = 0;
129 std::map<wxString, double> providerMs;
130 bool timedOut = false;
131 double fraction = 1.0;
132};
133
134
136struct STAT
137{
138 double median = 0.0;
139 double mad = 0.0;
140};
141
142
143STAT computeStat( std::vector<double> aValues )
144{
145 STAT stat;
146
147 if( aValues.empty() )
148 return stat;
149
150 std::sort( aValues.begin(), aValues.end() );
151
152 size_t n = aValues.size();
153 stat.median = ( n % 2 ) ? aValues[n / 2] : 0.5 * ( aValues[n / 2 - 1] + aValues[n / 2] );
154
155 std::vector<double> devs;
156 devs.reserve( n );
157
158 for( double v : aValues )
159 devs.push_back( std::fabs( v - stat.median ) );
160
161 std::sort( devs.begin(), devs.end() );
162 stat.mad = ( n % 2 ) ? devs[n / 2] : 0.5 * ( devs[n / 2 - 1] + devs[n / 2] );
163
164 return stat;
165}
166
167
173double readOneMinuteLoad()
174{
175 std::ifstream in( "/proc/loadavg" );
176
177 if( !in.is_open() )
178 return -1.0;
179
180 double load = -1.0;
181 in >> load;
182
183 return load;
184}
185
186
194void applyThreadConfig( int aThreads )
195{
196 size_t n = aThreads > 0 ? static_cast<size_t>( aThreads ) : 0;
197
198 Pgm().GetThreadPool().reset( n );
199
200 // The cached GetKiCadThreadPool() pointer still aims at the same PGM-owned pool
201 // object, but invalidating it keeps us honest with the documented contract and
202 // forces a fresh fetch on the next provider call.
204}
205
206
208wxFileName resolveRules( const wxFileName& aBoardName, const std::optional<wxString>& aDefaultRules,
209 const std::optional<wxString>& aHeavyRules, RULES_VARIANT aVariant )
210{
211 if( aVariant == RULES_VARIANT::NONE )
212 return wxFileName();
213
214 if( aVariant == RULES_VARIANT::HEAVY )
215 {
216 if( aHeavyRules )
217 return wxFileName( *aHeavyRules );
218
219 return wxFileName();
220 }
221
222 if( aDefaultRules )
223 return wxFileName( *aDefaultRules );
224
225 wxFileName sidecar( aBoardName );
226 sidecar.SetExt( FILEEXT::DesignRulesFileExtension );
227
228 if( sidecar.Exists() )
229 return sidecar;
230
231 return wxFileName();
232}
233
234
240std::unique_ptr<BOARD> loadBoard( const wxFileName& aBoardName, SETTINGS_MANAGER& aManager,
241 const wxFileName& aProjectName )
242{
243 std::unique_ptr<BOARD> board;
244
245 try
246 {
248 std::string( aBoardName.GetFullPath().ToUTF8() ) );
249 }
250 catch( const IO_ERROR& ioe )
251 {
252 std::printf( "error loading board: %s\n", TO_UTF8( ioe.What() ) );
253 return nullptr;
254 }
255
256 if( !board )
257 {
258 std::printf( "error: board failed to load\n" );
259 return nullptr;
260 }
261
262 if( aProjectName.Exists() )
263 board->SetProject( &aManager.Prj() );
264
265 board->BuildListOfNets();
266 board->BuildConnectivity();
267 board->GetLengthCalculation()->SynchronizeTuningProfileProperties();
268
269 if( board->GetProject() )
270 {
271 std::unordered_set<wxString> dummy;
272 board->SynchronizeComponentClasses( dummy );
273 }
274
275 return board;
276}
277
278
286const std::map<wxString, std::vector<int>>& providerErrorCodes()
287{
288 static const std::map<wxString, std::vector<int>> codes = {
289 { wxT( "annular_width" ), { DRCE_ANNULAR_WIDTH } },
290 { wxT( "copper width" ), { DRCE_CONNECTION_WIDTH } },
291 { wxT( "connectivity" ), { DRCE_DANGLING_TRACK, DRCE_DANGLING_VIA,
296 { wxT( "clearance" ), { DRCE_CLEARANCE, DRCE_HOLE_CLEARANCE,
298 { wxT( "courtyard_clearance" ), { DRCE_MALFORMED_COURTYARD, DRCE_MISSING_COURTYARD,
301 { wxT( "creepage" ), { DRCE_CREEPAGE } },
302 { wxT( "diff_pair_coupling" ), { DRCE_DP_GAP_OUT_OF_RANGE, DRCE_DP_UNCOUPLED_LENGTH_TOO_LONG } },
303 { wxT( "disallow" ), { DRCE_ALLOWED_ITEMS, DRCE_TEXT_ON_EDGECUTS } },
304 { wxT( "edge_clearance" ), { DRCE_EDGE_CLEARANCE, DRCE_SILK_EDGE_CLEARANCE } },
305 { wxT( "footprint checks" ), { DRCE_FOOTPRINT_TYPE_MISMATCH, DRCE_PADSTACK,
308 DRCE_PADSTACK } },
309 { wxT( "hole_to_hole_clearance" ), { DRCE_DRILLED_HOLES_COLOCATED, DRCE_DRILLED_HOLES_TOO_CLOSE } },
313 { wxT( "line_width" ), { DRCE_LINE_WIDTH } },
314 { wxT( "physical_clearance" ), { DRCE_CLEARANCE, DRCE_HOLE_CLEARANCE } },
315 { wxT( "silk_clearance" ), { DRCE_SILK_CLEARANCE, DRCE_SILK_MASK_CLEARANCE } },
316 { wxT( "sliver checker" ), { DRCE_COPPER_SLIVER } },
317 { wxT( "solder_mask_issues" ), { DRCE_SILK_MASK_CLEARANCE, DRCE_SOLDERMASK_BRIDGE } },
318 { wxT( "text_dimensions" ), { DRCE_TEXT_HEIGHT, DRCE_LINE_WIDTH } },
320 { wxT( "angle" ), { DRCE_TRACK_ANGLE } },
321 { wxT( "segment_length" ), { DRCE_TRACK_SEGMENT_LENGTH } },
322 { wxT( "width" ), { DRCE_TRACK_WIDTH } },
323 { wxT( "diameter" ), { DRCE_VIA_DIAMETER } },
324 { wxT( "zone connections" ), { DRCE_STARVED_THERMAL } }
325 };
326
327 return codes;
328}
329
330
336bool applyIsolate( BOARD* aBoard, const wxString& aProvider )
337{
338 auto it = providerErrorCodes().find( aProvider );
339
340 if( it == providerErrorCodes().end() )
341 return false;
342
343 std::set<int> keep( it->second.begin(), it->second.end() );
344
346
347 for( int code = DRCE_FIRST; code <= DRCE_LAST; ++code )
348 {
349 if( !keep.count( code ) )
351 }
352
353 return true;
354}
355
356
358
367class BENCH_PROGRESS : public PROGRESS_REPORTER_BASE
368{
369public:
370 explicit BENCH_PROGRESS( double aTimeoutSec ) :
371 PROGRESS_REPORTER_BASE( 1 ),
372 m_enabled( aTimeoutSec > 0.0 ),
373 m_deadline( std::chrono::steady_clock::now()
374 + std::chrono::duration_cast<std::chrono::steady_clock::duration>(
375 std::chrono::duration<double>( aTimeoutSec ) ) )
376 {
377 }
378
379 bool TimedOut() const { return m_timedOut.load(); }
380
381protected:
382 bool updateUI() override
383 {
384 if( m_enabled && std::chrono::steady_clock::now() >= m_deadline )
385 {
386 m_timedOut.store( true );
387 m_cancelled.store( true );
388
389 return false;
390 }
391
392 return true;
393 }
394
395private:
396 bool m_enabled;
397 std::chrono::steady_clock::time_point m_deadline;
398 std::atomic_bool m_timedOut{ false };
399};
400
401
402double timeCompile( BOARD* aBoard, const wxFileName& aRulesFile )
403{
404 std::shared_ptr<DRC_ENGINE> engine =
405 std::make_shared<DRC_ENGINE>( aBoard, &aBoard->GetDesignSettings() );
406
407 aBoard->GetDesignSettings().m_DRCEngine = engine;
408
409 PROF_TIMER timer;
410 engine->InitEngine( aRulesFile );
411 timer.Stop();
412
413 return timer.msecs();
414}
415
416
421RUN_SAMPLE timeRun( BOARD* aBoard, const wxFileName& aRulesFile, double aTimeoutSec )
422{
423 RUN_SAMPLE sample;
424
425 std::shared_ptr<DRC_ENGINE> engine =
426 std::make_shared<DRC_ENGINE>( aBoard, &aBoard->GetDesignSettings() );
427
428 aBoard->GetDesignSettings().m_DRCEngine = engine;
429
430 std::atomic<int> violationCount{ 0 };
431
432 engine->SetViolationHandler(
433 [&]( const std::shared_ptr<DRC_ITEM>& aItem, const VECTOR2I& aPos, int aLayer,
434 const std::function<void( PCB_MARKER* )>& aCreateMarker )
435 {
436 violationCount.fetch_add( 1, std::memory_order_relaxed );
437 } );
438
439 PROF_TIMER compileTimer;
440 engine->InitEngine( aRulesFile );
441 compileTimer.Stop();
442 sample.compileMs = compileTimer.msecs();
443
444 // Total provider count anchors the "% complete" reported on a timeout. The profile log
445 // records one entry per provider that finished, so completed/total is how far the check got.
446 size_t totalProviders = engine->GetTestProviders().size();
447
448 // Install the profile-trace scraper for the duration of RunTests so the engine's
449 // per-provider "DRC provider ... took" lines land in our maps. The previous target is
450 // chained, so ordinary logging still reaches the console.
451 wxLog::AddTraceMask( wxT( "KICAD_DRC_PROFILE" ) );
452
453 // Deadline starts at the check phase so a slow rule compile spends its own time without
454 // eating the evaluator's budget. InitEngine is not cancellable, so the timeout only bounds
455 // RunTests, which is the long pole this tool exists to measure.
456 BENCH_PROGRESS progress( aTimeoutSec );
457 engine->SetProgressReporter( &progress );
458
459 DRC_PROFILE_LOG profileLog;
460 std::chrono::milliseconds runDuration{ 0 };
461 {
462 SCOPED_WXLOG_TARGET logOverride( &profileLog );
463
464 try
465 {
466 SCOPED_PROF_TIMER scopedTimer( runDuration );
467 engine->RunTests( EDA_UNITS::MM, true, false );
468 }
469 catch( const std::exception& e )
470 {
471 std::printf( "error during RunTests: %s\n", e.what() );
472 }
473 }
474
475 engine->SetProgressReporter( nullptr );
476
477 sample.providerMs = profileLog.ProviderMs();
478
479 sample.timedOut = progress.TimedOut();
480
481 if( sample.timedOut )
482 sample.fraction = totalProviders > 0
483 ? static_cast<double>( sample.providerMs.size() )
484 / static_cast<double>( totalProviders )
485 : 0.0;
486
487 // Prefer the engine's own "DRC took" total as the check denominator since it brackets
488 // the same span the provider rows live in. Fall back to our outer timer if the trace
489 // line did not arrive (e.g. RunTests bailed early).
490 double engineTotal = profileLog.TotalMs();
491
492 sample.checkMs = engineTotal > 0.0 ? engineTotal : static_cast<double>( runDuration.count() );
493
494 double providerSum = 0.0;
495
496 for( const auto& [name, ms] : sample.providerMs )
497 providerSum += ms;
498
499 // Cache generation has no trace line of its own; it is everything the check spent that
500 // is not attributable to a provider. Clamp at zero to absorb timer jitter.
501 sample.cacheGenMs = std::max( 0.0, sample.checkMs - providerSum );
502
503 sample.violations = violationCount.load( std::memory_order_relaxed );
504
505 return sample;
506}
507
508
510struct SWEEP_RESULT
511{
512 BENCH_CONFIG config;
513 bool ran = false;
514 bool underLoad = false;
515 STAT compile;
516 STAT check;
517 STAT cacheGen;
518 int violations = 0;
519 bool timedOut = false;
520 double fraction = 1.0;
521 std::map<wxString, STAT> providerStats;
522};
523
524
533SWEEP_RESULT runConfig( const wxFileName& aBoardName, SETTINGS_MANAGER& aManager,
534 const wxFileName& aProjectName, const wxFileName& aRulesFile,
535 const BENCH_CONFIG& aConfig, const std::optional<wxString>& aIsolate,
536 double aMaxLoad, double aTimeoutSec )
537{
538 SWEEP_RESULT result;
539 result.config = aConfig;
540
541 applyThreadConfig( aConfig.threads );
542
543 std::vector<double> compileSamples;
544 std::vector<double> checkSamples;
545 std::vector<double> cacheSamples;
546 std::map<wxString, std::vector<double>> providerSamples;
547
548 std::unique_ptr<BOARD> warmBoard;
549
550 if( aConfig.cache == CACHE_MODE::WARM )
551 {
552 warmBoard = loadBoard( aBoardName, aManager, aProjectName );
553
554 if( !warmBoard )
555 return result;
556
557 if( aIsolate )
558 applyIsolate( warmBoard.get(), *aIsolate );
559 }
560
561 for( int i = 0; i <= aConfig.repeat; ++i )
562 {
563 BOARD* board = warmBoard.get();
564
565 std::unique_ptr<BOARD> coldBoard;
566
567 if( aConfig.cache == CACHE_MODE::COLD )
568 {
569 coldBoard = loadBoard( aBoardName, aManager, aProjectName );
570
571 if( !coldBoard )
572 return result;
573
574 if( aIsolate )
575 applyIsolate( coldBoard.get(), *aIsolate );
576
577 board = coldBoard.get();
578 }
579
580 // Re-check load right before a timed pass so a spike that arrives mid-sweep taints
581 // only the rows it actually touched instead of silently corrupting the medians.
582 if( aMaxLoad > 0.0 && i > 0 )
583 {
584 double load = readOneMinuteLoad();
585
586 if( load > aMaxLoad )
587 result.underLoad = true;
588 }
589
590 RUN_SAMPLE sample = timeRun( board, aRulesFile, aTimeoutSec );
591
592 // A timeout means this cell is too slow to be worth repeating; bail immediately so a
593 // single deadline does not cost timeout x repeat. Keep the compile number (it finished
594 // before the check deadline) so the row still reports useful compiler data.
595 if( sample.timedOut )
596 {
597 result.timedOut = true;
598 result.fraction = sample.fraction;
599
600 if( compileSamples.empty() )
601 compileSamples.push_back( sample.compileMs );
602
603 break;
604 }
605
606 // The first pass primes allocators and any lazily built board state, so discard it.
607 if( i > 0 )
608 {
609 compileSamples.push_back( sample.compileMs );
610 checkSamples.push_back( sample.checkMs );
611 cacheSamples.push_back( sample.cacheGenMs );
612 result.violations = sample.violations;
613
614 for( const auto& [name, ms] : sample.providerMs )
615 providerSamples[name].push_back( ms );
616 }
617 }
618
619 if( warmBoard )
620 warmBoard->SetProject( nullptr );
621
622 result.ran = true;
623 result.compile = computeStat( compileSamples );
624 result.check = computeStat( checkSamples );
625 result.cacheGen = computeStat( cacheSamples );
626
627 for( auto& [name, samples] : providerSamples )
628 result.providerStats[name] = computeStat( samples );
629
630 return result;
631}
632
633
641STAT runCompileOnly( const wxFileName& aBoardName, SETTINGS_MANAGER& aManager,
642 const wxFileName& aProjectName, const wxFileName& aRulesFile, int aRepeat )
643{
644 std::unique_ptr<BOARD> board = loadBoard( aBoardName, aManager, aProjectName );
645
646 if( !board )
647 return STAT();
648
649 std::vector<double> samples;
650
651 for( int i = 0; i <= aRepeat; ++i )
652 {
653 double ms = timeCompile( board.get(), aRulesFile );
654
655 if( i > 0 )
656 samples.push_back( ms );
657 }
658
659 board->SetProject( nullptr );
660
661 return computeStat( samples );
662}
663
664
665const char* variantName( RULES_VARIANT aVariant )
666{
667 switch( aVariant )
668 {
669 case RULES_VARIANT::NONE: return "none";
670 case RULES_VARIANT::DEFAULT: return "default";
671 case RULES_VARIANT::HEAVY: return "heavy";
672 }
673
674 return "?";
675}
676
677
678bool parseVariant( const wxString& aArg, RULES_VARIANT& aVariant )
679{
680 if( aArg == wxT( "none" ) )
681 aVariant = RULES_VARIANT::NONE;
682 else if( aArg == wxT( "default" ) )
683 aVariant = RULES_VARIANT::DEFAULT;
684 else if( aArg == wxT( "heavy" ) )
685 aVariant = RULES_VARIANT::HEAVY;
686 else
687 return false;
688
689 return true;
690}
691
692
694wxString slurp( const wxFileName& aFile )
695{
696 std::ifstream in( aFile.GetFullPath().fn_str() );
697
698 if( !in.is_open() )
699 return wxEmptyString;
700
701 std::stringstream buffer;
702 buffer << in.rdbuf();
703
704 return wxString::FromUTF8( buffer.str().c_str() );
705}
706
707
709std::string jsonEscape( const wxString& aStr )
710{
711 std::string utf8( aStr.utf8_str() );
712 std::string out;
713 out.reserve( utf8.size() + 8 );
714
715 for( char c : utf8 )
716 {
717 switch( c )
718 {
719 case '"': out += "\\\""; break;
720 case '\\': out += "\\\\"; break;
721 case '\n': out += "\\n"; break;
722 case '\r': out += "\\r"; break;
723 case '\t': out += "\\t"; break;
724 default: out += c; break;
725 }
726 }
727
728 return out;
729}
730
731
733struct COVERAGE_ROW
734{
735 wxString board;
736 std::set<DRC_CONSTRAINT_T> constraints;
737 std::set<wxString> predicates;
738};
739
740
750COVERAGE_ROW collectCoverage( const wxFileName& aBoardName, SETTINGS_MANAGER& aManager,
751 const wxFileName& aProjectName, const wxFileName& aRulesFile )
752{
753 COVERAGE_ROW row;
754 row.board = aBoardName.GetFullName();
755
756 std::unique_ptr<BOARD> board = loadBoard( aBoardName, aManager, aProjectName );
757
758 if( !board )
759 return row;
760
761 std::shared_ptr<DRC_ENGINE> engine =
762 std::make_shared<DRC_ENGINE>( board.get(), &board->GetDesignSettings() );
763
764 board->GetDesignSettings().m_DRCEngine = engine;
765 engine->InitEngine( aRulesFile );
766
768 {
769 if( engine->HasRulesForConstraintType( type ) )
770 row.constraints.insert( type );
771 }
772
773 if( aRulesFile.IsOk() && aRulesFile.Exists() )
774 {
775 for( const wxString& pred : ScanPredicatesInRules( slurp( aRulesFile ) ) )
776 row.predicates.insert( pred );
777 }
778
779 board->SetProject( nullptr );
780
781 return row;
782}
783
784
789void emitCoverage( const std::vector<COVERAGE_ROW>& aRows, const wxString& aOutDir )
790{
791 std::set<DRC_CONSTRAINT_T> coveredConstraints;
792 std::set<wxString> coveredPredicates;
793
794 for( const COVERAGE_ROW& row : aRows )
795 {
796 coveredConstraints.insert( row.constraints.begin(), row.constraints.end() );
797 coveredPredicates.insert( row.predicates.begin(), row.predicates.end() );
798 }
799
800 std::printf( "=== coverage matrix ===\n" );
801 std::printf( "%-28s %s\n", "board", "constraints / predicates" );
802 std::printf( "%-28s %s\n", "----------------------------",
803 "-------------------------------------" );
804
805 for( const COVERAGE_ROW& row : aRows )
806 {
807 std::string cons;
808
809 for( DRC_CONSTRAINT_T type : row.constraints )
810 {
811 if( !cons.empty() )
812 cons += ",";
813
814 cons += ConstraintTypeName( type );
815 }
816
817 std::string preds;
818
819 for( const wxString& pred : row.predicates )
820 {
821 if( !preds.empty() )
822 preds += ",";
823
824 preds += std::string( pred.utf8_str() );
825 }
826
827 std::printf( "%-28s C[%s] P[%s]\n",
828 static_cast<const char*>( row.board.utf8_str() ), cons.c_str(),
829 preds.c_str() );
830 }
831
832 std::vector<const char*> uncoveredConstraints;
833
835 {
836 if( !coveredConstraints.count( type ) )
837 uncoveredConstraints.push_back( ConstraintTypeName( type ) );
838 }
839
840 std::vector<wxString> uncoveredPredicates;
841
842 for( const wxString& pred : AllPredicateNames() )
843 {
844 if( !coveredPredicates.count( pred ) )
845 uncoveredPredicates.push_back( pred );
846 }
847
848 std::printf( "\nUNCOVERED constraints:" );
849
850 for( const char* name : uncoveredConstraints )
851 std::printf( " %s", name );
852
853 std::printf( "%s\n", uncoveredConstraints.empty() ? " (none)" : "" );
854
855 std::printf( "UNCOVERED predicates:" );
856
857 for( const wxString& pred : uncoveredPredicates )
858 std::printf( " %s", static_cast<const char*>( pred.utf8_str() ) );
859
860 std::printf( "%s\n\n", uncoveredPredicates.empty() ? " (none)" : "" );
861
862 wxFileName outFile( aOutDir, wxT( "coverage.json" ) );
863 std::ofstream out( outFile.GetFullPath().fn_str() );
864
865 if( !out.is_open() )
866 return;
867
868 out << "{\n \"boards\": [\n";
869
870 for( size_t i = 0; i < aRows.size(); ++i )
871 {
872 const COVERAGE_ROW& row = aRows[i];
873
874 out << " {\n \"board\": \"" << jsonEscape( row.board ) << "\",\n";
875 out << " \"constraints\": [";
876
877 bool first = true;
878
879 for( DRC_CONSTRAINT_T type : row.constraints )
880 {
881 out << ( first ? "" : ", " ) << "\"" << ConstraintTypeName( type ) << "\"";
882 first = false;
883 }
884
885 out << "],\n \"predicates\": [";
886
887 first = true;
888
889 for( const wxString& pred : row.predicates )
890 {
891 out << ( first ? "" : ", " ) << "\"" << jsonEscape( pred ) << "\"";
892 first = false;
893 }
894
895 out << "]\n }" << ( i + 1 < aRows.size() ? "," : "" ) << "\n";
896 }
897
898 out << " ],\n \"uncovered_constraints\": [";
899
900 for( size_t i = 0; i < uncoveredConstraints.size(); ++i )
901 out << ( i ? ", " : "" ) << "\"" << uncoveredConstraints[i] << "\"";
902
903 out << "],\n \"uncovered_predicates\": [";
904
905 for( size_t i = 0; i < uncoveredPredicates.size(); ++i )
906 out << ( i ? ", " : "" ) << "\"" << jsonEscape( uncoveredPredicates[i] ) << "\"";
907
908 out << "]\n}\n";
909}
910
911
913struct RESULT_ROW
914{
915 wxString board;
916 wxString config;
917 STAT compile;
918 STAT cacheGen;
919 STAT check;
920 double evalOverheadMs = 0.0;
921 bool evalOverheadValid = false;
922 int violations = 0;
923 bool underLoad = false;
924 bool timedOut = false;
925 double fraction = 1.0;
926 std::map<wxString, STAT> perProvider;
927};
928
929
930void writeResultsJson( const std::vector<RESULT_ROW>& aRows, const wxString& aOutDir )
931{
932 wxFileName outFile( aOutDir, wxT( "results.json" ) );
933 std::ofstream out( outFile.GetFullPath().fn_str() );
934
935 if( !out.is_open() )
936 return;
937
938 out << "[\n";
939
940 for( size_t i = 0; i < aRows.size(); ++i )
941 {
942 const RESULT_ROW& row = aRows[i];
943
944 out << " {\n";
945 out << " \"board\": \"" << jsonEscape( row.board ) << "\",\n";
946 out << " \"config\": \"" << jsonEscape( row.config ) << "\",\n";
947 out << " \"compile_ms\": " << row.compile.median << ",\n";
948 out << " \"compile_mad\": " << row.compile.mad << ",\n";
949 out << " \"cache_gen_ms\": " << row.cacheGen.median << ",\n";
950 out << " \"cache_gen_mad\": " << row.cacheGen.mad << ",\n";
951 out << " \"check_ms\": " << row.check.median << ",\n";
952 out << " \"check_mad\": " << row.check.mad << ",\n";
953
954 if( row.evalOverheadValid )
955 out << " \"eval_overhead_ms\": " << row.evalOverheadMs << ",\n";
956 else
957 out << " \"eval_overhead_ms\": null,\n";
958
959 out << " \"n_violations\": " << row.violations << ",\n";
960 out << " \"under_load\": " << ( row.underLoad ? "true" : "false" ) << ",\n";
961 out << " \"timed_out\": " << ( row.timedOut ? "true" : "false" ) << ",\n";
962 out << " \"percent_complete\": " << ( row.timedOut ? row.fraction * 100.0 : 100.0 )
963 << ",\n";
964 out << " \"per_provider\": {";
965
966 bool first = true;
967
968 for( const auto& [name, stat] : row.perProvider )
969 {
970 out << ( first ? "\n" : ",\n" );
971 out << " \"" << jsonEscape( name ) << "\": { \"median_ms\": " << stat.median
972 << ", \"mad_ms\": " << stat.mad << " }";
973 first = false;
974 }
975
976 out << ( first ? "}" : "\n }" ) << "\n";
977 out << " }" << ( i + 1 < aRows.size() ? "," : "" ) << "\n";
978 }
979
980 out << "]\n";
981}
982
983
989void writeWorstOffenders( const std::vector<RESULT_ROW>& aRows, const wxString& aOutDir, int aTopN )
990{
991 std::vector<const RESULT_ROW*> byCompile;
992 std::vector<const RESULT_ROW*> byEval;
993 std::vector<const RESULT_ROW*> timedOut;
994
995 for( const RESULT_ROW& row : aRows )
996 {
997 byCompile.push_back( &row );
998
999 if( row.evalOverheadValid )
1000 byEval.push_back( &row );
1001
1002 if( row.timedOut )
1003 timedOut.push_back( &row );
1004 }
1005
1006 std::sort( byCompile.begin(), byCompile.end(),
1007 []( const RESULT_ROW* a, const RESULT_ROW* b )
1008 {
1009 return a->compile.median > b->compile.median;
1010 } );
1011
1012 std::sort( byEval.begin(), byEval.end(),
1013 []( const RESULT_ROW* a, const RESULT_ROW* b )
1014 {
1015 return a->evalOverheadMs > b->evalOverheadMs;
1016 } );
1017
1018 // Least-complete first: a cell that only reached 10% before the deadline is a worse
1019 // offender than one that reached 90%.
1020 std::sort( timedOut.begin(), timedOut.end(),
1021 []( const RESULT_ROW* a, const RESULT_ROW* b )
1022 {
1023 return a->fraction < b->fraction;
1024 } );
1025
1026 auto emitList = [&]( const char* aLabel )
1027 {
1028 std::printf( "=== worst offenders by %s ===\n", aLabel );
1029 std::printf( "%-28s %-22s %12s\n", "board", "config", aLabel );
1030 std::printf( "%-28s %-22s %12s\n", "----------------------------",
1031 "----------------------", "------------" );
1032 };
1033
1034 emitList( "compile_ms" );
1035
1036 for( int i = 0; i < aTopN && i < static_cast<int>( byCompile.size() ); ++i )
1037 {
1038 const RESULT_ROW* row = byCompile[i];
1039
1040 std::printf( "%-28s %-22s %12.3f\n", static_cast<const char*>( row->board.utf8_str() ),
1041 static_cast<const char*>( row->config.utf8_str() ), row->compile.median );
1042 }
1043
1044 std::printf( "\n" );
1045 emitList( "eval_overhead_ms" );
1046
1047 for( int i = 0; i < aTopN && i < static_cast<int>( byEval.size() ); ++i )
1048 {
1049 const RESULT_ROW* row = byEval[i];
1050
1051 std::printf( "%-28s %-22s %12.3f\n", static_cast<const char*>( row->board.utf8_str() ),
1052 static_cast<const char*>( row->config.utf8_str() ), row->evalOverheadMs );
1053 }
1054
1055 std::printf( "\n" );
1056
1057 if( !timedOut.empty() )
1058 {
1059 std::printf( "=== timed out (eval unbounded; ranked least-complete first) ===\n" );
1060 std::printf( "%-28s %-22s %12s\n", "board", "config", "percent" );
1061 std::printf( "%-28s %-22s %12s\n", "----------------------------",
1062 "----------------------", "------------" );
1063
1064 for( const RESULT_ROW* row : timedOut )
1065 {
1066 std::printf( "%-28s %-22s %11.1f%%\n",
1067 static_cast<const char*>( row->board.utf8_str() ),
1068 static_cast<const char*>( row->config.utf8_str() ), row->fraction * 100.0 );
1069 }
1070
1071 std::printf( "\n" );
1072 }
1073
1074 wxFileName outFile( aOutDir, wxT( "worst_offenders.json" ) );
1075 std::ofstream out( outFile.GetFullPath().fn_str() );
1076
1077 if( !out.is_open() )
1078 return;
1079
1080 auto writeRanked = [&]( const char* aKey, const std::vector<const RESULT_ROW*>& aList,
1081 bool aUseEval )
1082 {
1083 out << " \"" << aKey << "\": [\n";
1084
1085 int count = std::min<int>( aTopN, static_cast<int>( aList.size() ) );
1086
1087 for( int i = 0; i < count; ++i )
1088 {
1089 const RESULT_ROW* row = aList[i];
1090 double value = aUseEval ? row->evalOverheadMs : row->compile.median;
1091
1092 out << " { \"board\": \"" << jsonEscape( row->board ) << "\", \"config\": \""
1093 << jsonEscape( row->config ) << "\", \"" << ( aUseEval ? "eval_overhead_ms"
1094 : "compile_ms" )
1095 << "\": " << value << " }" << ( i + 1 < count ? "," : "" ) << "\n";
1096 }
1097
1098 out << " ]";
1099 };
1100
1101 out << "{\n";
1102 writeRanked( "by_compile_ms", byCompile, false );
1103 out << ",\n";
1104 writeRanked( "by_eval_overhead_ms", byEval, true );
1105 out << ",\n";
1106
1107 out << " \"timed_out\": [\n";
1108
1109 for( size_t i = 0; i < timedOut.size(); ++i )
1110 {
1111 const RESULT_ROW* row = timedOut[i];
1112
1113 out << " { \"board\": \"" << jsonEscape( row->board ) << "\", \"config\": \""
1114 << jsonEscape( row->config ) << "\", \"percent_complete\": " << row->fraction * 100.0
1115 << " }" << ( i + 1 < timedOut.size() ? "," : "" ) << "\n";
1116 }
1117
1118 out << " ]\n}\n";
1119}
1120
1121
1123wxString configTag( CACHE_MODE aCache, RULES_VARIANT aVariant, int aThreads )
1124{
1125 return wxString::Format( wxT( "%s/%s/t%d" ), aCache == CACHE_MODE::COLD ? "cold" : "warm",
1126 variantName( aVariant ), aThreads );
1127}
1128
1129} // namespace
1130
1131
1132int main( int argc, char** argv )
1133{
1134 wxInitialize( argc, argv );
1135
1136 // Force the C locale so numeric parsing of the trace lines stays on a dot decimal
1137 // separator regardless of the environment.
1138 std::setlocale( LC_ALL, "C" );
1139
1140 // Line-buffer stdout so per-board progress streams to a redirected log during a long sweep
1141 // instead of materializing only at exit.
1142 std::setvbuf( stdout, nullptr, _IOLBF, 0 );
1143
1144 // The self-check exercises the trace parser alone and needs no engine, project, or
1145 // board, so handle it before any heavier initialization.
1146 for( int i = 1; i < argc; ++i )
1147 {
1148 if( std::string( argv[i] ) == "--selftest" )
1149 {
1150 int rv = RunTraceCaptureSelftest();
1151 wxUninitialize();
1152
1153 return rv;
1154 }
1155 }
1156
1157 SetPgm( &g_program );
1158 Pgm().InitPgm( true, true );
1159
1161 propMgr.Rebuild();
1162
1163 static const wxCmdLineEntryDesc cmdLineDesc[] = {
1164 { wxCMD_LINE_SWITCH, nullptr, "selftest", "run the trace-parser self-check and exit",
1165 wxCMD_LINE_VAL_NONE, wxCMD_LINE_PARAM_OPTIONAL },
1166 { wxCMD_LINE_OPTION, nullptr, "board", "ad-hoc board override (skips the corpus manifest)",
1167 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1168 { wxCMD_LINE_OPTION, "r", "rules", "default-variant design rules file (.kicad_dru)",
1169 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1170 { wxCMD_LINE_OPTION, nullptr, "heavy-rules", "heavy-variant design rules file (.kicad_dru)",
1171 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1172 { wxCMD_LINE_OPTION, nullptr, "rules-variant",
1173 "none|default|heavy (default: sweep all three)", wxCMD_LINE_VAL_STRING,
1174 wxCMD_LINE_PARAM_OPTIONAL },
1175 { wxCMD_LINE_SWITCH, nullptr, "rules-only",
1176 "time only InitEngine() compile in a loop, no checks", wxCMD_LINE_VAL_NONE,
1177 wxCMD_LINE_PARAM_OPTIONAL },
1178 { wxCMD_LINE_OPTION, nullptr, "threads", "worker threads, 0=all (default 0)",
1179 wxCMD_LINE_VAL_NUMBER, wxCMD_LINE_PARAM_OPTIONAL },
1180 { wxCMD_LINE_OPTION, nullptr, "repeat", "timed repeats after warm-up (default 5)",
1181 wxCMD_LINE_VAL_NUMBER, wxCMD_LINE_PARAM_OPTIONAL },
1182 { wxCMD_LINE_OPTION, nullptr, "cache", "cold|warm|both (default both)",
1183 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1184 { wxCMD_LINE_OPTION, nullptr, "isolate",
1185 "ignore every provider but this one to attribute its eval cost",
1186 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1187 { wxCMD_LINE_OPTION, nullptr, "top-n", "worst-offender list length (default 10)",
1188 wxCMD_LINE_VAL_NUMBER, wxCMD_LINE_PARAM_OPTIONAL },
1189 { wxCMD_LINE_OPTION, nullptr, "out", "output directory for JSON artifacts (default cwd)",
1190 wxCMD_LINE_VAL_STRING, wxCMD_LINE_PARAM_OPTIONAL },
1191 { wxCMD_LINE_OPTION, nullptr, "max-load",
1192 "abort when 1-min loadavg exceeds this (default ncores*0.5)", wxCMD_LINE_VAL_STRING,
1193 wxCMD_LINE_PARAM_OPTIONAL },
1194 { wxCMD_LINE_OPTION, nullptr, "timeout",
1195 "per-test deadline in seconds; 0 disables (default 60)", wxCMD_LINE_VAL_NUMBER,
1196 wxCMD_LINE_PARAM_OPTIONAL },
1197 { wxCMD_LINE_SWITCH, nullptr, "quick",
1198 "fast iteration set: small/synthetic boards, warm none+heavy, repeat 3",
1199 wxCMD_LINE_VAL_NONE, wxCMD_LINE_PARAM_OPTIONAL },
1200 { wxCMD_LINE_OPTION, nullptr, "quick-max-mb",
1201 "in --quick, also keep real boards smaller than this many MB (default 0 = synthetic only)",
1202 wxCMD_LINE_VAL_NUMBER, wxCMD_LINE_PARAM_OPTIONAL },
1203 { wxCMD_LINE_PARAM, nullptr, nullptr, "board.kicad_pcb", wxCMD_LINE_VAL_STRING,
1204 wxCMD_LINE_PARAM_OPTIONAL },
1205 { wxCMD_LINE_NONE }
1206 };
1207
1208 wxCmdLineParser parser( argc, argv );
1209 parser.SetDesc( cmdLineDesc );
1210 parser.SetLogo( "qa_drc_benchmark: time DRC rule compile + expression evaluation" );
1211
1212 if( parser.Parse() != 0 )
1213 {
1214 Pgm().Destroy();
1215 wxUninitialize();
1216 return 1;
1217 }
1218
1220
1221 auto cleanupExit = [&]( int aCode )
1222 {
1223 Pgm().Destroy();
1224 wxUninitialize();
1225 return aCode;
1226 };
1227
1228 // Resolve which boards to run. An explicit --board or a positional board is an ad-hoc run;
1229 // otherwise the corpus manifest drives the sweep. An unconfigured corpus with no ad-hoc
1230 // board is a clean skip, never a failure.
1231 std::vector<CORPUS_ENTRY> entries;
1232
1233 wxString adHocBoard;
1234 bool haveAdHocBoard = parser.Found( "board", &adHocBoard );
1235
1236 if( !haveAdHocBoard && parser.GetParamCount() > 0 )
1237 {
1238 adHocBoard = parser.GetParam( 0 );
1239 haveAdHocBoard = true;
1240 }
1241
1242 std::optional<wxString> cliDefaultRules;
1243 wxString rulesArg;
1244
1245 if( parser.Found( "r", &rulesArg ) )
1246 cliDefaultRules = rulesArg;
1247
1248 std::optional<wxString> cliHeavyRules;
1249 wxString heavyArg;
1250
1251 if( parser.Found( "heavy-rules", &heavyArg ) )
1252 cliHeavyRules = heavyArg;
1253
1254 if( haveAdHocBoard )
1255 {
1256 CORPUS_ENTRY entry;
1257 wxFileName board( adHocBoard );
1258 board.MakeAbsolute();
1259 entry.board = board.GetFullPath();
1260
1261 if( cliDefaultRules )
1262 {
1263 wxFileName rules( *cliDefaultRules );
1264 rules.MakeAbsolute();
1265 entry.rules = rules.GetFullPath();
1266 }
1267
1268 entry.tier = wxT( "adhoc" );
1269 entries.push_back( entry );
1270 }
1271 else
1272 {
1273 if( !CORPUS::IsConfigured() )
1274 {
1275 std::printf( "KICAD_DRC_BENCH_CORPUS is unset or does not name a directory; "
1276 "nothing to benchmark.\n"
1277 "Set it to a corpus root containing corpus.json, or pass a board "
1278 "with --board <file.kicad_pcb>. Skipping.\n" );
1279 return cleanupExit( 0 );
1280 }
1281
1282 wxString loadError;
1283
1284 if( !CORPUS::Load( entries, loadError ) )
1285 {
1286 std::printf( "error loading corpus: %s\n",
1287 static_cast<const char*>( loadError.utf8_str() ) );
1288 return cleanupExit( 1 );
1289 }
1290
1291 if( entries.empty() )
1292 {
1293 std::printf( "corpus at %s has no entries; nothing to benchmark.\n",
1294 static_cast<const char*>( CORPUS::Root().utf8_str() ) );
1295 return cleanupExit( 0 );
1296 }
1297 }
1298
1299 long threadsArg = 0;
1300 int threads = 0;
1301
1302 if( parser.Found( "threads", &threadsArg ) )
1303 threads = static_cast<int>( threadsArg );
1304
1305 bool quick = parser.Found( "quick" );
1306
1307 double timeoutSec = 60.0;
1308 long timeoutArg = 60;
1309
1310 if( parser.Found( "timeout", &timeoutArg ) )
1311 timeoutSec = static_cast<double>( std::max( 0L, timeoutArg ) );
1312
1313 // Default 0 keeps the quick set to the curated synthetic stressors only; a positive value
1314 // opts small real boards back in for a slightly broader but slower iteration loop.
1315 double quickMaxMb = 0.0;
1316 long quickMaxArg = 0;
1317
1318 if( parser.Found( "quick-max-mb", &quickMaxArg ) )
1319 quickMaxMb = static_cast<double>( std::max( 0L, quickMaxArg ) );
1320
1321 long repeatArg = 5;
1322 int repeat = quick ? 3 : 5;
1323
1324 if( parser.Found( "repeat", &repeatArg ) )
1325 repeat = std::max( 1, static_cast<int>( repeatArg ) );
1326
1327 long topNArg = 10;
1328 int topN = 10;
1329
1330 if( parser.Found( "top-n", &topNArg ) )
1331 topN = std::max( 1, static_cast<int>( topNArg ) );
1332
1333 std::optional<wxString> isolate;
1334 wxString isolateArg;
1335
1336 if( parser.Found( "isolate", &isolateArg ) )
1337 {
1338 isolate = isolateArg;
1339
1340 if( !providerErrorCodes().count( isolateArg ) )
1341 std::printf( "warning: --isolate '%s' has no known error codes; the full provider "
1342 "set will run.\n",
1343 static_cast<const char*>( isolateArg.utf8_str() ) );
1344 }
1345
1346 wxString outDir = wxFileName::GetCwd();
1347 wxString outArg;
1348
1349 if( parser.Found( "out", &outArg ) )
1350 outDir = outArg;
1351
1352 unsigned ncores = std::max( 1u, std::thread::hardware_concurrency() );
1353 double maxLoad = ncores * 0.5;
1354 wxString maxLoadArg;
1355
1356 if( parser.Found( "max-load", &maxLoadArg ) )
1357 maxLoadArg.ToCDouble( &maxLoad );
1358
1359 // Low-load guard. The benchmark only means anything on an otherwise-idle machine, so refuse
1360 // to start under contention rather than emit noisy numbers the optimizer might trust.
1361 double startLoad = readOneMinuteLoad();
1362
1363 if( maxLoad > 0.0 && startLoad > maxLoad )
1364 {
1365 std::printf( "aborting: 1-min loadavg %.2f exceeds limit %.2f (cores=%u). "
1366 "Run on an idle machine or raise --max-load.\n",
1367 startLoad, maxLoad, ncores );
1368 return cleanupExit( 2 );
1369 }
1370
1371 std::vector<CACHE_MODE> cacheModes = { CACHE_MODE::COLD, CACHE_MODE::WARM };
1372 wxString cacheArg;
1373
1374 if( parser.Found( "cache", &cacheArg ) )
1375 {
1376 if( cacheArg == wxT( "cold" ) )
1377 cacheModes = { CACHE_MODE::COLD };
1378 else if( cacheArg == wxT( "warm" ) )
1379 cacheModes = { CACHE_MODE::WARM };
1380 else if( cacheArg != wxT( "both" ) )
1381 {
1382 std::printf( "error: --cache must be cold|warm|both\n" );
1383 return cleanupExit( 1 );
1384 }
1385 }
1386 else if( quick )
1387 {
1388 // Warm is the iteration-relevant number and halves the work versus cold+warm.
1389 cacheModes = { CACHE_MODE::WARM };
1390 }
1391
1392 std::vector<RULES_VARIANT> variants = { RULES_VARIANT::NONE, RULES_VARIANT::DEFAULT,
1393 RULES_VARIANT::HEAVY };
1394 wxString variantArg;
1395
1396 if( parser.Found( "rules-variant", &variantArg ) )
1397 {
1398 RULES_VARIANT v = RULES_VARIANT::DEFAULT;
1399
1400 if( !parseVariant( variantArg, v ) )
1401 {
1402 std::printf( "error: --rules-variant must be none|default|heavy\n" );
1403 return cleanupExit( 1 );
1404 }
1405
1406 variants = { v };
1407 }
1408 else if( quick )
1409 {
1410 // none gives the geometric baseline and heavy gives compile_ms plus the eval overhead
1411 // over that baseline; the default variant adds cost without new signal in quick mode.
1412 variants = { RULES_VARIANT::NONE, RULES_VARIANT::HEAVY };
1413 }
1414
1415 // In quick mode keep only the immediately meaningful boards: the curated fast cells that
1416 // expose compiler and evaluator cost in seconds. Honor explicit "quick" manifest flags when
1417 // present, otherwise fall back to the synthetic tier C. --quick-max-mb opts small real boards
1418 // back in. The multi-minute giants belong to the full sweep, not the iteration loop.
1419 if( quick && !haveAdHocBoard )
1420 {
1421 bool anyFlagged = std::any_of( entries.begin(), entries.end(),
1422 []( const CORPUS_ENTRY& e ) { return e.quick; } );
1423
1424 std::vector<CORPUS_ENTRY> kept;
1425
1426 for( const CORPUS_ENTRY& entry : entries )
1427 {
1428 wxULongLong size = wxFileName::GetSize( entry.board );
1429 bool small = quickMaxMb > 0.0 && size != wxInvalidSize
1430 && size.ToDouble() < quickMaxMb * 1.0e6;
1431 bool flagged = anyFlagged ? entry.quick : ( entry.tier == wxT( "C" ) );
1432
1433 if( flagged || small )
1434 kept.push_back( entry );
1435 }
1436
1437 entries.swap( kept );
1438 }
1439
1440 std::printf( "corpus: %s\n", haveAdHocBoard
1441 ? "ad-hoc"
1442 : static_cast<const char*>( CORPUS::Root().utf8_str() ) );
1443 std::printf( "boards: %zu%s\n", entries.size(), quick ? " (quick set)" : "" );
1444 std::printf( "threads: %d (0=all)\n", threads );
1445 std::printf( "repeat: %d\n", repeat );
1446 std::printf( "timeout: %.0f s/test%s\n", timeoutSec, timeoutSec > 0.0 ? "" : " (disabled)" );
1447 std::printf( "max-load: %.2f (start %.2f, cores %u)\n", maxLoad, startLoad, ncores );
1448
1449 if( isolate )
1450 std::printf( "isolate: %s\n", static_cast<const char*>( isolate->utf8_str() ) );
1451
1452 std::printf( "out: %s\n\n", static_cast<const char*>( outDir.utf8_str() ) );
1453
1454 int rv = 0;
1455 std::vector<COVERAGE_ROW> coverageRows;
1456 std::vector<RESULT_ROW> resultRows;
1457
1458 bool rulesOnly = parser.Found( "rules-only" );
1459
1460 for( const CORPUS_ENTRY& entry : entries )
1461 {
1462 wxFileName boardName( entry.board );
1463 wxFileName projectName( boardName );
1464 projectName.SetExt( FILEEXT::ProjectFileExtension );
1465
1466 // A single malformed corpus board must not take down the whole sweep, so isolate every
1467 // board's load + timing behind a catch that records the failure and moves on.
1468 try
1469 {
1470
1471 if( projectName.Exists() )
1472 manager.LoadProject( projectName.GetFullPath() );
1473
1474 // The manifest rules path feeds both the default and heavy variants; ad-hoc runs may
1475 // instead carry an explicit --rules. Heavy falls back to --heavy-rules when present.
1476 std::optional<wxString> entryDefaultRules;
1477
1478 if( !entry.rules.IsEmpty() )
1479 entryDefaultRules = entry.rules;
1480 else if( cliDefaultRules )
1481 entryDefaultRules = cliDefaultRules;
1482
1483 std::optional<wxString> entryHeavyRules = cliHeavyRules ? cliHeavyRules : entryDefaultRules;
1484
1485 std::printf( "### board: %s (tier %s)\n",
1486 static_cast<const char*>( boardName.GetFullName().utf8_str() ),
1487 static_cast<const char*>( entry.tier.utf8_str() ) );
1488
1489 // Coverage uses the default-variant rules so the matrix reflects the rule set the corpus
1490 // ships, independent of any heavy sibling used only for stress timing.
1491 wxFileName coverageRules =
1492 resolveRules( boardName, entryDefaultRules, entryHeavyRules, RULES_VARIANT::DEFAULT );
1493
1494 coverageRows.push_back(
1495 collectCoverage( boardName, manager, projectName, coverageRules ) );
1496
1497 if( rulesOnly )
1498 {
1499 std::printf( "%-10s %14s %12s\n", "variant", "compile_med", "compile_mad" );
1500 std::printf( "%-10s %14s %12s\n", "----------", "--------------", "------------" );
1501
1502 for( RULES_VARIANT variant : variants )
1503 {
1504 wxFileName rulesFile =
1505 resolveRules( boardName, entryDefaultRules, entryHeavyRules, variant );
1506
1507 STAT compile =
1508 runCompileOnly( boardName, manager, projectName, rulesFile, repeat );
1509
1510 std::printf( "%-10s %14.3f %12.3f\n", variantName( variant ), compile.median,
1511 compile.mad );
1512
1513 RESULT_ROW resultRow;
1514 resultRow.board = boardName.GetFullName();
1515 resultRow.config = configTag( CACHE_MODE::COLD, variant, threads ) + wxT( "/compile" );
1516 resultRow.compile = compile;
1517 resultRows.push_back( resultRow );
1518 }
1519
1520 std::printf( "\n" );
1521 continue;
1522 }
1523
1524 for( CACHE_MODE cache : cacheModes )
1525 {
1526 const char* cacheLabel = cache == CACHE_MODE::COLD ? "cold" : "warm";
1527
1528 std::printf( "--- cache: %s ---\n", cacheLabel );
1529 std::printf( "%-10s %12s %10s %12s %12s %12s %10s\n", "variant", "compile_ms", "(mad)",
1530 "cache_gen", "check_ms", "eval_ovhd", "violations" );
1531 std::printf( "%-10s %12s %10s %12s %12s %12s %10s\n", "----------", "------------",
1532 "----------", "------------", "------------", "------------",
1533 "----------" );
1534
1535 std::optional<double> noneCheck;
1536
1537 for( RULES_VARIANT variant : variants )
1538 {
1539 BENCH_CONFIG config;
1540 config.rulesVariant = variant;
1541 config.cache = cache;
1542 config.threads = threads;
1543 config.repeat = repeat;
1544
1545 wxFileName rulesFile =
1546 resolveRules( boardName, entryDefaultRules, entryHeavyRules, variant );
1547
1548 SWEEP_RESULT result = runConfig( boardName, manager, projectName, rulesFile, config,
1549 isolate, maxLoad, timeoutSec );
1550
1551 if( !result.ran )
1552 {
1553 rv = 1;
1554 continue;
1555 }
1556
1557 // A timed-out none baseline cannot anchor eval_overhead, so only record it when
1558 // the check actually completed.
1559 if( variant == RULES_VARIANT::NONE && !result.timedOut )
1560 noneCheck = result.check.median;
1561
1562 RESULT_ROW resultRow;
1563 resultRow.board = boardName.GetFullName();
1564 resultRow.config = configTag( cache, variant, threads );
1565 resultRow.compile = result.compile;
1566 resultRow.cacheGen = result.cacheGen;
1567 resultRow.check = result.check;
1568 resultRow.violations = result.violations;
1569 resultRow.underLoad = result.underLoad;
1570 resultRow.timedOut = result.timedOut;
1571 resultRow.fraction = result.fraction;
1572 resultRow.perProvider = result.providerStats;
1573
1574 // eval_overhead isolates what the evaluator adds over the rules-free geometric
1575 // baseline for the same board and cache mode. It only has meaning when both this
1576 // check and the none baseline completed in this same sweep; a timeout leaves the
1577 // check time partial, so the overhead stays unrecorded.
1578 wxString ovhd = wxT( "n/a" );
1579
1580 if( result.timedOut )
1581 {
1582 ovhd = wxT( "timeout" );
1583 }
1584 else if( variant == RULES_VARIANT::NONE )
1585 {
1586 resultRow.evalOverheadMs = 0.0;
1587 resultRow.evalOverheadValid = true;
1588 ovhd = wxT( "0.000" );
1589 }
1590 else if( noneCheck )
1591 {
1592 resultRow.evalOverheadMs = result.check.median - *noneCheck;
1593 resultRow.evalOverheadValid = true;
1594 ovhd = wxString::Format( wxT( "%.3f" ), resultRow.evalOverheadMs );
1595 }
1596
1597 resultRows.push_back( resultRow );
1598
1599 if( result.timedOut )
1600 {
1601 std::printf( "%-10s %12.3f %10.3f %12s %12s %12s [TIMEOUT %.0f%%]\n",
1602 variantName( variant ), result.compile.median, result.compile.mad,
1603 "-", "-", static_cast<const char*>( ovhd.utf8_str() ),
1604 result.fraction * 100.0 );
1605 }
1606 else
1607 {
1608 std::printf( "%-10s %12.3f %10.3f %12.3f %12.3f %12s %10d%s\n",
1609 variantName( variant ), result.compile.median, result.compile.mad,
1610 result.cacheGen.median, result.check.median,
1611 static_cast<const char*>( ovhd.utf8_str() ), result.violations,
1612 result.underLoad ? " [UNDER_LOAD]" : "" );
1613 }
1614 }
1615
1616 std::printf( "\n" );
1617 }
1618
1619 }
1620 catch( const std::exception& e )
1621 {
1622 std::printf( "error: board '%s' failed and was skipped: %s\n\n",
1623 static_cast<const char*>( boardName.GetFullName().utf8_str() ), e.what() );
1624 rv = 1;
1625 }
1626 }
1627
1628 emitCoverage( coverageRows, outDir );
1629 writeWorstOffenders( resultRows, outDir, topN );
1630 writeResultsJson( resultRows, outDir );
1631
1632 std::printf( "wrote results.json, worst_offenders.json, coverage.json to %s\n",
1633 static_cast<const char*>( outDir.utf8_str() ) );
1634
1635 return cleanupExit( rv );
1636}
const char * name
General utilities for PCB file IO for QA programs.
Container for design settings for a BOARD object.
std::map< int, SEVERITY > m_DRCSeverities
std::shared_ptr< DRC_ENGINE > m_DRCEngine
Information pertinent to a Pcbnew printed circuit board.
Definition board.h:411
BOARD_DESIGN_SETTINGS & GetDesignSettings()
Definition board.cpp:1318
static bool Load(std::vector< CORPUS_ENTRY > &aEntries, wxString &aError)
Parse <root>/corpus.json into resolved entries.
Definition corpus.cpp:74
static bool IsConfigured()
True when KICAD_DRC_BENCH_CORPUS is set and names an existing directory.
Definition corpus.cpp:31
static wxString Root()
The resolved corpus root, or an empty string when unconfigured.
Definition corpus.cpp:42
wxLog chain target that scrapes the engine's "KICAD_DRC_PROFILE" trace channel.
const std::map< wxString, double > & ProviderMs() const
double TotalMs() const
Hold an error message and may be used when throwing exceptions containing meaningful error messages.
virtual const wxString What() const
A composite of Problem() and Where()
Container for data for KiCad programs.
Definition pgm_base.h:102
void Destroy()
Definition pgm_base.cpp:190
BS::priority_thread_pool & GetThreadPool()
Definition pgm_base.cpp:150
bool InitPgm(bool aHeadless=false, bool aIsUnitTest=false)
Initialize this program.
Definition pgm_base.cpp:373
virtual SETTINGS_MANAGER & GetSettingsManager() const
Definition pgm_base.h:124
A small class to help profiling.
Definition profile.h:46
void Stop()
Save the time when this function was called, and set the counter stane to stop.
Definition profile.h:86
double msecs(bool aSinceLast=false)
Definition profile.h:148
This implements all the tricky bits for thread safety, but the GUI is left to derived classes.
Provide class metadata.
static PROPERTY_MANAGER & Instance()
void Rebuild()
Rebuild the list of all registered properties.
A simple RAII class to measure the time of an operation.
Definition profile.h:223
A scoped application of a wxLog target.
PROJECT & Prj() const
A helper while we are not MDI-capable – return the one and only project.
bool LoadProject(const wxString &aFullPath, bool aSetActive=true, bool aOverrideLock=false)
Load a project or sets up a new project with a specified path.
const char * ConstraintTypeName(DRC_CONSTRAINT_T aType)
Human-readable token for a DRC_CONSTRAINT_T, matching the .kicad_dru keyword where one exists.
Definition corpus.cpp:151
std::vector< wxString > ScanPredicatesInRules(const wxString &aRulesText)
Scan raw .kicad_dru text for occurrences of each registered predicate name.
Definition corpus.cpp:293
const std::vector< wxString > & AllPredicateNames()
Every pcbexpr predicate registered in pcbexpr_functions.cpp, used for textual coverage scans.
Definition corpus.cpp:251
const std::vector< DRC_CONSTRAINT_T > & AllConstraintTypes()
Every DRC_CONSTRAINT_T the engine can carry rules for, in enum order, excluding NULL_CONSTRAINT.
Definition corpus.cpp:203
std::string utf8(const wxString &aText)
@ DRCE_DP_UNCOUPLED_LENGTH_TOO_LONG
Definition drc_item.h:126
@ DRCE_SKEW_OUT_OF_RANGE
Definition drc_item.h:123
@ DRCE_CREEPAGE
Definition drc_item.h:53
@ DRCE_HOLE_CLEARANCE
Definition drc_item.h:63
@ DRCE_SILK_EDGE_CLEARANCE
Definition drc_item.h:114
@ DRCE_SILK_MASK_CLEARANCE
Definition drc_item.h:112
@ DRCE_VIA_DIAMETER
Definition drc_item.h:70
@ DRCE_UNCONNECTED_ITEMS
Definition drc_item.h:48
@ DRCE_DP_GAP_OUT_OF_RANGE
Definition drc_item.h:125
@ DRCE_TRACK_WIDTH
Definition drc_item.h:64
@ DRCE_PADSTACK
Definition drc_item.h:71
@ DRCE_MIRRORED_TEXT_ON_FRONT_LAYER
Definition drc_item.h:128
@ DRCE_OVERLAPPING_FOOTPRINTS
Definition drc_item.h:80
@ DRCE_TRACK_ON_POST_MACHINED_LAYER
Definition drc_item.h:134
@ DRCE_TEXT_ON_EDGECUTS
Definition drc_item.h:51
@ DRCE_NET_CHAIN_STUB_TOO_LONG
Definition drc_item.h:120
@ DRCE_DRILL_OUT_OF_RANGE
Definition drc_item.h:69
@ DRCE_EDGE_CLEARANCE
Definition drc_item.h:55
@ DRCE_NET_CHAIN_RETURN_PATH_BREAK
Definition drc_item.h:121
@ DRCE_STARVED_THERMAL
Definition drc_item.h:58
@ DRCE_TRACK_SEGMENT_LENGTH
Definition drc_item.h:66
@ DRCE_MISSING_COURTYARD
Definition drc_item.h:81
@ DRCE_TRACK_ANGLE
Definition drc_item.h:65
@ DRCE_TRACK_NOT_CENTERED_ON_VIA
Definition drc_item.h:136
@ DRCE_CLEARANCE
Definition drc_item.h:52
@ DRCE_ISOLATED_COPPER
Definition drc_item.h:57
@ DRCE_DRILLED_HOLES_TOO_CLOSE
Definition drc_item.h:61
@ DRCE_ALLOWED_ITEMS
Definition drc_item.h:50
@ DRCE_COPPER_SLIVER
Definition drc_item.h:108
@ DRCE_PTH_IN_COURTYARD
Definition drc_item.h:84
@ DRCE_MICROVIA_DRILL_OUT_OF_RANGE
Definition drc_item.h:73
@ DRCE_SHORTING_ITEMS
Definition drc_item.h:49
@ DRCE_MALFORMED_COURTYARD
Definition drc_item.h:82
@ DRCE_FIRST
Definition drc_item.h:47
@ DRCE_DANGLING_VIA
Definition drc_item.h:59
@ DRCE_FOOTPRINT_TYPE_MISMATCH
Definition drc_item.h:96
@ DRCE_DANGLING_TRACK
Definition drc_item.h:60
@ DRCE_TEXT_HEIGHT
Definition drc_item.h:116
@ DRCE_SOLDERMASK_BRIDGE
Definition drc_item.h:109
@ DRCE_DRILLED_HOLES_COLOCATED
Definition drc_item.h:62
@ DRCE_SILK_CLEARANCE
Definition drc_item.h:115
@ DRCE_LENGTH_OUT_OF_RANGE
Definition drc_item.h:119
@ DRCE_LAST
Definition drc_item.h:145
@ DRCE_PAD_TH_WITH_NO_HOLE
Definition drc_item.h:99
@ DRCE_UNMIRRORED_TEXT_ON_BACK_LAYER
Definition drc_item.h:129
@ DRCE_LINE_WIDTH
Definition drc_item.h:117
@ DRCE_NPTH_IN_COURTYARD
Definition drc_item.h:85
@ DRCE_CONNECTION_WIDTH
Definition drc_item.h:68
@ DRCE_TRACKS_CROSSING
Definition drc_item.h:54
@ DRCE_VIA_COUNT_OUT_OF_RANGE
Definition drc_item.h:124
@ DRCE_ANNULAR_WIDTH
Definition drc_item.h:67
@ NONE
Definition eda_fill.h:42
static const std::string ProjectFileExtension
static const std::string DesignRulesFileExtension
std::unique_ptr< BOARD > ReadBoardFromFileOrStream(const std::string &aFilename, std::istream &aFallback)
Read a board from a file, or another stream, as appropriate.
std::chrono::duration< double, std::milli > ms
void SetPgm(PGM_BASE *pgm)
PGM_BASE & Pgm()
The global program "get" accessor.
see class PGM_BASE
@ RPT_SEVERITY_IGNORE
std::vector< FAB_LAYER_COLOR > dummy
#define TO_UTF8(wxstring)
Convert a wxString to a UTF8 encoded C string for all wxWidgets build modes.
One board+rules pairing from the out-of-tree corpus manifest.
Definition corpus.h:36
wxString tier
Free-form tier tag from the manifest (A/B/C).
Definition corpus.h:39
wxString board
Absolute path to the .kicad_pcb.
Definition corpus.h:37
wxString rules
Absolute path to the .kicad_dru, or empty for none.
Definition corpus.h:38
VECTOR2I end
wxString result
Test unit parsing edge cases and error handling.
void InvalidateKiCadThreadPool()
Invalidate the cached thread pool pointer.
int RunTraceCaptureSelftest()
Run the three fixed self-test lines through a fresh parser and confirm the parsed provider map and to...
VECTOR2< int32_t > VECTOR2I
Definition vector2d.h:708
Definition of file extensions used in Kicad.