KiCad PCB EDA Suite
Loading...
Searching...
No Matches
sch_io_database.cpp
Go to the documentation of this file.
1/*
2 * This program source code file is part of KiCad, a free EDA CAD application.
3 *
4 * Copyright (C) 2022 Jon Evans <[email protected]>
5 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
6 *
7 * This program is free software: you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation, either version 3 of the License, or (at your
10 * option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful, but
13 * WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <https://www.gnu.org/licenses/>.
19 */
20
21#include <algorithm>
22#include <chrono>
23#include <filesystem>
24#include <fstream>
25#include <iostream>
26#include <map>
27#include <random>
28#include <string_view>
29#include <set>
30#include <utility>
31#include <vector>
32#include <bs_thread_pool.hpp>
33#include <wx/datetime.h>
34#include <wx/filename.h>
35#include <wx/log.h>
36#include <wx/tokenzr.h>
37
38#include <boost/algorithm/string.hpp>
39#include <json_common.h>
40#include <pin_map.h>
41
46#include <fmt.h>
47#include <gestfich.h>
48#include <hash.h>
49#include <ki_exception.h>
50#include <lib_symbol.h>
51
52#include "sch_io_database.h"
53
55
56
57thread_local std::unordered_set<wxString> SCH_IO_DATABASE::s_inProgressLoads;
58
59
61 SCH_IO( wxS( "Database library" ) ),
62 m_adapter( nullptr ),
63 m_settings(),
64 m_conn()
65{
68}
69
70
75
76
77void SCH_IO_DATABASE::EnumerateSymbolLib( wxArrayString& aSymbolNameList,
78 const wxString& aLibraryPath,
79 const std::map<std::string, UTF8>* aProperties )
80{
81 std::vector<LIB_SYMBOL*> symbols;
82 EnumerateSymbolLib( symbols, aLibraryPath, aProperties );
83
84 for( LIB_SYMBOL* symbol : symbols )
85 {
86 aSymbolNameList.Add( symbol->GetName() );
87 delete symbol;
88 }
89}
90
91
93{
94 // Set under the wait mutex so a waiter between its check and its sleep cannot miss the wakeup
95 {
96 std::lock_guard lock( m_refreshMutex );
97 m_loadCancelled = true;
98 }
99
100 m_refreshCV.notify_all();
101}
102
103
105{
106 std::lock_guard lock( m_refreshMutex );
107 m_loadCancelled = false;
108}
109
110
111void SCH_IO_DATABASE::EnumerateSymbolLib( std::vector<LIB_SYMBOL*>& aSymbolList,
112 const wxString& aLibraryPath,
113 const std::map<std::string, UTF8>* aProperties )
114{
115 wxCHECK_RET( m_adapter, "Database plugin missing library manager adapter handle!" );
116 ensureSettings( aLibraryPath );
118
119 bool powerSymbolsOnly = ( aProperties && aProperties->contains( SYMBOL_LIBRARY_ADAPTER::PropPowerSymsOnly ) );
120
121 std::shared_lock lock( m_cacheMutex );
122
123 for( auto const& pair : m_nameToSymbolcache )
124 {
125 LIB_SYMBOL* symbol = pair.second.get();
126
127 if( !powerSymbolsOnly || symbol->IsPower() )
128 aSymbolList.emplace_back( symbol->Duplicate() );
129 }
130}
131
132
133void SCH_IO_DATABASE::CheckLibrary( const wxString& aLibraryPath,
134 const std::map<std::string, UTF8>* aProperties )
135{
136 ensureSettings( aLibraryPath );
138}
139
140
141LIB_SYMBOL* SCH_IO_DATABASE::LoadSymbol( const wxString& aLibraryPath,
142 const wxString& aAliasName,
143 const std::map<std::string, UTF8>* aProperties )
144{
145 wxCHECK_MSG( m_adapter, nullptr, "Database plugin missing library manager adapter handle!" );
146 ensureSettings( aLibraryPath );
147 std::shared_ptr<DATABASE_CONNECTION> conn = ensureConnection();
148
149 cacheLib( conn );
150
151 std::string tableName;
152 std::string symbolName( aAliasName.ToUTF8() );
153
154 {
155 std::shared_lock lock( m_cacheMutex );
156
157 if( auto cacheIt = m_nameToSymbolcache.find( aAliasName ); cacheIt != m_nameToSymbolcache.end() )
158 {
159 LIB_SYMBOL* cached = cacheIt->second.get();
160 return cached->Duplicate();
161 }
162
163 auto sanitizedIt = m_sanitizedNameMap.find( aAliasName );
164
165 if( sanitizedIt != m_sanitizedNameMap.end() )
166 {
167 tableName = sanitizedIt->second.first;
168 symbolName = sanitizedIt->second.second;
169 }
170 }
171
172 /*
173 * Table names are tricky, in order to allow maximum flexibility to the user.
174 * The slash character is used as a separator between a table name and symbol name, but symbol
175 * names may also contain slashes and table names may now also be empty (which results in the
176 * slash being dropped in the symbol name when placing a new symbol). So, if a slash is found,
177 * we check if the string before the slash is a valid table name. If not, we assume the table
178 * name is blank if our config has an entry for the null table.
179 */
180
181 if( tableName.empty() && aAliasName.Contains( '/' ) )
182 {
183 tableName = std::string( aAliasName.BeforeFirst( '/' ).ToUTF8() );
184 symbolName = std::string( aAliasName.AfterFirst( '/' ).ToUTF8() );
185 }
186
187 std::vector<const DATABASE_LIB_TABLE*> tablesToTry;
188
189 for( const DATABASE_LIB_TABLE& tableIter : m_settings->m_Tables )
190 {
191 // no table means globally unique keys, try all tables
192 if( tableName.empty() || tableIter.name == tableName )
193 tablesToTry.emplace_back( &tableIter );
194 }
195
196 if( tablesToTry.empty() )
197 {
198 wxLogTrace( traceDatabase, wxT( "LoadSymbol: table '%s' not found in config" ), tableName );
199 return nullptr;
200 }
201
202 const DATABASE_LIB_TABLE* foundTable = nullptr;
204
205 for( const DATABASE_LIB_TABLE* table : tablesToTry )
206 {
207 if( conn->SelectOne( table->table, std::make_pair( table->key_col, symbolName ), result ) )
208 {
209 foundTable = table;
210 wxLogTrace( traceDatabase, wxT( "LoadSymbol: SelectOne (%s, %s) found in %s" ),
211 table->key_col, symbolName, table->table );
212 }
213 else
214 {
215 wxLogTrace( traceDatabase, wxT( "LoadSymbol: SelectOne (%s, %s) failed for table %s" ),
216 table->key_col, symbolName, table->table );
217 }
218 }
219
220 if( !foundTable )
221 return nullptr;
222
223 return loadSymbolFromRow( aAliasName, *foundTable, result ).release();
224}
225
226
227void SCH_IO_DATABASE::GetSubLibraryNames( std::vector<wxString>& aNames )
228{
229 ensureSettings( wxEmptyString );
230
231 aNames.clear();
232
233 std::set<wxString> tableNames;
234
235 for( const DATABASE_LIB_TABLE& tableIter : m_settings->m_Tables )
236 {
237 if( tableNames.count( tableIter.name ) )
238 continue;
239
240 aNames.emplace_back( tableIter.name );
241 tableNames.insert( tableIter.name );
242 }
243}
244
245
246void SCH_IO_DATABASE::GetAvailableSymbolFields( std::vector<wxString>& aNames )
247{
248 std::lock_guard<std::mutex> lock( m_symbolLoadMutex );
249
250 std::copy( m_customFields.begin(), m_customFields.end(), std::back_inserter( aNames ) );
251}
252
253
254void SCH_IO_DATABASE::GetDefaultSymbolFields( std::vector<wxString>& aNames )
255{
256 std::lock_guard<std::mutex> lock( m_symbolLoadMutex );
257
258 std::copy( m_defaultShownFields.begin(), m_defaultShownFields.end(), std::back_inserter( aNames ) );
259}
260
261
262size_t SCH_IO_DATABASE::computeSignature( const TABLE_RESULT_LIST& aTableResults ) const
263{
264 size_t signature = 0;
265
266 for( const auto& [table, results] : aTableResults )
267 {
268 for( const DATABASE_CONNECTION::ROW& result : results )
269 {
270 size_t rowSignature = 0;
271
272 for( const auto& [column, value] : result )
273 {
274 hash_combine( signature, std::string_view( column ) );
275
276 if( const std::string* str = std::any_cast<std::string>( &value ) )
277 hash_combine( rowSignature, std::string_view( *str ) );
278 }
279
280 hash_combine( signature, rowSignature );
281
282 if( auto it = result.find( table->symbols_col ); it != result.end() )
283 {
284 if( const std::string* str = std::any_cast<std::string>( &it->second ) )
285 {
286 LIB_ID symbolId;
287 symbolId.Parse( *str );
288
289 if( symbolId.IsValid() )
290 {
291 const UTF8& nickname = symbolId.GetLibNickname();
292 hash_combine( signature, std::string_view( nickname.c_str() ) );
293
294 if( std::optional<int> libHash = m_adapter->GetLibraryModifyHash( nickname ) )
295 hash_combine( signature, *libHash );
296 }
297 }
298 }
299 }
300 }
301
302 return signature;
303}
304
305
307{
308 if( !m_adapter )
309 return nullptr;
310
311 if( !m_previewMode )
312 return m_adapter->LoadSymbol( aId );
313
314 std::optional<LIBRARY_TABLE_ROW*> row = m_adapter->GetRow( aId.GetLibNickname() );
315
316 // A database library would query its own tables, which the preview must not do
317 if( row && SCH_IO_MGR::EnumFromStr( ( *row )->Type() ) == SCH_IO_MGR::SCH_DATABASE )
318 {
319 reportIssue( PREVIEW_ISSUE::FIELD::SYMBOLS,
320 wxString::Format( _( "Symbol '%s' is in a database library and is not previewed." ),
321 aId.Format().wx_str() ),
323 return nullptr;
324 }
325
326 // Not memoized; the pointer belongs to another library's cache, which a later load may rebuild
327 LIB_SYMBOL* symbol = m_adapter->LoadSymbol( aId );
328
329 if( !symbol )
330 {
331 reportIssue( PREVIEW_ISSUE::FIELD::SYMBOLS,
332 wxString::Format( _( "Symbol '%s' was not found." ), aId.Format().wx_str() ),
334 }
335
336 return symbol;
337}
338
339
341static wxString cwdDirFor( const DATABASE_LIB_SETTINGS& aSettings, const wxString& aBaseDir )
342{
343 return aBaseDir.IsEmpty() ? wxFileName( aSettings.GetFilename() ).GetPath() : aBaseDir;
344}
345
346
348 bool aConnectNow, const wxString& aBaseDir )
349{
350 return connectionFor( DATABASE_BACKEND::Create( aSettings.m_Source, cwdDirFor( aSettings, aBaseDir ) ),
351 aConnectNow );
352}
353
354
356{
357 if( !aBackend.backend )
358 return { nullptr, aBackend.failure, std::move( aBackend.error ) };
359
360 return { std::make_unique<DATABASE_CONNECTION>( std::move( aBackend.backend ), aConnectNow ), DB_FAILURE::UNKNOWN,
361 std::string() };
362}
363
364
365wxString SCH_IO_DATABASE::SymbolNameForRow( const DATABASE_LIB_TABLE& aTable, bool aGloballyUniqueKeys,
366 const DATABASE_CONNECTION::ROW& aRow, std::string* aRawKey )
367{
368 auto it = aRow.find( aTable.key_col );
369
370 if( it == aRow.end() )
371 return wxEmptyString;
372
373 std::string rawName = std::any_cast<std::string>( it->second );
374 UTF8 sanitizedName = LIB_ID::FixIllegalChars( rawName, false );
375 std::string prefix = ( aGloballyUniqueKeys || aTable.name.empty() ) ? "" : fmt::format( "{}/", aTable.name );
376
377 if( aRawKey )
378 *aRawKey = rawName;
379
380 return wxString::FromUTF8( fmt::format( "{}{}", prefix, sanitizedName.c_str() ) );
381}
382
383
384std::vector<std::unique_ptr<LIB_SYMBOL>>
386 bool aGloballyUniqueKeys, const std::vector<DATABASE_CONNECTION::ROW>& aRows,
387 PREVIEW_ISSUES* aIssues )
388{
389 SCH_IO_DATABASE preview;
390 preview.m_adapter = aLibs;
391 preview.m_previewIssues = aIssues;
392 preview.m_previewMode = true;
393
394 std::vector<std::unique_ptr<LIB_SYMBOL>> symbols;
395 std::map<wxString, std::pair<size_t, std::string>> listed;
396
397 for( const DATABASE_CONNECTION::ROW& row : aRows )
398 {
399 std::string rawKey;
400 wxString name = SymbolNameForRow( aTable, aGloballyUniqueKeys, row, &rawKey );
401
402 if( name.IsEmpty() )
403 continue;
404
405 try
406 {
407 std::unique_ptr<LIB_SYMBOL> symbol = preview.loadSymbolFromRow( name, aTable, row );
408
409 if( !symbol )
410 continue;
411
412 // The library keeps the last row listed under a name, so the preview does the same
413 if( auto it = listed.find( name ); it != listed.end() )
414 {
415 if( it->second.second == rawKey )
416 {
417 preview.reportIssue( PREVIEW_ISSUE::FIELD::REPEATED_KEY,
418 wxString::Format( _( "Key '%s' appears in more than one row; only the last "
419 "row is kept." ),
420 wxString::FromUTF8( rawKey ) ),
422 }
423 else
424 {
425 preview.reportIssue( PREVIEW_ISSUE::FIELD::KEY,
426 wxString::Format( _( "Keys '%s' and '%s' are both listed as '%s'; only the "
427 "last is kept." ),
428 wxString::FromUTF8( it->second.second ),
429 wxString::FromUTF8( rawKey ), name ),
431 }
432
433 symbols[it->second.first] = std::move( symbol );
434 it->second.second = rawKey;
435 continue;
436 }
437
438 listed[name] = { symbols.size(), rawKey };
439 symbols.push_back( std::move( symbol ) );
440 }
441 catch( const IO_ERROR& e )
442 {
443 preview.reportIssue( PREVIEW_ISSUE::FIELD::KEY, e.What(), RPT_SEVERITY_ERROR );
444 }
445 }
446
447 return symbols;
448}
449
450
451void SCH_IO_DATABASE::reportIssue( PREVIEW_ISSUE::FIELD aField, const wxString& aMessage, SEVERITY aSeverity ) const
452{
453 if( m_previewIssues )
454 m_previewIssues->push_back( { aField, aMessage } );
455 else
456 Report( aMessage, aSeverity );
457}
458
459
461 std::map<wxString, std::unique_ptr<LIB_SYMBOL>>& aSymbolCache,
462 std::map<wxString, std::pair<std::string, std::string>>& aSanitizedNameMap )
463{
464 for( const auto& [table, results] : aTableResults )
465 {
466 for( const DATABASE_CONNECTION::ROW& result : results )
467 {
468 std::string rawName;
469 wxString name = SymbolNameForRow( *table, m_settings->m_GloballyUniqueKeys, result, &rawName );
470
471 if( name.IsEmpty() )
472 continue;
473
474 aSanitizedNameMap[name] = std::make_pair( table->name, rawName );
475
476 std::unique_ptr<LIB_SYMBOL> symbol = loadSymbolFromRow( name, *table, result );
477
478 if( symbol )
479 aSymbolCache[symbol->GetName()] = std::move( symbol );
480 }
481 }
482
483 return !aSymbolCache.empty();
484}
485
486
487void SCH_IO_DATABASE::cacheLib( const std::shared_ptr<DATABASE_CONNECTION>& aConn )
488{
489 // Guard against re-entrant cacheLib() calls. A self-referential symbol row (issue #24249)
490 // causes m_adapter->LoadSymbol to route back into SCH_IO_DATABASE::LoadSymbol, which would
491 // otherwise call cacheLib() again while it is in the middle of populating its caches.
492 if( m_inCacheLib )
493 return;
494
495 long long currentTimestampSeconds = wxDateTime::Now().GetValue().GetValue() / 1000;
496
497 // After the initial load, the background refresh thread handles all cache updates.
498 {
499 std::shared_lock lock( m_cacheMutex );
500
501 if( m_cachePopulated )
502 return;
503 }
504
505 m_inCacheLib = true;
506
507 struct CACHE_LIB_GUARD
508 {
509 bool* flag;
510 ~CACHE_LIB_GUARD() { *flag = false; }
511 } cacheLibGuard{ &m_inCacheLib };
512
513 // Re-query the database (the connection layer caches results subject to its own max_age) and
514 // compute a lightweight signature of the raw rows so we can skip the costly materialization
515 // when nothing relevant has changed.
516 TABLE_RESULT_LIST tableResults;
517
518 // A backend may open sessions for the load, so they are closed before the slow materialization
519 {
521 [this]()
522 {
523 return m_loadCancelled.load();
524 } );
525
526 for( const DATABASE_LIB_TABLE& table : m_settings->m_Tables )
527 {
528 // Nothing is published from a load that was cancelled
529 if( m_loadCancelled.load() )
530 THROW_IO_ERROR( _( "Database library load cancelled" ) );
531
532 std::vector<DATABASE_CONNECTION::ROW> results;
534
535 if( !aConn->SelectAll( table.table, table.key_col, results, &status ) )
536 {
537 // A read the cancel stopped says nothing about the table
538 if( m_loadCancelled.load() )
539 THROW_IO_ERROR( _( "Database library load cancelled" ) );
540
541 if( !status.error.empty() )
542 THROW_IO_ERRORF( _( "Error reading database table %s: %s" ), table.table, status.error );
543
544 continue;
545 }
546
547 tableResults.emplace_back( &table, std::move( results ) );
548 }
549 }
550
551 if( m_loadCancelled.load() )
552 THROW_IO_ERROR( _( "Database library load cancelled" ) );
553
554 size_t signature = computeSignature( tableResults );
555
556 {
557 std::unique_lock lock( m_cacheMutex );
558
559 if( m_cachePopulated && signature == m_cacheSignature )
560 {
561 m_cacheTimestamp = currentTimestampSeconds;
562 return;
563 }
564 }
565
566 std::map<wxString, std::unique_ptr<LIB_SYMBOL>> newSymbolCache;
567 std::map<wxString, std::pair<std::string, std::string>> newSanitizedNameMap;
568
569 materializeCache( tableResults, newSymbolCache, newSanitizedNameMap );
570
571 {
572 std::unique_lock lock( m_cacheMutex );
573
574 m_nameToSymbolcache = std::move( newSymbolCache );
575 m_sanitizedNameMap = std::move( newSanitizedNameMap );
576
577 m_cacheTimestamp = currentTimestampSeconds;
578 m_cacheSignature = signature;
579 m_cachePopulated = true;
580 m_modifyHash++;
581 }
582
583 if( !m_refreshRunning.load() )
585}
586
587
589{
590 // The flag check and the thread install stay together so a stop cannot slip between them
591 std::lock_guard lock( m_refreshMutex );
592
593 if( m_stopping.load() || m_refreshRunning.load() )
594 return;
595
596 m_refreshRunning = true;
597
598 wxLogTrace( traceDatabase, wxT( "Starting background refresh thread" ) );
600}
601
602
604{
605 std::thread worker;
606
607 // Set under the wait mutex so a waiter between its check and its sleep cannot miss the wakeup
608 {
609 std::lock_guard lock( m_refreshMutex );
610
611 m_stopping = true;
612 m_refreshRunning = false;
613 worker = std::move( m_refreshThread );
614 }
615
616 m_refreshCV.notify_all();
617
618 if( worker.joinable() )
619 worker.join();
620}
621
622
623std::chrono::milliseconds SCH_IO_DATABASE::busyRetryWait( long long aMaxAgeSeconds, int aAttempt,
624 std::minstd_rand& aRng )
625{
626 // A writer holds its lock for seconds, so waiting a whole max_age after one leaves the cache stale for longer
627 static constexpr long long FIRST_RETRY_MS = 3000;
628
629 const long long ceiling =
630 std::min( aMaxAgeSeconds * 1000, FIRST_RETRY_MS << std::clamp( aAttempt, 0, MAX_BUSY_ATTEMPT ) );
631
632 // Readers that failed on one lock must not all come back together and keep the writer out
633 std::uniform_int_distribution<long long> spread( ceiling / 2, ceiling );
634
635 return std::chrono::milliseconds( spread( aRng ) );
636}
637
638
640{
641 return m_stopping.load() || m_loadCancelled.load();
642}
643
644
646{
647 BS::this_thread::set_os_thread_name( "dblib bg" );
648
649 std::minstd_rand jitter( std::random_device{}() );
650 int busyRetries = 0;
651
652 while( m_refreshRunning.load() )
653 {
654 long long maxAge = 0;
655 bool busy = false;
656
657 {
658 std::unique_lock lock( m_cacheMutex );
659
660 if( m_settings )
661 maxAge = m_settings->m_Cache.max_age;
662 }
663
664 if( maxAge <= 0 )
665 maxAge = 1;
666
667 std::shared_ptr<DATABASE_CONNECTION> conn = currentConnection();
668 bool dropped = m_cachePopulated.load() && ( !conn || !conn->IsConnected() );
669
670 // A failed read leaves the backend disconnected, and only a new connection can end that
671 if( dropped && m_refreshRunning.load() )
672 {
673 // A catalog a writer holds is retried on the busy backoff, like a locked read
675 conn = currentConnection();
676 }
677
678 // The pass owns its connection, so a reconnect cannot free it and no lock is held across the reads
679 if( conn && conn->IsConnected() && m_cachePopulated.load() )
680 {
681 wxLogTrace( traceDatabase, wxT( "Initiating background refresh" ) );
682
683 try
684 {
685 TABLE_RESULT_LIST tableResults;
686 bool querySuccess = true;
687
688 {
690 [this]()
691 {
692 return !m_refreshRunning.load() || m_stopping.load();
693 } );
694
695 for( const DATABASE_LIB_TABLE& table : m_settings->m_Tables )
696 {
697 // A close must not wait for the tables left in the pass
698 if( !m_refreshRunning.load() )
699 {
700 querySuccess = false;
701 break;
702 }
703
704 conn->ClearCache( table.table );
705
706 std::vector<DATABASE_CONNECTION::ROW> results;
708
709 if( !conn->SelectAll( table.table, table.key_col, results, &status ) )
710 {
711 wxLogTrace( traceDatabase, wxT( "Background refresh: SelectAll failed for table %s" ),
712 table.table );
713 querySuccess = false;
714
715 // A lost connection goes to the reconnect at the top of the next pass instead
716 busy = status.connected && status.failure == DB_FAILURE::FILE_IN_USE;
717 break;
718 }
719
720 tableResults.emplace_back( &table, std::move( results ) );
721 }
722 }
723
724 // A stop during the last table must not wait for a publish
725 if( !m_refreshRunning.load() )
726 querySuccess = false;
727
728 if( querySuccess )
729 {
730 size_t signature = computeSignature( tableResults );
731
732 bool dataChanged = false;
733
734 {
735 std::unique_lock lock( m_cacheMutex );
736
737 if( signature == m_cacheSignature )
738 m_cacheTimestamp = wxDateTime::Now().GetValue().GetValue() / 1000;
739 else
740 dataChanged = true;
741 }
742
743 if( dataChanged )
744 {
745 std::map<wxString, std::unique_ptr<LIB_SYMBOL>> newSymbolCache;
746 std::map<wxString, std::pair<std::string, std::string>> newSanitizedNameMap;
747
748 materializeCache( tableResults, newSymbolCache, newSanitizedNameMap );
749
750 wxLogTrace( traceDatabase, wxT( "Background refresh: new data" ) );
751
752 {
753 std::unique_lock lock( m_cacheMutex );
754
755 m_nameToSymbolcache = std::move( newSymbolCache );
756 m_sanitizedNameMap = std::move( newSanitizedNameMap );
757
758 m_cacheTimestamp = wxDateTime::Now().GetValue().GetValue() / 1000;
759 m_cacheSignature = signature;
760 m_cachePopulated = true;
761 m_modifyHash++;
762 }
763 }
764 else
765 {
766 wxLogTrace( traceDatabase, wxT( "Background refresh: no new data" ) );
767 }
768 }
769 }
770 catch( const IO_ERROR& e )
771 {
772 wxLogTrace( traceDatabase, wxT( "Background refresh failed: %s; cache preserved" ), e.What() );
773 }
774 catch( const std::exception& e )
775 {
776 wxLogTrace( traceDatabase, wxT( "Background refresh failed: %s; cache preserved" ), e.what() );
777 }
778 }
779
780 // A connection the foreground replaced must not stay open through the wait
781 conn.reset();
782
783 std::chrono::milliseconds wait = std::chrono::seconds( maxAge );
784
785 if( busy )
786 {
787 wait = busyRetryWait( maxAge, busyRetries, jitter );
788 busyRetries = std::min( busyRetries + 1, MAX_BUSY_ATTEMPT );
789 }
790 else
791 {
792 busyRetries = 0;
793 }
794
795 {
796 std::unique_lock lock( m_refreshMutex );
797 m_refreshCV.wait_for( lock, wait,
798 [this]()
799 {
800 return !m_refreshRunning.load();
801 } );
802 }
803 }
804}
805
806
807void SCH_IO_DATABASE::ensureSettings( const wxString& aSettingsPath )
808{
809 if( !m_settings && !aSettingsPath.IsEmpty() )
810 {
811 std::string path( aSettingsPath.ToUTF8() );
812 m_settings = std::make_unique<DATABASE_LIB_SETTINGS>( path );
813 m_settings->SetReadOnly( true );
814
815 // A read that fails leaves these defaults to size the next attempt's wait
816 m_settings->Load();
817
818 loadSettingsBounded( m_settings->m_Source.timeout );
819 }
821 {
822 // Reload in case the user is editing; the read is bounded because the share may be down
823 loadSettingsBounded( m_settings->m_Source.timeout );
824 }
825 else if( currentConnection() && m_settings && !aSettingsPath.IsEmpty() )
826 {
827 wxASSERT_MSG( aSettingsPath == m_settings->GetFilename(),
828 "Path changed for database library without re-initializing plugin!" );
829 }
830 else if( !m_settings )
831 {
832 wxLogTrace( traceDatabase, wxT( "ensureSettings: no settings but no valid path!" ) );
833 }
834}
835
836
837namespace
838{
839// Owned jointly by the callers and the reader thread, which is all the thread touches
840struct SETTINGS_READ
841{
842 std::mutex m;
843 std::condition_variable cv;
844 bool done = false;
845 bool ok = false;
846 std::string bytes;
847};
848
849
850// Reached only from callers, so its static lifetime cannot race a detached reader
851struct SETTINGS_READ_REGISTRY
852{
853 std::mutex m;
854 std::map<std::string, std::weak_ptr<SETTINGS_READ>> reads;
855};
856
857
858SETTINGS_READ_REGISTRY& settingsReadRegistry()
859{
860 static SETTINGS_READ_REGISTRY* instance = new SETTINGS_READ_REGISTRY;
861
862 return *instance;
863}
864
865
866// Runs detached, so it uses no wx state and never throws
867void readSettingsFile( const std::shared_ptr<SETTINGS_READ>& aRead, const std::filesystem::path& aPath )
868{
869 std::string bytes;
870 bool ok = false;
871
872 try
873 {
874 std::ifstream in( aPath, std::ios::binary );
875 std::vector<char> chunk( 1 << 16 );
876
877 while( in )
878 {
879 in.read( chunk.data(), chunk.size() );
880 bytes.append( chunk.data(), static_cast<size_t>( in.gcount() ) );
881 }
882
883 ok = in.eof() && !in.bad();
884 }
885 catch( ... )
886 {
887 bytes.clear();
888 ok = false;
889 }
890
891 std::lock_guard lock( aRead->m );
892 aRead->bytes = std::move( bytes );
893 aRead->ok = ok;
894 aRead->done = true;
895 aRead->cv.notify_all();
896}
897} // namespace
898
899
900void SCH_IO_DATABASE::loadSettingsBounded( int aTimeoutSeconds )
901{
902 const wxString path = m_settings->GetLoadPath();
903
904 if( m_loadCancelled.load() )
905 THROW_IO_ERRORF( _( "Database library settings file %s could not be reached" ), path );
906
907 std::shared_ptr<SETTINGS_READ> read;
908
909 {
910 SETTINGS_READ_REGISTRY& registry = settingsReadRegistry();
911 std::lock_guard registryLock( registry.m );
912
913 for( auto it = registry.reads.begin(); it != registry.reads.end(); )
914 {
915 it = it->second.expired() ? registry.reads.erase( it ) : std::next( it );
916 }
917
918 // Normalised without touching the file system, since the share may be down
919 wxFileName file( path );
920 file.Normalize( wxPATH_NORM_DOTS | wxPATH_NORM_ABSOLUTE );
921
922 const std::filesystem::path nativePath = ToFsPath( file.GetFullPath() );
923
924 file.Normalize( wxPATH_NORM_CASE );
925
926 const std::string key( file.GetFullPath().ToUTF8() );
927
928 if( std::shared_ptr<SETTINGS_READ> pending = registry.reads[key].lock() )
929 {
930 std::lock_guard readLock( pending->m );
931
932 if( !pending->done )
933 read = pending;
934 }
935
936 if( !read )
937 {
938 read = std::make_shared<SETTINGS_READ>();
939 registry.reads[key] = read;
940
941 try
942 {
943 std::thread( readSettingsFile, read, nativePath ).detach();
944 }
945 catch( ... )
946 {
947 registry.reads.erase( key );
948 THROW_IO_ERRORF( _( "Database library settings file %s could not be reached" ), path );
949 }
950 }
951 }
952
953 const auto deadline =
954 std::chrono::steady_clock::now() + std::chrono::seconds( DatabaseFileWaitSeconds( aTimeoutSeconds ) );
955
956 std::unique_lock lock( read->m );
957
958 while( !read->done )
959 {
960 if( m_loadCancelled.load() || std::chrono::steady_clock::now() >= deadline )
961 THROW_IO_ERRORF( _( "Database library settings file %s could not be reached" ), path );
962
963 read->cv.wait_for( lock, std::chrono::milliseconds( 50 ) );
964 }
965
966 const bool ok = read->ok;
967 const std::string bytes = read->bytes;
968
969 lock.unlock();
970
971 if( !ok || !m_settings->LoadFromBuffer( bytes ) )
972 THROW_IO_ERRORF( _( "Could not load database library: settings file %s missing or invalid" ), path );
973
974 m_settings->ResolvePassword();
975}
976
977
978std::shared_ptr<DATABASE_CONNECTION> SCH_IO_DATABASE::currentConnection()
979{
980 std::shared_lock lock( m_cacheMutex );
981 return m_conn;
982}
983
984
985std::shared_ptr<DATABASE_CONNECTION> SCH_IO_DATABASE::ensureConnection()
986{
987 wxCHECK_MSG( m_settings, nullptr, "Call ensureSettings before ensureConnection!" );
988
989 CONNECT_STATUS status = connect();
990
991 std::shared_ptr<DATABASE_CONNECTION> conn = currentConnection();
992
993 if( !conn || !conn->IsConnected() )
994 {
995 // No backend exists for the source, so there is no database to name
997 THROW_IO_ERRORF( _( "Could not load database library: %s" ), status.error );
998
999 const DATABASE_SOURCE source = m_settings->ExpandedSource();
1000 const std::string name = source.Describe();
1001 const bool configured = !source.dsn.empty() || !source.connection_string.empty();
1002
1003 // Nothing safe to name means the source was withheld, unless none is configured at all
1004 const wxString shown = !name.empty() ? wxString::FromUTF8( name )
1005 : configured ? _( "connection string" )
1006 : _( "(not configured)" );
1007
1008 THROW_IO_ERRORF( _( "Could not load database library: could not connect to database %s (%s)" ), shown,
1009 status.error );
1010 }
1011
1012 return conn;
1013}
1014
1015
1017{
1018 wxCHECK_MSG( m_settings, CONNECT_STATUS(), "Call ensureSettings before connect()!" );
1019
1020 // Every load and symbol lookup comes through here, so a live connection returns before any other work
1021 if( std::shared_ptr<DATABASE_CONNECTION> live = currentConnection(); live && live->IsConnected() )
1022 return CONNECT_STATUS();
1023
1024 auto now = std::chrono::steady_clock::now;
1025
1026 const auto start = now();
1027 const auto deadline = start + FIRST_CONNECT_BUDGET;
1028 std::minstd_rand jitter( std::random_device{}() );
1029 bool firstOpen = false;
1030 std::chrono::milliseconds lastAttempt( 0 );
1031 CONNECT_STATUS status;
1032
1033 for( int attempt = 0;; ++attempt )
1034 {
1035 {
1036 // The refresh thread reconnects too, and two connections must not be built at once
1037 std::lock_guard connectGuard( m_connectMutex );
1038
1039 std::shared_ptr<DATABASE_CONNECTION> existing = currentConnection();
1040
1041 if( existing && existing->IsConnected() )
1042 return CONNECT_STATUS();
1043
1044 if( attempt == 0 )
1045 firstOpen = !existing;
1046
1047 // The wait or the mutex may have run past the budget the last check allowed for
1048 if( attempt > 0 && ( connectionStopped() || now() + lastAttempt > deadline ) )
1049 return status;
1050
1051 const auto attemptStart = now();
1052
1053 status = connectOnce();
1054 lastAttempt = std::chrono::duration_cast<std::chrono::milliseconds>( now() - attemptStart );
1055 }
1056
1057 // A reconnect has the refresh backoff, so only a library that never opened waits here
1058 if( !firstOpen || status.failure != DB_FAILURE::FILE_IN_USE || connectionStopped() )
1059 return status;
1060
1061 // Another attempt must also end inside the budget, as the file was in use for as long as the last one took
1062 const auto remaining = std::chrono::duration_cast<std::chrono::milliseconds>( deadline - now() ) - lastAttempt;
1063
1064 if( remaining <= std::chrono::milliseconds( 0 ) )
1065 return status;
1066
1067 const std::chrono::milliseconds wait = std::min( FileInUseBackoff( attempt, jitter ), remaining );
1068
1069 {
1070 std::unique_lock lock( m_refreshMutex );
1071
1072 m_refreshCV.wait_for( lock, wait,
1073 [this]()
1074 {
1075 return connectionStopped();
1076 } );
1077 }
1078
1079 if( connectionStopped() )
1080 return status;
1081 }
1082}
1083
1084
1086{
1087 // The old connection stays in place until the new one is ready, as other threads may still be reading from it
1089 DATABASE_BACKEND::Create( m_settings->m_Source, cwdDirFor( *m_settings, wxEmptyString ) ), true );
1090
1091 std::shared_ptr<DATABASE_CONNECTION> newConn = std::move( created.connection );
1092
1093 if( !newConn )
1094 return { created.failure, wxString::FromUTF8( created.error ) };
1095
1096 if( !newConn->IsConnected() )
1097 return { newConn->GetLastFailure(), newConn->GetLastError() };
1098
1099 for( const DATABASE_LIB_TABLE& tableIter : m_settings->m_Tables )
1100 {
1101 // Trusted even if unreported by the driver, to support generated columns (#16952)
1102 std::set<std::string> requiredColumns{ tableIter.key_col, tableIter.footprints_col, tableIter.symbols_col };
1103
1104 for( const DATABASE_FIELD_MAPPING& field : tableIter.fields )
1105 requiredColumns.insert( field.column );
1106
1107 if( !tableIter.pins_col.empty() )
1108 requiredColumns.insert( tableIter.pins_col );
1109
1110 // Only used if confirmed present, so a misconfigured mapping can't break the table (#23532)
1111 std::set<std::string> optionalColumns{
1113 tableIter.properties.keywords, tableIter.properties.exclude_from_sim,
1115 };
1116
1117 if( newConn->CacheTableInfo( tableIter.table, requiredColumns, optionalColumns ) )
1118 continue;
1119
1120 std::optional<DATABASE_CONNECTION::TABLE_FAILURE> failed = newConn->GetTableFailure( tableIter.table );
1121
1122 // A connection without a table that was only locked or out of reach would keep it missing until a reload
1123 if( failed && ( failed->failure == DB_FAILURE::FILE_IN_USE || failed->failure == DB_FAILURE::UNREACHABLE ) )
1124 {
1125 return { failed->failure,
1126 wxString::Format( _( "Error reading database table %s: %s" ),
1127 wxString::FromUTF8( tableIter.table ), wxString::FromUTF8( failed->error ) ) };
1128 }
1129 }
1130
1131 newConn->SetCacheParams( m_settings->m_Cache.max_size, m_settings->m_Cache.max_age );
1132
1133 std::unique_lock connLock( m_cacheMutex );
1134 m_conn = std::move( newConn );
1135
1136 return CONNECT_STATUS();
1137}
1138
1139
1140std::optional<bool> SCH_IO_DATABASE::boolFromAny( const std::any& aVal )
1141{
1142 try
1143 {
1144 bool val = std::any_cast<bool>( aVal );
1145 return val;
1146 }
1147 catch( const std::bad_any_cast& )
1148 {
1149 }
1150
1151 try
1152 {
1153 int val = std::any_cast<int>( aVal );
1154 return static_cast<bool>( val );
1155 }
1156 catch( const std::bad_any_cast& )
1157 {
1158 }
1159
1160 try
1161 {
1162 wxString strval( std::any_cast<std::string>( aVal ).c_str(), wxConvUTF8 );
1163
1164 if( strval.IsEmpty() )
1165 return std::nullopt;
1166
1167 strval.MakeLower();
1168
1169 for( const auto& trueVal : { wxS( "true" ), wxS( "yes" ), wxS( "y" ), wxS( "1" ) } )
1170 {
1171 if( strval.Matches( trueVal ) )
1172 return true;
1173 }
1174
1175 for( const auto& falseVal : { wxS( "false" ), wxS( "no" ), wxS( "n" ), wxS( "0" ) } )
1176 {
1177 if( strval.Matches( falseVal ) )
1178 return false;
1179 }
1180 }
1181 catch( const std::bad_any_cast& )
1182 {
1183 }
1184
1185 return std::nullopt;
1186}
1187
1188
1189std::unique_ptr<LIB_SYMBOL> SCH_IO_DATABASE::loadSymbolFromRow( const wxString& aSymbolName,
1190 const DATABASE_LIB_TABLE& aTable,
1191 const DATABASE_CONNECTION::ROW& aRow )
1192{
1193 std::unique_ptr<LIB_SYMBOL> symbol = nullptr;
1194
1195 if( aRow.contains( aTable.symbols_col ) )
1196 {
1197 std::string symbols = std::any_cast<std::string>( aRow.at( aTable.symbols_col ) );
1198 wxString symbolsStr = wxString( symbols.c_str(), wxConvUTF8 );
1199 wxStringTokenizer tokenizer( symbolsStr, ";\t\r\n", wxTOKEN_STRTOK );
1200
1201 std::vector<LIB_ID> symbolIds;
1202
1203 while( tokenizer.HasMoreTokens() )
1204 {
1205 wxString token = tokenizer.GetNextToken();
1206 LIB_ID id;
1207 id.Parse( std::string( token.ToUTF8() ) );
1208
1209 if( id.IsValid() )
1210 {
1211 symbolIds.push_back( id );
1212 }
1213 else
1214 {
1215 reportIssue( PREVIEW_ISSUE::FIELD::SYMBOLS,
1216 wxString::Format( _( "'%s' is not a symbol; expected Library:Symbol." ), token ),
1218 }
1219 }
1220
1221 // A row's Symbols column may resolve back into the same database library (issue #24249,
1222 // e.g. a mistyped library nickname). The adapter would route that lookup back into
1223 // SCH_IO_DATABASE::LoadSymbol and re-enter loadSymbolFromRow on the same row until the
1224 // stack overflows. Track in-flight LIB_IDs and skip the recursive load on re-entry.
1225 std::vector<LIB_SYMBOL*> sourceSymbols;
1226 std::vector<wxString> sourceSymbolNames;
1227
1228 for( const LIB_ID& symbolId : symbolIds )
1229 {
1230 wxString symbolIdStr = symbolId.Format().wx_str();
1231
1232 if( !s_inProgressLoads.insert( symbolIdStr ).second )
1233 {
1234 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: cycle detected resolving '%s' "
1235 "(row '%s' in table '%s'); skipping recursive load" ),
1236 symbolIdStr, aSymbolName, aTable.name );
1237 continue;
1238 }
1239
1240 struct CYCLE_GUARD
1241 {
1242 std::unordered_set<wxString>* set;
1243 wxString key;
1244 ~CYCLE_GUARD() { set->erase( key ); }
1245 } guard{ &s_inProgressLoads, symbolIdStr };
1246
1247 LIB_SYMBOL* src = nullptr;
1248
1249 {
1250 std::lock_guard lock( m_symbolLoadMutex );
1251 src = resolveSourceSymbol( symbolId );
1252 }
1253
1254 if( src )
1255 {
1256 sourceSymbols.push_back( src );
1257 sourceSymbolNames.push_back( src->GetName() );
1258 }
1259 else
1260 {
1261 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: source symbol '%s' not found" ), symbolIdStr );
1262 }
1263 }
1264
1265 if( sourceSymbols.empty() )
1266 {
1267 // Actual symbol not found: return metadata only; error will be indicated in the
1268 // symbol chooser
1269 symbol.reset( new LIB_SYMBOL( aSymbolName ) );
1270 }
1271 else if( sourceSymbols.size() == 1 )
1272 {
1273 symbol.reset( sourceSymbols[0]->Duplicate() );
1274 symbol->SetSourceLibId( symbolIds[0] );
1275 symbol->SetName( aSymbolName );
1276 }
1277 else
1278 {
1279 // If the database row specifies multiple symbols, put together a composite,
1280 // but since source symbols may already have multiple body styles, we
1281 // need to flatten out any common to all body styles items on source symbols.
1282 // Since we don't support varying all properties, take those from the first one.
1283
1284 symbol.reset( new LIB_SYMBOL( aSymbolName ) );
1285 symbol->SetSourceLibId( symbolIds[0] );
1286
1287 symbol->SetUnitCount( sourceSymbols[0]->GetUnitCount(), false );
1288 symbol->SetPowerSymbolProp( sourceSymbols[0]->IsPower() );
1289 symbol->SetShowPinNames( sourceSymbols[0]->GetShowPinNames() );
1290 symbol->SetShowPinNumbers( sourceSymbols[0]->GetShowPinNumbers() );
1291 symbol->SetPinNameOffset( sourceSymbols[0]->GetPinNameOffset() );
1292
1293 for( FIELD_T fieldId : MANDATORY_FIELDS )
1294 {
1295 if( SCH_FIELD* srcField = sourceSymbols[0]->GetField( fieldId ) )
1296 {
1297 SCH_FIELD* dstField = symbol->GetField( fieldId );
1298 *dstField = *srcField;
1299 dstField->SetParent( symbol.get() );
1300 }
1301 }
1302
1303 std::vector<wxString> bodyStyleNames;
1304 std::vector<int> sourceBodyStyleCounts;
1305 std::vector<int> compositeBodyStyleBase;
1306
1307 int nextBodyStyle = 1;
1308
1309 for( size_t i = 0; i < sourceSymbols.size(); ++i )
1310 {
1311 int srcCount = std::max( 1, sourceSymbols[i]->GetBodyStyleCount() );
1312 sourceBodyStyleCounts.push_back( srcCount );
1313 compositeBodyStyleBase.push_back( nextBodyStyle );
1314
1315 if( srcCount == 1 )
1316 {
1317 bodyStyleNames.push_back( sourceSymbolNames[i] );
1318 }
1319 else
1320 {
1321 for( int style = 1; style <= srcCount; ++style )
1322 {
1323 wxString styleName = sourceSymbols[i]->GetBodyStyleDescription( style, false );
1324 bodyStyleNames.push_back( sourceSymbolNames[i] + wxT( " (" ) + styleName + wxT( ")" ) );
1325 }
1326 }
1327
1328 nextBodyStyle += srcCount;
1329 }
1330
1331 int totalBodyStyles = nextBodyStyle - 1;
1332
1333 symbol->SetHasDeMorganBodyStyles( false );
1334 symbol->SetBodyStyleNames( bodyStyleNames );
1335
1336 std::set<wxString> mergedFieldNames;
1337
1338 for( size_t i = 0; i < sourceSymbols.size(); ++i )
1339 {
1340 std::unique_ptr<LIB_SYMBOL> srcSymbol = sourceSymbols[i]->Flatten();
1341 int srcCount = sourceBodyStyleCounts[i];
1342 int base = compositeBodyStyleBase[i];
1343
1344 for( SCH_ITEM& item : srcSymbol->GetDrawItems() )
1345 {
1346 if( item.Type() == SCH_FIELD_T )
1347 {
1348 SCH_FIELD& field = static_cast<SCH_FIELD&>( item );
1349
1350 if( field.IsMandatory() || !mergedFieldNames.insert( field.GetName() ).second )
1351 continue;
1352
1353 int targetStyle;
1354
1355 if( item.GetBodyStyle() == ALL_BODY_STYLES )
1356 targetStyle = base;
1357 else
1358 targetStyle = base + ( item.GetBodyStyle() - 1 );
1359
1360 if( targetStyle > totalBodyStyles )
1361 continue;
1362
1363 SCH_ITEM* newItem = item.Duplicate( IGNORE_PARENT_GROUP );
1364 newItem->SetParent( symbol.get() );
1365 newItem->SetBodyStyle( targetStyle );
1366 symbol->AddDrawItem( newItem, false );
1367 }
1368 else if( item.GetBodyStyle() == ALL_BODY_STYLES )
1369 {
1370 for( int bodyStyle = 0; bodyStyle < srcCount; ++bodyStyle )
1371 {
1372 SCH_ITEM* newItem = item.Duplicate( IGNORE_PARENT_GROUP );
1373 newItem->SetParent( symbol.get() );
1374 newItem->SetBodyStyle( base + bodyStyle );
1375 symbol->AddDrawItem( newItem, false );
1376 }
1377 }
1378 else
1379 {
1380 int targetStyle = base + ( item.GetBodyStyle() - 1 );
1381
1382 if( targetStyle > totalBodyStyles )
1383 continue;
1384
1385 SCH_ITEM* newItem = item.Duplicate( IGNORE_PARENT_GROUP );
1386 newItem->SetParent( symbol.get() );
1387 newItem->SetBodyStyle( targetStyle );
1388 symbol->AddDrawItem( newItem, false );
1389 }
1390 }
1391 }
1392
1393 symbol->GetDrawItems().sort();
1394 }
1395 }
1396 else
1397 {
1398 symbol.reset( new LIB_SYMBOL( aSymbolName ) );
1399 }
1400
1401 LIB_ID libId = symbol->GetLibId();
1402 libId.SetSubLibraryName( aTable.name );
1403 symbol->SetLibId( libId );
1404
1405 wxArrayString footprintsList;
1406
1407 if( aRow.count( aTable.footprints_col ) )
1408 {
1409 std::string footprints = std::any_cast<std::string>( aRow.at( aTable.footprints_col ) );
1410
1411 wxString footprintsStr = wxString( footprints.c_str(), wxConvUTF8 );
1412 wxStringTokenizer tokenizer( footprintsStr, ";\t\r\n", wxTOKEN_STRTOK );
1413
1414 while( tokenizer.HasMoreTokens() )
1415 footprintsList.Add( tokenizer.GetNextToken() );
1416
1417 if( footprintsList.size() > 0 )
1418 symbol->GetFootprintField().SetText( footprintsList[0] );
1419 }
1420 else
1421 {
1422 wxLogTrace( traceDatabase, wxT( "loadSymboFromRow: footprint field %s not found." ), aTable.footprints_col );
1423 }
1424
1425 // Pin-to-pad maps (issue #2282): attach non-destructively. The pins column carries either the
1426 // spec-form named object { "pin_maps": [...], "associated_footprints": [...] } or the legacy
1427 // flat MR !2540 array (read for one release, bound to the row's footprints).
1428 if( !aTable.pins_col.empty() && aRow.count( aTable.pins_col ) )
1429 {
1430 try
1431 {
1432 std::string jsonStr = std::any_cast<std::string>( aRow.at( aTable.pins_col ) );
1433
1434 if( !jsonStr.empty() )
1435 {
1436 nlohmann::json json = nlohmann::json::parse( jsonStr );
1437
1438 if( json.is_object() && json.contains( "pin_maps" ) )
1439 {
1440 symbol->SetPinMaps( ParsePinMapSet( json ) );
1441 symbol->SetAssociatedFootprints( ParseAssociatedFootprints( json ) );
1442 }
1443 else if( !footprintsList.IsEmpty() )
1444 {
1445 std::unordered_map<wxString, std::vector<wxString>> assignments = ParseLegacyPinAssignments( json );
1446
1447 if( !assignments.empty() )
1448 {
1449 const wxString mapName = wxS( "Database" );
1450
1451 symbol->PinMaps().AddOrReplace( MakeLegacyPinMap( mapName, assignments ) );
1452
1453 // Bind the one named map to every footprint the row offers, mirroring the
1454 // old behaviour where the assignment applied regardless of footprint.
1455 std::vector<ASSOCIATED_FOOTPRINT> associations;
1456
1457 for( const wxString& footprint : footprintsList )
1458 {
1459 LIB_ID fpId;
1460 fpId.Parse( footprint );
1461 associations.push_back( { fpId, mapName } );
1462 }
1463
1464 symbol->SetAssociatedFootprints( std::move( associations ) );
1465 }
1466 }
1467 }
1468 }
1469 catch( const std::exception& e )
1470 {
1471 // Surface a malformed pin-map payload to the user instead of silently dropping it; the
1472 // symbol still loads without the map (issue #2282).
1473 reportIssue( PREVIEW_ISSUE::FIELD::PIN_MAP,
1474 wxString::Format( _( "Error parsing pin map for database symbol '%s': %s" ), aSymbolName,
1475 e.what() ),
1477 }
1478 }
1479
1480 if( !aTable.properties.description.empty() && aRow.count( aTable.properties.description ) )
1481 {
1482 wxString value(
1483 std::any_cast<std::string>( aRow.at( aTable.properties.description ) ).c_str(),
1484 wxConvUTF8 );
1485 symbol->SetDescription( value );
1486 }
1487
1488 if( !aTable.properties.keywords.empty() && aRow.count( aTable.properties.keywords ) )
1489 {
1490 wxString value( std::any_cast<std::string>( aRow.at( aTable.properties.keywords ) ).c_str(),
1491 wxConvUTF8 );
1492 symbol->SetKeyWords( value );
1493 }
1494
1495 if( !aTable.properties.footprint_filters.empty()
1496 && aRow.count( aTable.properties.footprint_filters ) )
1497 {
1498 wxString value( std::any_cast<std::string>( aRow.at( aTable.properties.footprint_filters ) )
1499 .c_str(),
1500 wxConvUTF8 );
1501 footprintsList.push_back( value );
1502 }
1503
1504 symbol->SetFPFilters( footprintsList );
1505
1506 if( !aTable.properties.exclude_from_sim.empty()
1507 && aRow.count( aTable.properties.exclude_from_sim ) )
1508 {
1509 std::optional<bool> val = boolFromAny( aRow.at( aTable.properties.exclude_from_sim ) );
1510
1511 if( val )
1512 {
1513 symbol->SetExcludedFromSim( *val );
1514 }
1515 else
1516 {
1517 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: exclude_from_sim value for %s "
1518 "could not be cast to a boolean" ), aSymbolName );
1519 }
1520 }
1521
1522 if( !aTable.properties.exclude_from_board.empty()
1523 && aRow.count( aTable.properties.exclude_from_board ) )
1524 {
1525 std::optional<bool> val = boolFromAny( aRow.at( aTable.properties.exclude_from_board ) );
1526
1527 if( val )
1528 {
1529 symbol->SetExcludedFromBoard( *val );
1530 }
1531 else
1532 {
1533 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: exclude_from_board value for %s "
1534 "could not be cast to a boolean" ), aSymbolName );
1535 }
1536 }
1537
1538 if( !aTable.properties.exclude_from_bom.empty()
1539 && aRow.count( aTable.properties.exclude_from_bom ) )
1540 {
1541 std::optional<bool> val = boolFromAny( aRow.at( aTable.properties.exclude_from_bom ) );
1542
1543 if( val )
1544 {
1545 symbol->SetExcludedFromBOM( *val );
1546 }
1547 else
1548 {
1549 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: exclude_from_bom value for %s "
1550 "could not be cast to a boolean" ), aSymbolName );
1551 }
1552 }
1553
1554 std::vector<SCH_FIELD*> fields;
1555 symbol->GetFields( fields );
1556
1557 std::unordered_map<wxString, SCH_FIELD*> fieldsMap;
1558
1559 for( SCH_FIELD* field : fields )
1560 fieldsMap[field->GetName()] = field;
1561
1562 static const wxString c_valueFieldName( wxS( "Value" ) );
1563 static const wxString c_datasheetFieldName( wxS( "Datasheet" ) );
1564 static const wxString c_footprintFieldName( wxS( "Footprint" ) );
1565
1566 // User fields produced from the database mapping must appear in the order they are
1567 // declared in the .kicad_dbl file, not in lexicographic order of their values. Start
1568 // assigning ordinals above whatever the source LIB_SYMBOL already uses so that any
1569 // pre-existing user fields keep their relative position before the database fields.
1570 int dbFieldOrdinal = symbol->GetNextFieldOrdinal();
1571
1572 for( const DATABASE_FIELD_MAPPING& mapping : aTable.fields )
1573 {
1574 if( !aRow.count( mapping.column ) )
1575 {
1576 wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: field %s not found in result" ), mapping.column );
1577 continue;
1578 }
1579
1580 // Skip footprint field if it maps to the footprints column, since that column is
1581 // already processed above with tokenization for semicolon-separated multiple footprints.
1582 if( mapping.name_wx == c_footprintFieldName && mapping.column == aTable.footprints_col )
1583 continue;
1584
1585 std::string strValue;
1586
1587 try
1588 {
1589 strValue = std::any_cast<std::string>( aRow.at( mapping.column ) );
1590 }
1591 catch( std::bad_any_cast& )
1592 {
1593 }
1594
1595 wxString value( strValue.c_str(), wxConvUTF8 );
1596
1597 if( mapping.name_wx == c_valueFieldName )
1598 {
1599 SCH_FIELD& field = symbol->GetValueField();
1600 field.SetText( value );
1601
1602 if( !mapping.inherit_properties )
1603 {
1604 field.SetVisible( mapping.visible_on_add );
1605 field.SetNameShown( mapping.show_name );
1606 }
1607
1608 continue;
1609 }
1610 else if( mapping.name_wx == c_datasheetFieldName )
1611 {
1612 SCH_FIELD& field = symbol->GetDatasheetField();
1613 field.SetText( value );
1614
1615 if( !mapping.inherit_properties )
1616 {
1617 field.SetVisible( mapping.visible_on_add );
1618 field.SetNameShown( mapping.show_name );
1619
1620 if( mapping.visible_on_add )
1621 field.SetAutoAdded( true );
1622 }
1623
1624 continue;
1625 }
1626
1627 SCH_FIELD* field;
1628 bool isNew = false;
1629
1630 if( fieldsMap.count( mapping.name_wx ) )
1631 {
1632 field = fieldsMap[mapping.name_wx];
1633 }
1634 else
1635 {
1636 field = new SCH_FIELD( nullptr, FIELD_T::USER );
1637 field->SetName( mapping.name_wx );
1638 isNew = true;
1639 fieldsMap[mapping.name_wx] = field;
1640 }
1641
1642 // Assign a sort-order ordinal so the property editor and BOM see the fields in the
1643 // order declared in the .kicad_dbl file. Without this, all USER fields share the
1644 // same FIELD_T::USER id and fall through to value-based comparison in operator<.
1645 // NB: a DB mapping that lands on a mandatory field by name (e.g. Reference or
1646 // Description) must keep its FIELD_T identity for downstream lookups.
1647 field->SetOrdinal( dbFieldOrdinal++, field->IsMandatory() ? field->GetId() : FIELD_T::USER );
1648
1649 if( !mapping.inherit_properties || isNew )
1650 {
1651 field->SetVisible( mapping.visible_on_add );
1652 field->SetAutoAdded( true );
1653 field->SetNameShown( mapping.show_name );
1654 }
1655
1656 field->SetText( value );
1657
1658 if( isNew )
1659 symbol->AddDrawItem( field, false );
1660
1661 m_customFields.insert( mapping.name_wx );
1662
1663 if( mapping.visible_in_chooser )
1664 m_defaultShownFields.insert( mapping.name_wx );
1665 }
1666
1667 symbol->GetDrawItems().sort();
1668
1669 // Field mappings (including Description) are applied with SCH_FIELD::SetText, which does not
1670 // refresh the cached values the library tree and chooser read. Without this the upper chooser
1671 // panel keeps the source symbol's description while the details panel shows the database value.
1672 symbol->RefreshLibraryTreeCaches();
1673
1674 return symbol;
1675}
1676
1677
1678DIALOG_SHIM* SCH_IO_DATABASE::CreateConfigurationDialog( wxWindow* aParent, const wxString& aLibraryPath )
1679{
1680 return new DIALOG_DATABASE_LIB_SETTINGS( aParent, aLibraryPath, m_adapter );
1681}
const char * name
static DB_BACKEND_RESULT Create(const DATABASE_SOURCE &aSource, const wxString &aBasePath)
Builds the backend for a source.
Brackets the reads of one library load or refresh, so the backend can keep what it opens for them.
std::map< std::string, std::any > ROW
Edits every setting of a .kicad_dbl file and previews the resulting symbols.
Dialog helper object to sit in the inheritance tree between wxDialog and any class written by wxFormB...
Definition dialog_shim.h:81
virtual void SetParent(EDA_ITEM *aParent)
Definition eda_item.cpp:153
virtual void SetVisible(bool aVisible)
Definition eda_text.cpp:348
bool IsEmpty() const
virtual void Report(const wxString &aText, SEVERITY aSeverity=RPT_SEVERITY_UNDEFINED) const
Definition io_base.cpp:124
Hold an error message and may be used when throwing exceptions containing meaningful error messages.
virtual const wxString What() const
A composite of Problem() and Where()
virtual const char * what() const override
std::exception interface, returned as UTF-8
wxString GetFilename() const
A logical library item identifier and consists of various portions much like a URI.
Definition lib_id.h:45
int Parse(const UTF8 &aId, bool aFix=false)
Parse LIB_ID with the information from aId.
Definition lib_id.cpp:65
bool IsValid() const
Check if this LID_ID is valid.
Definition lib_id.h:168
UTF8 Format() const
Definition lib_id.cpp:132
void SetSubLibraryName(const UTF8 &aName)
Definition lib_id.h:127
const UTF8 & GetLibNickname() const
Return the logical library name portion of a LIB_ID.
Definition lib_id.h:83
static UTF8 FixIllegalChars(const UTF8 &aLibItemName, bool aLib)
Replace illegal LIB_ID item name characters with underscores '_'.
Definition lib_id.cpp:205
Define a library symbol object.
Definition lib_symbol.h:114
bool IsPower() const override
virtual LIB_SYMBOL * Duplicate() const
Create a copy of a LIB_SYMBOL and assigns unique KIIDs to the copy and its children.
Definition lib_symbol.h:130
wxString GetName() const override
Definition lib_symbol.h:183
bool IsMandatory() const
void SetOrdinal(int aOrdinal, FIELD_T aType)
FIELD_T GetId() const
Definition sch_field.h:143
void SetAutoAdded(bool aAutoAdded)
Definition sch_field.h:244
wxString GetName(bool aUseDefaultName=true) const
Return the field name (not translated).
void SetName(const wxString &aName)
void SetText(const wxString &aText) override
void SetNameShown(bool aShown=true)
Definition sch_field.h:226
void reportIssue(PREVIEW_ISSUE::FIELD aField, const wxString &aMessage, SEVERITY aSeverity) const
Reports to the preview's issue list when there is one, otherwise to the reporter.
void EnumerateSymbolLib(wxArrayString &aSymbolNameList, const wxString &aLibraryPath, const std::map< std::string, UTF8 > *aProperties=nullptr) override
Populate a list of LIB_SYMBOL alias names contained within the library aLibraryPath.
CONNECT_STATUS connect()
Connects, and waits out a file another instance is opening at the same moment when the library never ...
DIALOG_SHIM * CreateConfigurationDialog(wxWindow *aParent, const wxString &aLibraryPath) override
std::shared_mutex m_cacheMutex
PREVIEW_ISSUES * m_previewIssues
Collects the problems found while building a preview; null outside BuildPreviewSymbols()
std::mutex m_connectMutex
std::shared_ptr< DATABASE_CONNECTION > m_conn
Generally will be null if no valid connection is established.
void CancelLoad() override
Ends the waits of a load in progress and of any that starts before ResumeLoad(), so closing a project...
static wxString SymbolNameForRow(const DATABASE_LIB_TABLE &aTable, bool aGloballyUniqueKeys, const DATABASE_CONNECTION::ROW &aRow, std::string *aRawKey=nullptr)
void GetDefaultSymbolFields(std::vector< wxString > &aNames) override
Retrieves a list of (custom) field names that should be shown by default for this library in the symb...
void loadSettingsBounded(int aTimeoutSeconds)
Loads the settings file on a thread that can be abandoned, so a share that went away cannot hold the ...
void ensureSettings(const wxString &aSettingsPath)
static thread_local std::unordered_set< wxString > s_inProgressLoads
LIB_IDs whose resolution is in flight on the calling thread, used to break self-referential cycles wh...
std::shared_ptr< DATABASE_CONNECTION > currentConnection()
Copies the connection under the cache lock so the caller keeps it alive across a reconnect.
std::atomic< bool > m_loadCancelled
Set by CancelLoad(), cleared by ResumeLoad()
static std::vector< std::unique_ptr< LIB_SYMBOL > > BuildPreviewSymbols(SYMBOL_LIBRARY_ADAPTER *aLibs, const DATABASE_LIB_TABLE &aTable, bool aGloballyUniqueKeys, const std::vector< DATABASE_CONNECTION::ROW > &aRows, PREVIEW_ISSUES *aIssues)
Builds symbols from already-fetched rows exactly as the library would name and load them,...
std::atomic< bool > m_cachePopulated
True once the LIB_SYMBOL cache has been materialized at least once.
bool m_inCacheLib
Re-entrancy guard for cacheLib(), tripped when a self-referential load routes back through the adapte...
std::map< wxString, std::unique_ptr< LIB_SYMBOL > > m_nameToSymbolcache
std::map< wxString, std::pair< std::string, std::string > > m_sanitizedNameMap
CONNECT_STATUS connectOnce()
One attempt at connect(), with m_connectMutex held.
std::set< wxString > m_defaultShownFields
virtual ~SCH_IO_DATABASE()
void cacheLib(const std::shared_ptr< DATABASE_CONNECTION > &aConn)
size_t m_cacheSignature
Signature of the raw database rows at last materialization; used to skip rebuilding the LIB_SYMBOL ca...
std::vector< std::pair< const DATABASE_LIB_TABLE *, std::vector< DATABASE_CONNECTION::ROW > > > TABLE_RESULT_LIST
bool connectionStopped() const
static CONNECTION_RESULT CreateConnection(const DATABASE_LIB_SETTINGS &aSettings, bool aConnectNow=true, const wxString &aBaseDir=wxEmptyString)
Opens a connection described by aSettings, expanding ${CWD} in a connection string to the directory h...
void GetSubLibraryNames(std::vector< wxString > &aNames) override
Retrieves a list of sub-libraries in this library.
static std::chrono::milliseconds busyRetryWait(long long aMaxAgeSeconds, int aAttempt, std::minstd_rand &aRng)
std::unique_ptr< DATABASE_LIB_SETTINGS > m_settings
static std::optional< bool > boolFromAny(const std::any &aVal)
size_t computeSignature(const TABLE_RESULT_LIST &aTableResults) const
void CheckLibrary(const wxString &aLibraryPath, const std::map< std::string, UTF8 > *aProperties=nullptr) override
Validate that the library at aLibraryPath is reachable and well-formed, without necessarily loading s...
SYMBOL_LIBRARY_ADAPTER * m_adapter
std::thread m_refreshThread
static constexpr int MAX_BUSY_ATTEMPT
Busy retries past this attempt stop doubling the window.
long long m_cacheTimestamp
std::condition_variable m_refreshCV
void GetAvailableSymbolFields(std::vector< wxString > &aNames) override
Retrieves a list of (custom) field names that are present on symbols in this library.
std::atomic< bool > m_refreshRunning
bool m_previewMode
Set on the throwaway instance behind BuildPreviewSymbols()
std::atomic< bool > m_stopping
bool materializeCache(const TABLE_RESULT_LIST &aTableResults, std::map< wxString, std::unique_ptr< LIB_SYMBOL > > &aSymbolCache, std::map< wxString, std::pair< std::string, std::string > > &aSanitizedNameMap)
std::unique_ptr< LIB_SYMBOL > loadSymbolFromRow(const wxString &aSymbolName, const DATABASE_LIB_TABLE &aTable, const DATABASE_CONNECTION::ROW &aRow)
static constexpr std::chrono::seconds FIRST_CONNECT_BUDGET
The race this covers lasts milliseconds, so a file in use for longer has a real holder and is reporte...
static CONNECTION_RESULT connectionFor(DB_BACKEND_RESULT &&aBackend, bool aConnectNow)
Wraps a backend in a connection, or reports why there is none.
void ResumeLoad() noexcept override
Called after the loads that were cancelled have ended, to clear what CancelLoad() left behind.
std::set< wxString > m_customFields
std::mutex m_symbolLoadMutex
LIB_SYMBOL * resolveSourceSymbol(const LIB_ID &aId)
Resolves one entry of a row's symbols column through the library adapter.
std::shared_ptr< DATABASE_CONNECTION > ensureConnection()
Returns a live connection or throws.
std::vector< PREVIEW_ISSUE > PREVIEW_ISSUES
LIB_SYMBOL * LoadSymbol(const wxString &aLibraryPath, const wxString &aAliasName, const std::map< std::string, UTF8 > *aProperties=nullptr) override
Load a LIB_SYMBOL object having aPartName from the aLibraryPath containing a library format that this...
std::mutex m_refreshMutex
static SCH_FILE_T EnumFromStr(const wxString &aFileType)
Return the file type from the corresponding plugin type name: "kicad", "legacy", etc.
SCH_IO(const wxString &aName)
Definition sch_io.h:406
Base class for any item which can be embedded within the SCHEMATIC container class,...
Definition sch_item.h:170
SCH_ITEM * Duplicate(bool addToParentGroup, SCH_COMMIT *aCommit=nullptr, bool aDoClone=false) const
Routine to create a new copy of given item.
Definition sch_item.cpp:191
virtual void SetBodyStyle(int aBodyStyle)
Definition sch_item.h:252
An interface to the global shared library manager that is schematic-specific and linked to one projec...
static const char * PropPowerSymsOnly
An 8 bit string that is assuredly encoded in UTF8, and supplies special conversion support to and fro...
Definition utf8.h:67
const char * c_str() const
Definition utf8.h:104
wxString wx_str() const
Definition utf8.cpp:41
@ INVALID_TYPE
The source type is not one KiCad knows.
@ UNREACHABLE
Nothing answered at the server address, or it refused the connection.
@ NOT_SUPPORTED_ON_PLATFORM
No backend for the source type exists on this platform.
@ FILE_IN_USE
The database file is locked by another process.
int DatabaseFileWaitSeconds(int aTimeoutSeconds)
#define _(s)
#define IGNORE_PARENT_GROUP
Definition eda_item.h:55
std::chrono::milliseconds FileInUseBackoff(int aAttempt, std::minstd_rand &aRng)
nlohmann::json json
Definition gerbview.cpp:50
std::filesystem::path ToFsPath(const wxString &aPath)
Give aPath to std::filesystem without a detour through the C locale.
Definition gestfich.cpp:977
#define THROW_IO_ERROR(msg)
macro which captures the "call site" values of FILE_, __FUNCTION & LINE
#define THROW_IO_ERRORF(msg,...)
const wxChar *const traceDatabase
Flag to enable database library tracing.
static constexpr void hash_combine(std::size_t &seed)
This is a dummy function to take the final case of hash_combine below.
Definition hash.h:28
std::string source
PIN_MAP MakeLegacyPinMap(const wxString &aName, const std::unordered_map< wxString, std::vector< wxString > > &aAssignments)
Build a single named PIN_MAP from a legacy symbol-pin to footprint-pad(s) assignment table (the flat ...
Definition pin_map.cpp:151
std::vector< ASSOCIATED_FOOTPRINT > ParseAssociatedFootprints(const nlohmann::json &aParent)
Parse the spec-form footprint associations from aParent["associated_footprints"] (issue #2282).
Definition pin_map.cpp:254
std::unordered_map< wxString, std::vector< wxString > > ParseLegacyPinAssignments(const nlohmann::json &aArray)
Parse the legacy flat pin-assignment JSON array (MR !2540 form) into a symbol-pin to footprint-pad(s)...
Definition pin_map.cpp:187
PIN_MAP_SET ParsePinMapSet(const nlohmann::json &aParent)
Parse the spec-form named maps from aParent["pin_maps"] into a PIN_MAP_SET (issue #2282).
Definition pin_map.cpp:221
SEVERITY
@ RPT_SEVERITY_WARNING
@ RPT_SEVERITY_ERROR
static std::string strValue(double aValue)
static wxString cwdDirFor(const DATABASE_LIB_SETTINGS &aSettings, const wxString &aBaseDir)
The directory ${CWD} stands for, which is the settings file's unless the caller supplies one.
What a failed query left behind, copied before another thread's query can replace it.
bool visible_in_chooser
Whether the column is shown by default in the chooser.
std::string column
Database column name.
bool inherit_properties
Whether or not to inherit properties from symbol field.
bool visible_on_add
Whether to show the field when placing the symbol.
bool show_name
Whether or not to show the field name as well as its value.
wxString name_wx
KiCad field name (converted)
A database library table will be mapped to a sub-library provided by the database library entry in th...
std::string key_col
Unique key column name (will form part of the LIB_ID)
std::string name
KiCad library nickname (will form part of the LIB_ID)
std::string symbols_col
Column name containing KiCad symbol refs.
std::string footprints_col
Column name containing KiCad footprint refs.
std::vector< DATABASE_FIELD_MAPPING > fields
std::string table
Database table to pull content from.
MAPPABLE_SYMBOL_PROPERTIES properties
std::string pins_col
Column name containing JSON pin assignments (optional)
The outcome of building a backend; backend is null when failure and error say why.
A connection built from settings, or the reason none could be built.
std::unique_ptr< DATABASE_CONNECTION > connection
Null when no backend exists for the source type.
DB_FAILURE failure
Why connection is null.
std::string error
The message naming the problem when connection is null.
Why the last connect() attempt failed, for the caller to report from its own copy.
#define ALL_BODY_STYLES
Definition symbol.h:151
#define MANDATORY_FIELDS
FIELD_T
The set of all field indices assuming an array like sequence that a SCH_COMPONENT or LIB_PART can hol...
@ USER
The field ID hasn't been set yet; field is invalid.
static std::vector< uint8_t > bytes(const std::string &aText)
const std::string table
std::string path
wxString result
Test unit parsing edge cases and error handling.
@ SCH_FIELD_T
Definition typeinfo.h:146