KiCad PCB EDA Suite
Loading...
Searching...
No Matches
easyedapro_v3_parser.cpp
Go to the documentation of this file.
1/*
2 * This program source code file is part of KiCad, a free EDA CAD application.
3 *
4 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, you may find one here:
18 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
19 * or you may search the http://www.gnu.org website for the version 2 license,
20 * or you may write to the Free Software Foundation, Inc.,
21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
22 */
23
25
26#include <algorithm>
27#include <cmath>
28#include <optional>
29#include <set>
30#include <string_view>
31
33
35#include <json_common.h>
36#include <ki_exception.h>
37
38#include <wx/filename.h>
39#include <wx/log.h>
40#include <wx/mstream.h>
41#include <wx/wfstream.h>
42#include <wx/zipstrm.h>
43
44
47
48
49static std::string ToStdString( const wxString& aStr )
50{
51 return std::string( aStr.ToUTF8() );
52}
53
54
55namespace EASYEDAPRO
56{
57
58wxString V3JsonToString( const nlohmann::json& aValue, const wxString& aDefault )
59{
60 if( aValue.is_string() )
61 return aValue.get<wxString>();
62
63 if( aValue.is_number_integer() )
64 return wxString::Format( "%lld", static_cast<long long>( aValue.get<int64_t>() ) );
65
66 if( aValue.is_number_unsigned() )
67 return wxString::Format( "%llu", static_cast<unsigned long long>( aValue.get<uint64_t>() ) );
68
69 if( aValue.is_number_float() )
70 return wxString::FromDouble( aValue.get<double>() );
71
72 if( aValue.is_boolean() )
73 return aValue.get<bool>() ? wxS( "true" ) : wxS( "false" );
74
75 return aDefault;
76}
77
78
79wxString V3GetString( const nlohmann::json& aObj, const char* aKey, const wxString& aDefault )
80{
81 if( !aObj.is_object() )
82 return aDefault;
83
84 auto it = aObj.find( aKey );
85
86 if( it == aObj.end() )
87 return aDefault;
88
89 return V3JsonToString( *it, aDefault );
90}
91
92
93wxString V3GetString( const nlohmann::json& aObj, const wxString& aKey, const wxString& aDefault )
94{
95 return V3GetString( aObj, ToStdString( aKey ).c_str(), aDefault );
96}
97
98
99double V3GetDouble( const nlohmann::json& aObj, const char* aKey, double aDefault )
100{
101 if( !aObj.is_object() )
102 return aDefault;
103
104 auto it = aObj.find( aKey );
105
106 if( it == aObj.end() )
107 return aDefault;
108
109 if( it->is_number() )
110 return it->get<double>();
111
112 if( it->is_string() )
113 {
114 double value;
115
116 if( it->get<wxString>().ToCDouble( &value ) )
117 return value;
118 }
119
120 return aDefault;
121}
122
123
124int V3GetInt( const nlohmann::json& aObj, const char* aKey, int aDefault )
125{
126 if( !aObj.is_object() )
127 return aDefault;
128
129 auto it = aObj.find( aKey );
130
131 if( it == aObj.end() )
132 return aDefault;
133
134 if( it->is_number_integer() )
135 return it->get<int>();
136
137 if( it->is_number_float() )
138 return static_cast<int>( std::lround( it->get<double>() ) );
139
140 if( it->is_boolean() )
141 return it->get<bool>() ? 1 : 0;
142
143 if( it->is_string() )
144 {
145 long value = 0;
146
147 if( it->get<wxString>().ToLong( &value ) )
148 return static_cast<int>( value );
149 }
150
151 return aDefault;
152}
153
154
155bool V3GetBool( const nlohmann::json& aObj, const char* aKey, bool aDefault )
156{
157 if( !aObj.is_object() )
158 return aDefault;
159
160 auto it = aObj.find( aKey );
161
162 if( it == aObj.end() )
163 return aDefault;
164
165 if( it->is_boolean() )
166 return it->get<bool>();
167
168 if( it->is_number() )
169 return it->get<double>() != 0.0;
170
171 if( it->is_string() )
172 {
173 wxString s = it->get<wxString>();
174
175 if( s.CmpNoCase( wxS( "true" ) ) == 0 )
176 return true;
177
178 if( s.CmpNoCase( wxS( "false" ) ) == 0 )
179 return false;
180
181 long value = 0;
182
183 if( s.ToLong( &value ) )
184 return value != 0;
185 }
186
187 return aDefault;
188}
189
190
191bool V3IsNullOrMissing( const nlohmann::json& aObj, const char* aKey )
192{
193 if( !aObj.is_object() )
194 return true;
195
196 auto it = aObj.find( aKey );
197
198 return it == aObj.end() || it->is_null();
199}
200
201
202nlohmann::json V3ParseIdArray( const wxString& aId )
203{
204 if( aId.empty() )
205 return nlohmann::json();
206
207 try
208 {
209 return nlohmann::json::parse( ToStdString( aId ) );
210 }
211 catch( ... )
212 {
213 return nlohmann::json();
214 }
215}
216
217} // namespace EASYEDAPRO
218
219
220namespace
221{
222
223// Helper functions for internal use
224static inline wxString GetString( const nlohmann::json& aObj, const char* aKey,
225 const wxString& aDefault = wxEmptyString )
226{
227 return EASYEDAPRO::V3GetString( aObj, aKey, aDefault );
228}
229
230static inline int GetInt( const nlohmann::json& aObj, const char* aKey, int aDefault = 0 )
231{
232 return EASYEDAPRO::V3GetInt( aObj, aKey, aDefault );
233}
234
235
236static wxString ParseDocType( const nlohmann::json& aHead )
237{
238 return GetString( aHead, "docType" );
239}
240
241
242static wxString ParseDocUuid( const nlohmann::json& aHead )
243{
244 return GetString( aHead, "uuid" );
245}
246
247
248static std::string ReadEntryText( wxInputStream& aInput )
249{
250 wxMemoryOutputStream mem;
251 mem << aInput;
252
253 wxStreamBuffer* buf = mem.GetOutputStreamBuffer();
254 return std::string( static_cast<const char*>( buf->GetBufferStart() ), buf->GetBufferSize() );
255}
256
257
258static std::string_view NextEpruLine( std::string_view& aText )
259{
260 size_t eol = aText.find_first_of( "\r\n" );
261 std::string_view line = aText.substr( 0, eol );
262
263 if( eol == std::string_view::npos )
264 {
265 aText = {};
266 return line;
267 }
268
269 bool crlf = aText[eol] == '\r' && eol + 1 < aText.size() && aText[eol + 1] == '\n';
270 aText.remove_prefix( eol + ( crlf ? 2 : 1 ) );
271
272 return line;
273}
274
275
276static bool SplitEpruLine( std::string_view aLine, std::string_view& aOuter, std::string_view& aInner )
277{
278 if( !aLine.empty() && aLine.back() == '|' )
279 aLine.remove_suffix( 1 );
280
281 size_t sepPos = aLine.find( "||" );
282
283 if( sepPos == std::string_view::npos )
284 return false;
285
286 aOuter = aLine.substr( 0, sepPos );
287 aInner = aLine.substr( sepPos + 2 );
288
289 return true;
290}
291
292
293static void ParseEpruStream( wxInputStream& aInput, const wxString& aSource,
294 std::vector<V3_DOC_RAW>& aDocs )
295{
296 std::string text = ReadEntryText( aInput );
297 std::string_view remaining = text;
298 int currentLine = 1;
299 std::optional<V3_DOC_RAW> currentDoc;
300
301 auto flushDoc = [&]()
302 {
303 if( currentDoc )
304 {
305 // EasyEDA Pro encodes deleted objects as rows with an empty inner
306 // payload. Deduplication already keeps the highest-ticket row, so
307 // any null winner is a deletion tombstone and must not reach
308 // document consumers, which dereference the inner JSON directly.
309 std::vector<V3_ROW> keptRows;
310 keptRows.reserve( currentDoc->rows.size() );
311
312 for( V3_ROW& row : currentDoc->rows )
313 {
314 if( !row.inner.is_null() )
315 keptRows.push_back( std::move( row ) );
316 }
317
318 currentDoc->rows = std::move( keptRows );
319 currentDoc->rowById.clear();
320
321 for( size_t i = 0; i < currentDoc->rows.size(); ++i )
322 {
323 if( !currentDoc->rows[i].id.empty() )
324 currentDoc->rowById[currentDoc->rows[i].id] = i;
325 }
326
327 aDocs.push_back( std::move( *currentDoc ) );
328 currentDoc.reset();
329 }
330 };
331
332 while( !remaining.empty() )
333 {
334 std::string_view outerStr;
335 std::string_view innerStr;
336
337 if( !SplitEpruLine( NextEpruLine( remaining ), outerStr, innerStr ) )
338 {
339 currentLine++;
340 continue;
341 }
342
343 try
344 {
345 nlohmann::json outer = nlohmann::json::parse( outerStr );
346 nlohmann::json inner;
347
348 if( !innerStr.empty() )
349 inner = nlohmann::json::parse( innerStr );
350
351 wxString type = GetString( outer, "type" );
352
353 if( type == wxS( "DOCHEAD" ) )
354 {
355 flushDoc();
356
357 V3_DOC_RAW doc;
358 doc.head = inner;
359 doc.docType = ParseDocType( inner );
360 doc.uuid = ParseDocUuid( inner );
361
362 if( doc.docType.empty() || doc.uuid.empty() )
363 THROW_IO_ERRORF( _( "Invalid DOCHEAD in '%s' at line %d" ), aSource, currentLine );
364
365 currentDoc = std::move( doc );
366 currentLine++;
367 continue;
368 }
369
370 if( !currentDoc )
371 THROW_IO_ERRORF( _( "Expected DOCHEAD in '%s' at line %d" ), aSource, currentLine );
372
373 V3_ROW row;
374 row.outer = std::move( outer );
375 row.inner = std::move( inner );
376 row.type = type;
377 row.id = GetString( row.outer, "id" );
378 row.ticket = GetInt( row.outer, "ticket", 0 );
379
380 wxString dedupId = row.id;
381
382 if( dedupId.empty() )
383 dedupId = wxString::Format( "__line_%d", currentLine );
384
385 auto it = currentDoc->rowById.find( dedupId );
386
387 if( it == currentDoc->rowById.end() )
388 {
389 currentDoc->rowById[dedupId] = currentDoc->rows.size();
390 currentDoc->rows.emplace_back( std::move( row ) );
391 }
392 else if( row.ticket >= currentDoc->rows[it->second].ticket )
393 {
394 currentDoc->rows[it->second] = std::move( row );
395 }
396 }
397 catch( nlohmann::json::exception& e )
398 {
399 THROW_IO_ERRORF( _( "JSON error in '%s' at line %d: %s" ), aSource, currentLine, e.what() );
400 }
401
402 currentLine++;
403 }
404
405 flushDoc();
406}
407
408
409struct V3_LIBRARY_CONTENTS
410{
411 bool hasIndex = false;
412 bool hasElibu = false;
413 std::set<wxString> docTypes;
414};
415
416
417static bool HasValidV3LibraryContents( const V3_LIBRARY_CONTENTS& aContents, const wxString& aRequiredDocType )
418{
419 if( !aContents.hasIndex || !aContents.hasElibu )
420 return false;
421
422 if( aRequiredDocType.empty() )
423 return true;
424
425 return aContents.docTypes.contains( aRequiredDocType );
426}
427
428
429static void ScanEpruDocTypes( wxInputStream& aInput, const wxString& aSource, std::set<wxString>& aDocTypes )
430{
431 std::string text = ReadEntryText( aInput );
432 std::string_view remaining = text;
433 int currentLine = 1;
434
435 while( !remaining.empty() )
436 {
437 std::string_view outerStr;
438 std::string_view innerStr;
439
440 if( !SplitEpruLine( NextEpruLine( remaining ), outerStr, innerStr ) )
441 {
442 currentLine++;
443 continue;
444 }
445
446 try
447 {
448 nlohmann::json outer = nlohmann::json::parse( outerStr );
449
450 if( GetString( outer, "type" ) == wxS( "DOCHEAD" ) && !innerStr.empty() )
451 {
452 nlohmann::json inner = nlohmann::json::parse( innerStr );
453 wxString docType = ParseDocType( inner );
454
455 if( !docType.empty() )
456 aDocTypes.insert( docType );
457 }
458 }
459 catch( nlohmann::json::exception& e )
460 {
461 THROW_IO_ERRORF( _( "JSON error in '%s' at line %d: %s" ), aSource, currentLine, e.what() );
462 }
463
464 currentLine++;
465 }
466}
467
468
469static nlohmann::json ReadJsonEntry( wxInputStream& aInput, const wxString& aEntryName )
470{
471 try
472 {
473 return nlohmann::json::parse( ReadEntryText( aInput ) );
474 }
475 catch( nlohmann::json::exception& e )
476 {
477 THROW_IO_ERRORF( _( "JSON error reading '%s': %s" ), aEntryName, e.what() );
478 }
479}
480
481
482static void MergeLibraryIndex( nlohmann::json& aTarget, nlohmann::json&& aIndex )
483{
484 if( !aIndex.is_object() )
485 return;
486
487 for( auto& [key, value] : aIndex.items() )
488 {
489 if( aTarget.contains( key ) && aTarget[key].is_object() && value.is_object() )
490 {
491 for( auto& [itemKey, itemValue] : value.items() )
492 aTarget[key][itemKey] = std::move( itemValue );
493 }
494 else if( !aTarget.contains( key ) || !value.is_object() || !value.empty() )
495 {
496 aTarget[key] = std::move( value );
497 }
498 }
499}
500
501
502static V3_LIBRARY_CONTENTS ScanV3LibraryArchive( const wxString& aFileName, const wxString& aRequiredDocType )
503{
504 V3_LIBRARY_CONTENTS contents;
505
506 std::shared_ptr<wxZipEntry> entry;
507 wxFFileInputStream in( aFileName );
508 wxZipInputStream zip( in );
509
510 if( !zip.IsOk() )
511 THROW_IO_ERRORF( _( "Cannot read ZIP archive '%s'" ), aFileName );
512
513 while( entry.reset( zip.GetNextEntry() ), entry.get() )
514 {
515 wxString entryName = entry->GetName();
516 wxString baseName = entryName.AfterLast( '\\' ).AfterLast( '/' ).Lower();
517
518 if( baseName == wxS( "symbol2.json" ) || baseName == wxS( "footprint2.json" )
519 || baseName == wxS( "device2.json" ) )
520 {
521 contents.hasIndex = true;
522 }
523 else if( baseName.EndsWith( wxS( ".elibu" ) ) )
524 {
525 contents.hasElibu = true;
526
527 if( !aRequiredDocType.empty() )
528 ScanEpruDocTypes( zip, entryName, contents.docTypes );
529 }
530
531 if( HasValidV3LibraryContents( contents, aRequiredDocType ) )
532 break;
533 }
534
535 return contents;
536}
537
538
539static std::vector<V3_DOC_RAW> ParseV3DocsFromArchive( const wxString& aFileName, nlohmann::json& aProject2 )
540{
541 std::vector<V3_DOC_RAW> docs;
542
543 bool hasProject2 = false;
544 bool hasEpru = false;
545
546 std::shared_ptr<wxZipEntry> entry;
547 wxFFileInputStream in( aFileName );
548 wxZipInputStream zip( in );
549
550 if( !zip.IsOk() )
551 THROW_IO_ERRORF( _( "Cannot read ZIP archive '%s'" ), aFileName );
552
553 while( entry.reset( zip.GetNextEntry() ), entry.get() )
554 {
555 wxString entryName = entry->GetName();
556 wxString baseName = entryName.AfterLast( '\\' ).AfterLast( '/' );
557
558 if( baseName.CmpNoCase( wxS( "project2.json" ) ) == 0 )
559 {
560 aProject2 = ReadJsonEntry( zip, entryName );
561 hasProject2 = true;
562 }
563 else if( baseName.Lower().EndsWith( wxS( ".epru" ) ) )
564 {
565 hasEpru = true;
566 ParseEpruStream( zip, entryName, docs );
567 }
568 }
569
570 if( !hasProject2 || !hasEpru )
571 {
572 THROW_IO_ERRORF( _( "'%s' does not appear to be a valid EasyEDA (JLCEDA) Pro v3 project. "
573 "Cannot find project2.json and .epru documents." ), aFileName );
574 }
575
576 return docs;
577}
578
579
580static std::vector<V3_DOC_RAW> ParseV3DocsFromLibraryArchive( const wxString& aFileName, nlohmann::json& aLibraryIndex )
581{
582 std::vector<V3_DOC_RAW> docs;
583 V3_LIBRARY_CONTENTS contents;
584
585 std::shared_ptr<wxZipEntry> entry;
586 wxFFileInputStream in( aFileName );
587 wxZipInputStream zip( in );
588
589 if( !zip.IsOk() )
590 THROW_IO_ERRORF( _( "Cannot read ZIP archive '%s'" ), aFileName );
591
592 aLibraryIndex = nlohmann::json::object();
593
594 while( entry.reset( zip.GetNextEntry() ), entry.get() )
595 {
596 wxString entryName = entry->GetName();
597 wxString baseName = entryName.AfterLast( '\\' ).AfterLast( '/' );
598 wxString lowerBaseName = baseName.Lower();
599
600 if( lowerBaseName == wxS( "symbol2.json" ) || lowerBaseName == wxS( "footprint2.json" )
601 || lowerBaseName == wxS( "device2.json" ) )
602 {
603 nlohmann::json index = ReadJsonEntry( zip, entryName );
604
605 MergeLibraryIndex( aLibraryIndex, std::move( index ) );
606
607 contents.hasIndex = true;
608 }
609 else if( lowerBaseName.EndsWith( wxS( ".elibu" ) ) )
610 {
611 contents.hasElibu = true;
612 ParseEpruStream( zip, entryName, docs );
613
614 for( const V3_DOC_RAW& doc : docs )
615 contents.docTypes.insert( doc.docType );
616 }
617 }
618
619 if( !HasValidV3LibraryContents( contents, wxEmptyString ) )
620 {
621 THROW_IO_ERRORF( _( "'%s' does not appear to be a valid EasyEDA (JLCEDA) Pro v3 library. Cannot find "
622 "symbol2.json, footprint2.json or device2.json and .elibu documents." ),
623 aFileName );
624 }
625
626 return docs;
627}
628
629
630} // namespace
631
632
633namespace EASYEDAPRO
634{
635
636V3_DOC_PARSER::V3_DOC_PARSER( const wxString& aFileName ) :
637 m_fileName( aFileName )
638{
639}
640
641
642bool V3_DOC_PARSER::IsV3Archive( const wxString& aFileName )
643{
644 wxFileName fn( aFileName );
645 wxString ext = fn.GetExt().Lower();
646
647 if( ext != wxS( "epro2" ) && ext != wxS( "zip" ) )
648 return false;
649
650 try
651 {
652 std::shared_ptr<wxZipEntry> entry;
653 wxFFileInputStream in( aFileName );
654 wxZipInputStream zip( in );
655
656 if( !zip.IsOk() )
657 return false;
658
659 bool hasProject2 = false;
660 bool hasEpru = false;
661
662 while( entry.reset( zip.GetNextEntry() ), entry.get() != NULL )
663 {
664 wxString entryName = entry->GetName();
665 wxString baseName = entryName.AfterLast( '\\' ).AfterLast( '/' );
666
667 if( baseName.CmpNoCase( wxS( "project2.json" ) ) == 0 )
668 hasProject2 = true;
669 else if( baseName.Lower().EndsWith( wxS( ".epru" ) ) )
670 hasEpru = true;
671
672 if( hasProject2 && hasEpru )
673 return true;
674 }
675
676 return false;
677 }
678 catch( ... )
679 {
680 return false;
681 }
682}
683
684
685bool V3_DOC_PARSER::IsV3Library( const wxString& aFileName, const wxString& aRequiredDocType )
686{
687 wxFileName fn( aFileName );
688
689 if( fn.GetExt().Lower() != wxS( "elibz2" ) )
690 return false;
691
692 try
693 {
694 V3_LIBRARY_CONTENTS contents = ScanV3LibraryArchive( aFileName, aRequiredDocType );
695 return HasValidV3LibraryContents( contents, aRequiredDocType );
696 }
697 catch( ... )
698 {
699 return false;
700 }
701}
702
703
705{
706 m_rawDocs.clear();
707 m_project2 = nlohmann::json();
708 m_libraryIndex = nlohmann::json();
709 m_skippedCount = 0;
710
711 std::vector<V3_DOC_RAW> rawDocs = ParseV3DocsFromArchive( m_fileName, m_project2 );
712
713 for( V3_DOC_RAW& raw : rawDocs )
714 {
715 m_rawDocs[raw.docType][raw.uuid] = std::move( raw );
716 }
717}
718
719
721{
722 m_rawDocs.clear();
723 m_project2 = nlohmann::json();
724 m_libraryIndex = nlohmann::json();
725 m_skippedCount = 0;
726
727 std::vector<V3_DOC_RAW> rawDocs = ParseV3DocsFromLibraryArchive( m_fileName, m_libraryIndex );
728
729 for( V3_DOC_RAW& raw : rawDocs )
730 m_rawDocs[raw.docType][raw.uuid] = std::move( raw );
731}
732
733
734const std::map<wxString, V3_DOC_RAW>& V3_DOC_PARSER::GetRawDocs( const wxString& aDocType ) const
735{
736 static const std::map<wxString, V3_DOC_RAW> empty;
737
738 auto it = m_rawDocs.find( aDocType );
739
740 if( it == m_rawDocs.end() )
741 return empty;
742
743 return it->second;
744}
745
746
747const V3_DOC_RAW* V3_DOC_PARSER::FindRawDoc( const wxString& aDocType, const wxString& aUuid ) const
748{
749 auto typeIt = m_rawDocs.find( aDocType );
750
751 if( typeIt == m_rawDocs.end() )
752 return nullptr;
753
754 auto docIt = typeIt->second.find( aUuid );
755
756 if( docIt == typeIt->second.end() )
757 return nullptr;
758
759 return &docIt->second;
760}
761
762} // namespace EASYEDAPRO
int index
void Load()
Load all v3 docs from the archive.
const V3_DOC_RAW * FindRawDoc(const wxString &aDocType, const wxString &aUuid) const
const std::map< wxString, V3_DOC_RAW > & GetRawDocs(const wxString &aDocType) const
static bool IsV3Library(const wxString &aFileName, const wxString &aRequiredDocType=wxEmptyString)
Return true if aFileName looks like an EasyEDA Pro v3 library archive.
std::map< wxString, std::map< wxString, V3_DOC_RAW > > m_rawDocs
V3_DOC_PARSER(const wxString &aFileName)
static bool IsV3Archive(const wxString &aFileName)
Return true if aFileName looks like an EasyEDA Pro v3 archive.
void LoadLibrary()
Load all v3 docs from a library archive.
static bool empty(const wxTextEntryBase *aCtrl)
static std::string ToStdString(const wxString &aStr)
#define _(s)
#define THROW_IO_ERRORF(msg,...)
bool V3GetBool(const nlohmann::json &aObj, const char *aKey, bool aDefault)
nlohmann::json V3ParseIdArray(const wxString &aId)
bool V3IsNullOrMissing(const nlohmann::json &aObj, const char *aKey)
double V3GetDouble(const nlohmann::json &aObj, const char *aKey, double aDefault)
wxString V3JsonToString(const nlohmann::json &aValue, const wxString &aDefault)
int V3GetInt(const nlohmann::json &aObj, const char *aKey, int aDefault)
wxString V3GetString(const nlohmann::json &aObj, const char *aKey, const wxString &aDefault)
Raw parsed document from an EasyEDA Pro v3 .epru stream.
One parsed row from an EasyEDA Pro v3 .epru document stream.