KiCad PCB EDA Suite
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages Concepts
altium_binary_parser.cpp
Go to the documentation of this file.
1/*
2 * This program source code file is part of KiCad, a free EDA CAD application.
3 *
4 * Copyright (C) 2019-2020 Thomas Pointhuber <thomas.pointhuber@gmx.at>
5 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 2
10 * of the License, or (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, you may find one here:
19 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
20 * or you may search the http://www.gnu.org website for the version 2 license,
21 * or you may write to the Free Software Foundation, Inc.,
22 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
23 */
24
26#include "altium_parser_utils.h"
27
28#include <compoundfilereader.h>
29#include <charconv>
30#include <ki_exception.h>
31#include <math/util.h>
32#include <numeric>
33#include <sstream>
34#include <utf.h>
35#include <wx/log.h>
36#include <wx/translation.h>
37
38
39// Helper for debug logging
40std::string FormatPath( const std::vector<std::string>& aVectorPath )
41{
42 return std::accumulate( aVectorPath.cbegin(), aVectorPath.cend(), std::string(),
43 []( const std::string& ss, const std::string& s )
44 {
45 return ss.empty() ? s : ss + '\\' + s;
46 } );
47}
48
49
51{
52}
53
54
56{
57 // Open file
58 FILE* fp = wxFopen( aFilePath, "rb" );
59
60 if( fp == nullptr )
61 {
62 THROW_IO_ERROR( wxString::Format( _( "Cannot open file '%s'." ), aFilePath ) );
63 }
64
65 fseek( fp, 0, SEEK_END );
66 long len = ftell( fp );
67
68 if( len < 0 )
69 {
70 fclose( fp );
71 THROW_IO_ERROR( _( "Error reading file: cannot determine length." ) );
72 }
73
74 // Read into buffer (TODO: add support for memory-mapped files to avoid this copy!)
75 m_buffer.resize( len );
76
77 fseek( fp, 0, SEEK_SET );
78
79 size_t bytesRead = fread( m_buffer.data(), sizeof( unsigned char ), len, fp );
80 fclose( fp );
81
82 if( static_cast<size_t>( len ) != bytesRead )
83 {
84 THROW_IO_ERROR( _( "Error reading file." ) );
85 }
86
87 try
88 {
89 m_reader = std::make_unique<CFB::CompoundFileReader>( m_buffer.data(), m_buffer.size() );
90 }
91 catch( CFB::CFBException& exception )
92 {
93 THROW_IO_ERROR( exception.what() );
94 }
95}
96
97
98ALTIUM_COMPOUND_FILE::ALTIUM_COMPOUND_FILE( const void* aBuffer, size_t aLen )
99{
100 InitFromBuffer( aBuffer, aLen );
101}
102
103
104void ALTIUM_COMPOUND_FILE::InitFromBuffer( const void* aBuffer, size_t aLen )
105{
106 m_buffer.resize( aLen );
107 memcpy( m_buffer.data(), aBuffer, aLen );
108
109 try
110 {
111 m_reader = std::make_unique<CFB::CompoundFileReader>( m_buffer.data(), m_buffer.size() );
112 }
113 catch( CFB::CFBException& exception )
114 {
115 THROW_IO_ERROR( exception.what() );
116 }
117}
118
119
120bool ALTIUM_COMPOUND_FILE::DecodeIntLibStream( const CFB::COMPOUND_FILE_ENTRY& cfe,
121 ALTIUM_COMPOUND_FILE* aOutput )
122{
123 wxCHECK( aOutput, false );
124 wxCHECK( cfe.size >= 1, false );
125
126 size_t streamSize = cfe.size;
127 wxMemoryBuffer buffer( streamSize );
128 buffer.SetDataLen( streamSize );
129
130 // read file into buffer
131 GetCompoundFileReader().ReadFile( &cfe, 0, reinterpret_cast<char*>( buffer.GetData() ),
132 streamSize );
133
134 // 0x02: compressed stream, 0x00: uncompressed
135 if( buffer[0] == 0x02 )
136 {
137 wxMemoryInputStream memoryInputStream( buffer.GetData(), streamSize );
138 memoryInputStream.SeekI( 1, wxFromStart );
139
140 wxZlibInputStream zlibInputStream( memoryInputStream );
141 wxMemoryOutputStream decodedPcbLibStream;
142 decodedPcbLibStream << zlibInputStream;
143
144 wxStreamBuffer* outStream = decodedPcbLibStream.GetOutputStreamBuffer();
145 aOutput->InitFromBuffer( outStream->GetBufferStart(), outStream->GetIntPosition() );
146 return true;
147 }
148 else if( buffer[0] == 0x00 )
149 {
150 aOutput->InitFromBuffer( static_cast<uint8_t*>( buffer.GetData() ) + 1, streamSize - 1 );
151 return true;
152 }
153 else
154 {
155 wxFAIL_MSG( wxString::Format( "Altium IntLib unknown header: %02x %02x %02x %02x %02x",
156 buffer[0], buffer[1], buffer[2], buffer[3], buffer[4] ) );
157 }
158
159 return false;
160}
161
162
163const CFB::COMPOUND_FILE_ENTRY*
164ALTIUM_COMPOUND_FILE::FindStreamSingleLevel( const CFB::COMPOUND_FILE_ENTRY* aEntry,
165 const std::string aName, const bool aIsStream ) const
166{
167 if( !m_reader || !aEntry )
168 return nullptr;
169
170 const CFB::COMPOUND_FILE_ENTRY* ret = nullptr;
171
172 m_reader->EnumFiles( aEntry, 1,
173 [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string& dir,
174 int level ) -> int
175 {
176 if( ret != nullptr )
177 return 1;
178
179 if( m_reader->IsStream( entry ) == aIsStream )
180 {
181 std::string name = UTF16ToUTF8( entry->name );
182 if( name == aName.c_str() )
183 {
184 ret = entry;
185 return 1;
186 }
187 }
188
189 return 0;
190 } );
191 return ret;
192}
193
194
195std::map<wxString, ALTIUM_SYMBOL_DATA>
196ALTIUM_COMPOUND_FILE::GetLibSymbols( const CFB::COMPOUND_FILE_ENTRY* aStart ) const
197{
198 const CFB::COMPOUND_FILE_ENTRY* root = aStart ? aStart : m_reader->GetRootEntry();
199
200 if( !root )
201 return {};
202
203 std::map<wxString, ALTIUM_SYMBOL_DATA> folders;
204
205 m_reader->EnumFiles( root, 1, [&]( const CFB::COMPOUND_FILE_ENTRY* tentry,
206 const CFB::utf16string&, int ) -> int
207 {
208 wxString dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
209
210 if( m_reader->IsStream( tentry ) )
211 return 0;
212
213 m_reader->EnumFiles( tentry, 1,
214 [&]( const CFB::COMPOUND_FILE_ENTRY* entry,
215 const CFB::utf16string&, int ) -> int
216 {
217 std::wstring fileName = UTF16ToWstring( entry->name, entry->nameLen );
218
219 if( m_reader->IsStream( entry ) && fileName == L"Data" )
220 folders[dirName].m_symbol = entry;
221
222 if( m_reader->IsStream( entry ) && fileName == L"PinFrac" )
223 folders[dirName].m_pinsFrac = entry;
224
225 if( m_reader->IsStream( entry ) && fileName == L"PinWideText" )
226 folders[dirName].m_pinsWideText = entry;
227
228 if( m_reader->IsStream( entry ) && fileName == L"PinTextData" )
229 folders[dirName].m_pinsTextData = entry;
230
231 return 0;
232 } );
233
234 return 0;
235 } );
236
237 return folders;
238}
239
240
241std::map<wxString, const CFB::COMPOUND_FILE_ENTRY*>
242ALTIUM_COMPOUND_FILE::EnumDir( const std::wstring& aDir ) const
243{
244 const CFB::COMPOUND_FILE_ENTRY* root = m_reader->GetRootEntry();
245
246 if( !root )
247 return {};
248
249 std::map<wxString, const CFB::COMPOUND_FILE_ENTRY*> files;
250
251 m_reader->EnumFiles(
252 root, 1,
253 [&]( const CFB::COMPOUND_FILE_ENTRY* tentry, const CFB::utf16string& dir,
254 int level ) -> int
255 {
256 if( m_reader->IsStream( tentry ) )
257 return 0;
258
259 std::wstring dirName = UTF16ToWstring( tentry->name, tentry->nameLen );
260
261 if( dirName != aDir )
262 return 0;
263
264 m_reader->EnumFiles(
265 tentry, 1,
266 [&]( const CFB::COMPOUND_FILE_ENTRY* entry, const CFB::utf16string&,
267 int ) -> int
268 {
269 if( m_reader->IsStream( entry ) )
270 {
271 std::wstring fileName =
272 UTF16ToWstring( entry->name, entry->nameLen );
273
274 files[fileName] = entry;
275 }
276
277 return 0;
278 } );
279 return 0;
280 } );
281
282 return files;
283}
284
285
286const CFB::COMPOUND_FILE_ENTRY*
287ALTIUM_COMPOUND_FILE::FindStream( const CFB::COMPOUND_FILE_ENTRY* aStart,
288 const std::vector<std::string>& aStreamPath ) const
289{
290 if( !m_reader )
291 return nullptr;
292
293 if( !aStart )
294 aStart = m_reader->GetRootEntry();
295
296 auto it = aStreamPath.cbegin();
297
298 while( aStart != nullptr )
299 {
300 const std::string& name = *it;
301
302 if( ++it == aStreamPath.cend() )
303 {
304 const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, true );
305 return ret;
306 }
307 else
308 {
309 const CFB::COMPOUND_FILE_ENTRY* ret = FindStreamSingleLevel( aStart, name, false );
310 aStart = ret;
311 }
312 }
313
314 return nullptr;
315}
316
317
318const CFB::COMPOUND_FILE_ENTRY*
319ALTIUM_COMPOUND_FILE::FindStream( const std::vector<std::string>& aStreamPath ) const
320{
321 return FindStream( nullptr, aStreamPath );
322}
323
324
326 const CFB::COMPOUND_FILE_ENTRY* aEntry )
327{
328 m_subrecord_end = nullptr;
329 m_size = static_cast<size_t>( aEntry->size );
330 m_error = false;
331 m_content.reset( new char[m_size] );
332 m_pos = m_content.get();
333
334 // read file into buffer
335 aFile.GetCompoundFileReader().ReadFile( aEntry, 0, m_content.get(), m_size );
336}
337
338
339ALTIUM_BINARY_PARSER::ALTIUM_BINARY_PARSER( std::unique_ptr<char[]>& aContent, size_t aSize )
340{
341 m_subrecord_end = nullptr;
342 m_size = aSize;
343 m_error = false;
344 m_content = std::move( aContent );
345 m_pos = m_content.get();
346}
347
348
349std::map<wxString, wxString> ALTIUM_BINARY_PARSER::ReadProperties(
350 std::function<std::map<wxString, wxString>( const std::string& )> handleBinaryData )
351{
352 // TSAN reports calling this wx macro is not thread-safe
353 static wxCSConv convISO8859_1 = wxConvISO8859_1;
354
355 std::map<wxString, wxString> kv;
356
357 uint32_t length = Read<uint32_t>();
358 bool isBinary = ( length & 0xff000000 ) != 0;
359
360 length &= 0x00ffffff;
361
362 if( length > GetRemainingBytes() )
363 {
364 m_error = true;
365 return kv;
366 }
367
368 if( length == 0 )
369 {
370 return kv;
371 }
372
373 // There is one case by kliment where Board6 ends with "|NEARDISTANCE=1000mi".
374 // Both the 'l' and the null-byte are missing, which looks like Altium swallowed two bytes.
375 bool hasNullByte = m_pos[length - 1] == '\0';
376
377 if( !hasNullByte && !isBinary )
378 {
379 wxLogTrace( "ALTIUM", wxT( "Missing null byte at end of property list. Imported data "
380 "might be malformed or missing." ) );
381 }
382
383 // we use std::string because std::string can handle NULL-bytes
384 // wxString would end the string at the first NULL-byte
385 std::string str = std::string( m_pos, length - ( hasNullByte ? 1 : 0 ) );
386 m_pos += length;
387
388 if( isBinary )
389 {
390 return handleBinaryData( str );
391 }
392
393 std::size_t token_end = 0;
394
395 while( token_end < str.size() && token_end != std::string::npos )
396 {
397 std::size_t token_start = str.find( '|', token_end );
398 std::size_t token_equal = str.find( '=', token_end );
399 std::size_t key_start;
400
401 if( token_start <= token_equal )
402 {
403 key_start = token_start + 1;
404 }
405 else
406 {
407 // Leading "|" before "RECORD=28" may be missing in older schematic versions.
408 key_start = token_end;
409 }
410
411 token_end = str.find( '|', key_start );
412
413 if( token_equal >= token_end )
414 {
415 continue; // this looks like an error: skip the entry. Also matches on std::string::npos
416 }
417
418 if( token_end == std::string::npos )
419 {
420 token_end = str.size() + 1; // this is the correct offset
421 }
422
423 std::string keyS = str.substr( key_start, token_equal - key_start );
424 std::string valueS = str.substr( token_equal + 1, token_end - token_equal - 1 );
425
426 // convert the strings to wxStrings, since we use them everywhere
427 // value can have non-ASCII characters, so we convert them from LATIN1/ISO8859-1
428 wxString key( keyS.c_str(), convISO8859_1 );
429
430 // Altium stores keys either in Upper, or in CamelCase. Lets unify it.
431 wxString canonicalKey = key.Trim( false ).Trim( true ).MakeUpper();
432
433 // If the key starts with '%UTF8%' we have to parse the value using UTF8
434 wxString value;
435
436 if( canonicalKey.StartsWith( "%UTF8%" ) )
437 value = wxString( valueS.c_str(), wxConvUTF8 );
438 else
439 value = wxString( valueS.c_str(), convISO8859_1 );
440
441 if( canonicalKey != wxS( "PATTERN" ) && canonicalKey != wxS( "SOURCEFOOTPRINTLIBRARY" ) )
442 {
443 // Breathless hack because I haven't a clue what the story is here (but this character
444 // appears in a lot of radial dimensions and is rendered by Altium as a space).
445 value.Replace( wxT( "ÿ" ), wxT( " " ) );
446 }
447
448 if( canonicalKey == wxT( "DESIGNATOR" )
449 || canonicalKey == wxT( "NAME" )
450 || canonicalKey == wxT( "TEXT" ) )
451 {
452 if( kv[ wxT( "RECORD" ) ] != wxT( "4" ) )
453 value = AltiumPropertyToKiCadString( value.Trim() );
454 }
455
456 kv.insert( { canonicalKey, value.Trim() } );
457 }
458
459 return kv;
460}
const char * name
Definition: DXF_plotter.cpp:59
std::string FormatPath(const std::vector< std::string > &aVectorPath)
Helper for debug logging (vector -> string)
wxString AltiumPropertyToKiCadString(const wxString &aString)
size_t GetRemainingBytes() const
std::unique_ptr< char[]> m_content
ALTIUM_BINARY_PARSER(const ALTIUM_COMPOUND_FILE &aFile, const CFB::COMPOUND_FILE_ENTRY *aEntry)
std::map< wxString, wxString > ReadProperties(std::function< std::map< wxString, wxString >(const std::string &)> handleBinaryData=[](const std::string &) { return std::map< wxString, wxString >();})
ALTIUM_COMPOUND_FILE()
Create an uninitialized file for two-step initialization (e.g. with InitFromBuffer)
const CFB::CompoundFileReader & GetCompoundFileReader() const
const CFB::COMPOUND_FILE_ENTRY * FindStreamSingleLevel(const CFB::COMPOUND_FILE_ENTRY *aEntry, const std::string aName, const bool aIsStream) const
std::map< wxString, const CFB::COMPOUND_FILE_ENTRY * > EnumDir(const std::wstring &aDir) const
void InitFromBuffer(const void *aBuffer, size_t aLen)
Load a CFB file from memory; may throw an IO_ERROR.
std::vector< char > m_buffer
std::unique_ptr< CFB::CompoundFileReader > m_reader
std::map< wxString, ALTIUM_SYMBOL_DATA > GetLibSymbols(const CFB::COMPOUND_FILE_ENTRY *aStart) const
bool DecodeIntLibStream(const CFB::COMPOUND_FILE_ENTRY &cfe, ALTIUM_COMPOUND_FILE *aOutput)
const CFB::COMPOUND_FILE_ENTRY * FindStream(const std::vector< std::string > &aStreamPath) const
#define _(s)
#define THROW_IO_ERROR(msg)
Definition: ki_exception.h:39
#define kv