test_expand_text_vars.cpp

Go to the documentation of this file.

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
 
#define BOOST_TEST_NO_MAIN
#include <boost/test/unit_test.hpp>
#include <atomic>
#include <filesystem>
#include <optional>
#include <thread>
#include <vector>
#include <common.h>
#include <env_paths.h>
#include <jobs/job_export_pcb_stats.h>
#include <pgm_base.h>
#include <settings/environment.h>
#include <title_block.h>
#include <wx/filename.h>
#include <wx/utils.h>

struct ExpandTextVarsFixture
{
    // Simple resolver that maps VAR->value, X->5, Y->2
    std::function<bool( wxString* )> resolver = []( wxString* token ) -> bool
    {
        if( *token == wxT( "VAR" ) )
        {
            *token = wxT( "value" );
            return true;
        }
        else if( *token == wxT( "X" ) )
        {
            *token = wxT( "5" );
            return true;
        }
        else if( *token == wxT( "Y" ) )
        {
            *token = wxT( "2" );
            return true;
        }
 
        return false;
    };
};
 
BOOST_FIXTURE_TEST_SUITE( ExpandTextVarsTests, ExpandTextVarsFixture )
 
// Basic variable expansion
BOOST_AUTO_TEST_CASE( SimpleVariable )
{
    wxString result = ExpandTextVars( wxT( "${VAR}" ), &resolver );
    BOOST_CHECK( result == wxT( "value" ) );
}
 
// Multiple variables in one string
BOOST_AUTO_TEST_CASE( MultipleVariables )
{
    wxString result = ExpandTextVars( wxT( "${X}+${Y}" ), &resolver );
    BOOST_CHECK( result == wxT( "5+2" ) );
}
 
// Escaped variable should produce escape marker (not expanded)
BOOST_AUTO_TEST_CASE( EscapedVariable )
{
    wxString result = ExpandTextVars( wxT( "\\${VAR}" ), &resolver );
    // The escape marker should be in the output
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ) );
}
 
// Escaped variable followed by regular variable - both should be processed correctly
BOOST_AUTO_TEST_CASE( EscapedThenRegularVariable )
{
    wxString result = ExpandTextVars( wxT( "\\${literal}${VAR}" ), &resolver );
    // Should have escape marker for literal, and "value" for VAR
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ) );
    BOOST_CHECK( result.Contains( wxT( "value" ) ) );
}
 
// Regular variable followed by escaped variable
BOOST_AUTO_TEST_CASE( RegularThenEscapedVariable )
{
    wxString result = ExpandTextVars( wxT( "${VAR}\\${literal}" ), &resolver );
    // Should have "value" for VAR and escape marker for literal
    BOOST_CHECK( result.StartsWith( wxT( "value" ) ) );
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ) );
}
 
// Issue 22497: Escaped variable inside math expression should not prevent other expansions
// This is the key test case for the bug fix
BOOST_AUTO_TEST_CASE( EscapedInsideMathExpression )
{
    // First pass: @{\${X}+${Y}} should become @{<<<ESC_DOLLAR:X}+2}
    // Second pass: the marker should be preserved and +2 should NOT be lost
    wxString result = ExpandTextVars( wxT( "@{\\${X}+${Y}}" ), &resolver );
 
    // The result should contain the escape marker
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ),
                         "Expected escape marker in result" );
 
    // The result should also contain +2 (the expanded Y variable)
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "+2" ) ),
                         "Expected '+2' (from ${Y} expansion) in result" );
 
    // The result should be @{<<<ESC_DOLLAR:X}+2}
    BOOST_CHECK( result == wxT( "@{<<<ESC_DOLLAR:X}+2}" ) );
}
 
// Nested escaped variable in regular variable reference
BOOST_AUTO_TEST_CASE( EscapedInsideVariableReference )
{
    // ${prefix\${suffix}} - looking up variable with literal ${suffix} in name
    // This should try to resolve "prefix\${suffix}" which won't resolve,
    // but the recursive expansion should convert \${suffix} to the marker
    wxString result = ExpandTextVars( wxT( "${prefix\\${suffix}}" ), &resolver );
 
    // The unresolved reference should be preserved with escape marker
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ) );
}
 
// Multiple escape markers in a math expression
BOOST_AUTO_TEST_CASE( MultipleEscapedInMathExpression )
{
    wxString result = ExpandTextVars( wxT( "@{\\${A}+\\${B}+${Y}}" ), &resolver );
 
    // Should have two escape markers and the expanded Y (2)
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "+2" ) ),
                         "Expected '+2' (from ${Y} expansion) in: " + result );
 
    // Count escape markers (should be 2)
    int dollarCount = 0;
    size_t pos = 0;
 
    while( ( pos = result.find( wxT( "<<<ESC_DOLLAR:" ), pos ) ) != wxString::npos )
    {
        dollarCount++;
        pos += 14;
    }
 
    BOOST_CHECK_EQUAL( dollarCount, 2 );
}
 
// Math expression with escaped @ sign
BOOST_AUTO_TEST_CASE( EscapedAtInExpression )
{
    wxString result = ExpandTextVars( wxT( "${VAR}\\@{literal}" ), &resolver );
 
    // Should have "value" for VAR and escape marker for @{literal}
    BOOST_CHECK( result.StartsWith( wxT( "value" ) ) );
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_AT:" ) ) );
}
 
// Escaped followed by escaped (both should be preserved)
BOOST_AUTO_TEST_CASE( ConsecutiveEscaped )
{
    wxString result = ExpandTextVars( wxT( "\\${A}\\${B}" ), &resolver );
 
    // Should have two escape markers
    int dollarCount = 0;
    size_t pos = 0;
 
    while( ( pos = result.find( wxT( "<<<ESC_DOLLAR:" ), pos ) ) != wxString::npos )
    {
        dollarCount++;
        pos += 14;
    }
 
    BOOST_CHECK_EQUAL( dollarCount, 2 );
}
 
// Issue 23599: backslash path separator before text variable should NOT be treated as an escape.
// This test documents the ExpandTextVars behavior: \${ IS treated as an escape at this level.
// The fix is applied at call sites that deal with file paths, which normalize backslashes to
// forward slashes before calling ExpandTextVars.
BOOST_AUTO_TEST_CASE( BackslashBeforeVariableIsEscape )
{
    wxString result = ExpandTextVars( wxT( "subdir\\${VAR}_file.txt" ), &resolver );
 
    // ExpandTextVars treats \${ as an escape, so VAR is NOT expanded
    BOOST_CHECK( result.Contains( wxT( "<<<ESC_DOLLAR:" ) ) );
}
 
 
// With forward slashes the variable is expanded normally
BOOST_AUTO_TEST_CASE( ForwardSlashBeforeVariableExpands )
{
    wxString result = ExpandTextVars( wxT( "subdir/${VAR}_file.txt" ), &resolver );
 
    BOOST_CHECK( result == wxT( "subdir/value_file.txt" ) );
}
 
BOOST_AUTO_TEST_SUITE_END()
 
 
// Issue 23599: JOB::ResolveOutputPath must expand text variables even when preceded by backslash
// path separators. This suite tests the fix in ResolveOutputPath that normalizes backslashes
// before calling ExpandTextVars.
BOOST_AUTO_TEST_SUITE( JobResolveOutputPath )
 
BOOST_AUTO_TEST_CASE( BackslashPathSeparatorBeforeTextVar )
{
    JOB_EXPORT_PCB_STATS job;
 
    TITLE_BLOCK titleBlock;
    titleBlock.SetRevision( wxT( "RevA" ) );
    job.SetTitleBlock( titleBlock );
 
    // Simulates Windows path with text variable immediately after backslash separator
    wxString path = wxT( "Board Stats\\${REVISION}_Stats.txt" );
    wxString result = job.ResolveOutputPath( path, false, nullptr );
 
    // The variable should be expanded, not escaped
    BOOST_CHECK_MESSAGE( !result.Contains( wxT( "<<<ESC_DOLLAR:" ) ),
                         "Text variable after backslash path separator should not be escaped. Got: "
                                 + result );
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "RevA" ) ),
                         "Expected resolved REVISION in path. Got: " + result );
}
 
 
BOOST_AUTO_TEST_CASE( BackslashPathSeparatorBeforeMultipleTextVars )
{
    JOB_EXPORT_PCB_STATS job;
 
    TITLE_BLOCK titleBlock;
    titleBlock.SetRevision( wxT( "B" ) );
    titleBlock.SetComment( 0, wxT( "DWG-001" ) );
    job.SetTitleBlock( titleBlock );
 
    // The COMMENT1 variable maps to Comment(0) in TITLE_BLOCK
    wxString path = wxT( "Output\\${COMMENT1}_${REVISION}_file.txt" );
    wxString result = job.ResolveOutputPath( path, false, nullptr );
 
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "DWG-001" ) ),
                         "Expected COMMENT1 expanded in path. Got: " + result );
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "_B_" ) ),
                         "Expected REVISION expanded in path. Got: " + result );
}
 
 
BOOST_AUTO_TEST_CASE( TextVarNotFirstInFilename )
{
    JOB_EXPORT_PCB_STATS job;
 
    TITLE_BLOCK titleBlock;
    titleBlock.SetRevision( wxT( "C" ) );
    job.SetTitleBlock( titleBlock );
 
    // When there's literal text between the backslash and the variable, it always worked
    wxString path = wxT( "Output\\Generated_${REVISION}_file.txt" );
    wxString result = job.ResolveOutputPath( path, false, nullptr );
 
    BOOST_CHECK_MESSAGE( result.Contains( wxT( "Generated_C_file.txt" ) ),
                         "Expected variable expansion with preceding literal text. Got: " + result );
}
 
BOOST_AUTO_TEST_SUITE_END()
 
 

struct OverlappingEnvVarsFixture
{
    wxString rootDir;
    wxString outerDir;
    wxString innerDir;
    wxString targetDir;
 
    OverlappingEnvVarsFixture()
    {
        std::filesystem::path tmp = std::filesystem::temp_directory_path() /
                                     "kicad_qa_overlap_env_vars";
        std::error_code       ec;
        std::filesystem::remove_all( tmp, ec );
        std::filesystem::create_directories( tmp / "V10" / "symbols", ec );
 
        rootDir   = wxString::FromUTF8( tmp.string() );
        outerDir  = rootDir;
        innerDir  = wxString::FromUTF8( ( tmp / "V10" ).string() );
        targetDir = wxString::FromUTF8( ( tmp / "V10" / "symbols" ).string() );
 
        wxSetEnv( wxS( "KICAD_QA_3RD_PARTY_OUTER" ), outerDir );
        wxSetEnv( wxS( "KICAD_QA_USER_LIB_INNER" ), innerDir );
    }
 
    ~OverlappingEnvVarsFixture()
    {
        wxUnsetEnv( wxS( "KICAD_QA_3RD_PARTY_OUTER" ) );
        wxUnsetEnv( wxS( "KICAD_QA_USER_LIB_INNER" ) );
 
        std::filesystem::path tmp = std::filesystem::temp_directory_path() /
                                     "kicad_qa_overlap_env_vars";
        std::error_code ec;
        std::filesystem::remove_all( tmp, ec );
    }
 
    ENV_VAR_MAP BuildEnvMap() const
    {
        ENV_VAR_MAP map;
        map[wxS( "KICAD_QA_3RD_PARTY_OUTER" )] = ENV_VAR_ITEM( outerDir );
        map[wxS( "KICAD_QA_USER_LIB_INNER" )]  = ENV_VAR_ITEM( innerDir );
        return map;
    }
};
 
 
BOOST_FIXTURE_TEST_SUITE( OverlappingEnvVarPaths, OverlappingEnvVarsFixture )
 
 
BOOST_AUTO_TEST_CASE( NormalizePicksLongestPrefix )
{
    wxFileName target( targetDir, wxS( "test.kicad_sym" ) );
    ENV_VAR_MAP envMap = BuildEnvMap();
 
    wxString normalized = NormalizePath( target, &envMap, wxEmptyString );
 
    // NormalizePath should pick KICAD_QA_USER_LIB_INNER because it is a deeper match.
    BOOST_CHECK_MESSAGE(
            normalized == wxS( "${KICAD_QA_USER_LIB_INNER}/symbols/test.kicad_sym" ),
            wxString::Format( wxS( "Expected '%s' but got '%s'" ),
                              wxS( "${KICAD_QA_USER_LIB_INNER}/symbols/test.kicad_sym" ),
                              normalized ) );
}
 
 
BOOST_AUTO_TEST_CASE( RoundTripPreservesAbsolutePath )
{
    wxFileName target( targetDir, wxS( "test.kicad_sym" ) );
    ENV_VAR_MAP envMap = BuildEnvMap();
 
    wxString normalized = NormalizePath( target, &envMap, wxEmptyString );
    wxString expanded   = ExpandEnvVarSubstitutions( normalized, nullptr );
 
    wxFileName expandedFn( expanded );
    expandedFn.Normalize( wxPATH_NORM_DOTS | wxPATH_NORM_ABSOLUTE );
 
    wxFileName originalFn( target );
    originalFn.Normalize( wxPATH_NORM_DOTS | wxPATH_NORM_ABSOLUTE );
 
    BOOST_CHECK_MESSAGE(
            expandedFn.GetFullPath() == originalFn.GetFullPath(),
            wxString::Format(
                    wxS( "Round-trip mismatch: normalized='%s' expanded='%s' original='%s'" ),
                    normalized, expandedFn.GetFullPath(), originalFn.GetFullPath() ) );
}
 
 
BOOST_AUTO_TEST_SUITE_END()
 
 

BOOST_AUTO_TEST_SUITE( TextVarExpressionEvaluatorConcurrency )
 
BOOST_AUTO_TEST_CASE( ParallelResolveTextVarsWithMathExpressions )
{
    std::function<bool( wxString* )> resolver = []( wxString* token ) -> bool
    {
        if( *token == wxT( "#" ) )
        {
            *token = wxT( "3" );
            return true;
        }
 
        if( *token == wxT( "ROW" ) )
        {
            *token = wxT( "4" );
            return true;
        }
 
        return false;
    };
 
    const std::vector<wxString> inputs = {
        wxT( "Out@{(${#}-2)*8+0}" ),
        wxT( "Net_@{${ROW}*2+1}" ),
        wxT( "@{(2-2)*8+0}" ),
        wxT( "${ROW}:@{${ROW}*${ROW}}" ),
        wxT( "plain_label_no_expr" ),
        wxT( "@{1+1}_@{2+2}_@{3+3}" ),
    };
 
    const unsigned int numThreads = std::max( 4u, std::thread::hardware_concurrency() );
    const int          iterations = 2000;
 
    std::atomic<bool>        failed{ false };
    std::atomic<int>         totalRuns{ 0 };
    std::vector<std::thread> threads;
    threads.reserve( numThreads );
 
    for( unsigned int t = 0; t < numThreads; ++t )
    {
        threads.emplace_back(
                [&, t]()
                {
                    try
                    {
                        for( int i = 0; i < iterations; ++i )
                        {
                            const wxString& src = inputs[( t + i ) % inputs.size()];
                            int             depth = 0;
                            wxString        result = ResolveTextVars( src, &resolver, depth );
                            (void) result;
                            totalRuns.fetch_add( 1, std::memory_order_relaxed );
                        }
                    }
                    catch( ... )
                    {
                        failed.store( true, std::memory_order_relaxed );
                    }
                } );
    }
 
    for( auto& th : threads )
        th.join();
 
    BOOST_CHECK( !failed.load() );
    BOOST_CHECK_EQUAL( totalRuns.load(), static_cast<int>( numThreads ) * iterations );
}
 
BOOST_AUTO_TEST_SUITE_END()
 
 

struct EnvVarRecursiveExpansionFixture
{
    wxString                innerPath;
    std::optional<wxString> oldInner;
    std::optional<wxString> oldOuter;
 
    EnvVarRecursiveExpansionFixture()
    {
        wxString existing;
 
        if( wxGetEnv( wxS( "KICAD_QA_INNER" ), &existing ) )
            oldInner = existing;
 
        if( wxGetEnv( wxS( "KICAD_QA_OUTER" ), &existing ) )
            oldOuter = existing;
 
        innerPath = wxString::FromUTF8(
                ( std::filesystem::temp_directory_path() / "kicad-qa-24244" ).generic_string() );
 
        wxSetEnv( wxS( "KICAD_QA_INNER" ), innerPath );
        wxSetEnv( wxS( "KICAD_QA_OUTER" ), wxS( "${KICAD_QA_INNER}/templates" ) );
    }
 
    ~EnvVarRecursiveExpansionFixture()
    {
        if( oldInner )
            wxSetEnv( wxS( "KICAD_QA_INNER" ), *oldInner );
        else
            wxUnsetEnv( wxS( "KICAD_QA_INNER" ) );
 
        if( oldOuter )
            wxSetEnv( wxS( "KICAD_QA_OUTER" ), *oldOuter );
        else
            wxUnsetEnv( wxS( "KICAD_QA_OUTER" ) );
    }
};
 
BOOST_FIXTURE_TEST_SUITE( EnvVarRecursiveExpansion, EnvVarRecursiveExpansionFixture )
 
BOOST_AUTO_TEST_CASE( ExpandsNestedReferences )
{
    wxString rawValue;
    BOOST_REQUIRE( wxGetEnv( wxS( "KICAD_QA_OUTER" ), &rawValue ) );
 
    // The raw value should still contain the unexpanded reference.
    BOOST_CHECK( rawValue.Contains( wxS( "${KICAD_QA_INNER}" ) ) );
 
    wxString expanded = ExpandEnvVarSubstitutions( rawValue, nullptr );
    wxString expected = innerPath + wxS( "/templates" );
 
    // After expansion the inner reference must be resolved to its concrete path.
    BOOST_CHECK_MESSAGE( expanded == expected,
                         wxString::Format( wxS( "Expected '%s', got '%s'" ), expected, expanded ) );
}
 
 
BOOST_AUTO_TEST_CASE( UndefinedReferenceLeavesLiteralMarker )
{
    // If a referenced variable is undefined, ExpandEnvVarSubstitutions preserves the
    // original token.  Callers that then mkdir the result would create a literal
    // "${MISSING}" directory; production code must detect this and bail out.
    wxUnsetEnv( wxS( "KICAD_QA_INNER" ) );
 
    wxString rawValue;
    BOOST_REQUIRE( wxGetEnv( wxS( "KICAD_QA_OUTER" ), &rawValue ) );
 
    wxString expanded = ExpandEnvVarSubstitutions( rawValue, nullptr );
    BOOST_CHECK( expanded.Contains( wxS( "${" ) ) );
 
    // Restore so the fixture destructor sees a known state.
    wxSetEnv( wxS( "KICAD_QA_INNER" ), innerPath );
}
 
BOOST_AUTO_TEST_SUITE_END()