fields_table_data_model.h

Go to the documentation of this file.

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright The KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software: you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
 
#pragma once
 
#include <set>
#include <unordered_set>
#include <vector>
#include <algorithm>
#include <map>
#include <span>
 
#include <widgets/wx_grid.h>
#include <widgets/ui_common.h>
 
#include <common.h>
#include <eda_pattern_match.h>
#include <kiid.h>
#include <refdes_utils.h>
#include <settings/bom_settings.h>
#include <string_utils.h>
#include <wx/debug.h>
 
 
namespace FIELDS_TABLE_COLOR
{
inline const wxColour VARIANT_SYMBOL_OVERRIDE_DARK_BLUE( 40, 60, 80 );
inline const wxColour VARIANT_SYMBOL_OVERRIDE_LIGHT_BLUE( 220, 235, 255 );
inline const wxColour VARIANT_FIELD_OVERRIDE_DARK_YELLOW( 80, 80, 40 );
inline const wxColour VARIANT_FIELD_OVERRIDE_LIGHT_YELLOW( 255, 255, 200 );
inline const wxColour TEXT_VARIABLE_DARK_AMBER( 80, 70, 30 );
inline const wxColour TEXT_VARIABLE_LIGHT_YELLOW( 255, 252, 200 );
inline const wxColour STRIPED_CLEARED_EMPTY_FIELD_LIGHT_GREEN( 180, 220, 180 );
inline const wxColour STRIPED_CLEARED_NONEMPTY_FIELD_LIGHT_RED( 220, 180, 180 );
inline const wxColour STRIPED_EDITED_EMPTY_FIELD_MUTED_GREEN( 180, 200, 180 );
inline const wxColour STRIPED_EDITED_NONEMPTY_FIELD_MUTED_RED( 200, 180, 180 );
inline const wxColour EDITED_EMPTY_FIELD_BRIGHT_GREEN( 192, 255, 192 );
inline const wxColour CLEARED_FIELD_STRIPE_ON_DARK_LIGHT_RED( 220, 180, 180 );
inline const wxColour CLEARED_FIELD_STRIPE_ON_LIGHT_DARK_RED( 100, 10, 10 );
} // namespace FIELDS_TABLE_COLOR
 


 
struct DATA_MODEL_COL
{
    wxString m_fieldName;
    wxString m_label;
    bool     m_userAdded;
    bool     m_show;
    bool     m_group;
};
 
 
struct FIELD_VALUE_STATE
{
    wxString m_value;
    wxString m_baseline;
};
 
 
struct FIELD_STORE_VALUE
{
    // Field existence is shared by every variant; an empty value is still a present field.
    bool                                  m_present = false;
    bool                                  m_baselinePresent = false;
    std::map<wxString, FIELD_VALUE_STATE> m_variants;
};
 
 
template <typename ITEM_TYPE>
struct DATA_MODEL_ROW
{
    DATA_MODEL_ROW( const ITEM_TYPE& aFirstItem, ROW_STATE aGroupingState )
    {
        m_itemNumber = 0;
        m_items.push_back( aFirstItem );
        m_state = aGroupingState;
    }

    std::span<const ITEM_TYPE> GetCellItems() const
    {
        std::span<const ITEM_TYPE> items( m_items );
        return IsParentRow( m_state ) ? items.first( 1 ) : items;
    }
 
    int                   m_itemNumber;
    ROW_STATE             m_state;
    // All group members; real-parent states require the parent to remain first.
    std::vector<ITEM_TYPE> m_items;
};
 

class FIELDS_TABLE_DATA_MODEL_BASE : public WX_GRID_TABLE_BASE
{
public:
    FIELDS_TABLE_DATA_MODEL_BASE();
    ~FIELDS_TABLE_DATA_MODEL_BASE() override;
 
    static const wxString QUANTITY_VARIABLE;
    static const wxString ITEM_NUMBER_VARIABLE;
 
    bool IsEdited() { return m_edited; }
 
    virtual void AddColumn( const wxString& aFieldName, const wxString& aLabel, bool aAddedByUser ) = 0;
 
    void MoveColumn( int aCol, int aNewPos );
    void RemoveColumn( int aCol );
    void RenameColumn( int aCol, const wxString& newName );
 
    int GetNumberCols() override { return static_cast<int>( m_cols.size() ); }
 
    void     SetColLabelValue( int aCol, const wxString& aLabel ) override;
    wxString GetColLabelValue( int aCol ) override;
    wxString GetColFieldName( int aCol );
    int      GetColDataWidth( int aCol );
    int      GetFieldNameCol( const wxString& aFieldName ) const;
 
    std::vector<BOM_FIELD> GetFieldsOrdered();
    void                   SetFieldsOrder( const std::vector<wxString>& aNewOrder );
 
    bool IsEmptyCell( int aRow, int aCol ) override
    {
        return false; // don't allow adjacent cell overflow, even if we are actually empty
    }
 
    bool ColIsReference( int aCol ) const;
    bool ColIsQuantity( int aCol ) const;
    bool ColIsItemNumber( int aCol ) const;
    bool ColIsValue( int aCol ) const;
    bool ColIsFootprint( int aCol ) const;
    // Whether a column is something like DNP, e.g. one of the checkboxes in the Attributes section
    // of symbols/footprints
    bool ColIsAttribute( int aCol ) const;
    // Whether a column is an attribute, or a special property like ${SYMBOL_NAME}
    bool ColIsItemProperty( int aCol ) const;
    // Whether a column is magic variable that is calculated or resolved, e.g. ${QUANTITY}, ${ITEM_NUMBER},
    // but not something like ${SYMBOL_NAME}.
    // The important part is that the column’s value is derived at read time rather than staged as stored data
    bool ColIsComputed( int aCol ) const;

    virtual bool ColIsItemIdentifier( int aCol ) const { return ColIsReference( aCol ); }
 
    virtual bool ColIsReadOnly( int aCol ) const;
    bool         HasRowLabelExpanders() const override { return true; }
    virtual bool IsCellReadOnly( int aRow, int aCol );
 
    void SetSorting( int aCol, bool aAscending );
    int  GetSortCol() { return m_sortColumn; }
    bool GetSortAsc() { return m_sortAscending; }
 
    // These are used to disable RebuildRows() while generating batches of UI events, e.g.
    // applying a BOM preset, that would otherwise thrash the grid.
    void         EnableRebuilds();
    void         DisableRebuilds();
    virtual void RebuildRows() = 0;
 
    void             SetFilter( const wxString& aFilter ) { m_filter = aFilter; }
    const wxString&  GetFilter() { return m_filter; }
    void             SetFilterScope( BOM_FILTER_SCOPE aScope ) { m_filterScope = aScope; }
    BOM_FILTER_SCOPE GetFilterScope() const { return m_filterScope; }
 
    void SetSelectionItems( const std::unordered_set<KIID_PATH>& aItems ) { m_selectionItems = aItems; }
    void ClearSelectionItems() { m_selectionItems.clear(); }
 
    void SetGroupingEnabled( bool aGroup ) { m_groupingEnabled = aGroup; }
    bool GetGroupingEnabled() { return m_groupingEnabled; }
 
    /* These contradictorily named functions force including items that have the Exclude from
     * BOM flag set.  This is needed so they can be viewed in a fields table while still being
     * excluded from BOM export.
     */
    void SetIncludeExcludedFromBOM( bool aInclude ) { m_includeExcluded = aInclude; }
    bool GetIncludeExcludedFromBOM() { return m_includeExcluded; }
 
    void SetExcludeDNP( bool aExclude ) { m_excludeDNP = aExclude; }
    bool GetExcludeDNP() { return m_excludeDNP; }
 
    void SetGroupColumn( int aCol, bool aGroup );
    bool GetGroupColumn( int aCol );
 
    void SetShowColumn( int aCol, bool aShow );
    bool GetShowColumn( int aCol );
 
    void       ApplyBomPreset( const BOM_PRESET& aPreset );
    BOM_PRESET GetBomSettings();
    wxString   Export( const BOM_FMT_PRESET& aSettings );
 
    virtual wxString GetResolvedValue( int aRow, int aCol ) = 0;
    virtual wxString GetExportValue( int aRow, int aCol, const wxString& aRefDelimiter,
                                     const wxString& aRefRangeDelimiter ) = 0;
 
    virtual void ClearCell( int aRow, int aCol ) = 0;
    bool         CanClearCell( int aRow, int aCol );
    bool         CanCopyCellFromVariant( int aRow, int aCol );
    virtual void CopyCellFromVariant( int aRow, int aCol, const wxString& aVariantName ) = 0;
    virtual bool IsCellClear( int aRow, int aCol ) = 0;
    virtual bool IsCellEdited( int aRow, int aCol ) = 0;
    bool         IsRowEdited( int aRow );
    virtual void RevertRow( int aRow ) = 0;

    virtual std::vector<KIID_PATH> GetRowItemKeys( int aRow ) const = 0;

    virtual void    SetCurrentVariant( const wxString& aVariantName ) = 0;
    const wxString& GetCurrentVariant() const { return m_currentVariant; }
 
    void RenameStoredVariant( const wxString& aOldName, const wxString& aNewName );
    void DeleteStoredVariant( const wxString& aName );
 
    void SetVariantNames( const std::vector<wxString>& aVariantNames ) { m_variantNames = aVariantNames; }
    const std::vector<wxString>& GetVariantNames() const { return m_variantNames; }
 
    // Identity-based undo serialization (keyed by symbol, not row position) for the dialog's
    // Ctrl+Z, so it stays correct as rows are grouped/sorted/reordered.
    bool     HasUndoStateSerialization() const override { return true; }
    wxString SerializeUndoState() const override;
    void     RestoreUndoState( const wxString& aState ) override;
 
 
    // Stuff this class knows nothing about, but needs to guarantee that the templated class implements
    virtual void ExpandCollapseRow( int aRow ) = 0;
 
protected:
    // One of the special field names like ${DNP} that refer to the Attributes in symbols/footprints
    virtual bool fieldIsAttribute( const wxString& aFieldName ) const;
    // One of the special field names like ${SYMBOL_KEYWORDS} that refer to sometimes-editable
    // properties in the symbol/footprint itself. Especially used for lib symbols/footprints,
    // that have properties that overlap with mandatory fields, like lib footprints having
    // a library description property as well as a mandatory description field
    virtual bool fieldIsItemProperty( const wxString& aFieldName ) const;
 
    virtual bool fieldSupportsVariants( const wxString& aFieldName ) const { return false; }
    wxString     fieldVariant( const wxString& aFieldName, const wxString& aVariantName ) const;
    void         updateEditedState();
 
    // Helper function to translate named attribute values like ${DNP}.
    virtual wxString getAttributeResolvedValue( const wxString& aFieldName, bool aValue ) const;
 
    bool cellUsesResolvedTextRenderer( int aRow, int aCol );
    void applyResolvedTextRenderer( wxGridCellAttr* aAttr, bool aApplyTint );
 
    bool            cellUsesUrlEditor( int aRow, int aCol );
    wxGridCellAttr* cloneUrlEditorAttr();
 
    wxGridCellAttr* applyCellDecorations( wxGridCellAttr* aAttr, int aRow, int aCol );
    wxGridCellAttr* applyFieldPresenceRenderer( wxGridCellAttr* aAttr, int aRow, int aCol );
 
    // Column mutations invalidate the position-based coordinates held by an open wxGrid editor.
    void            commitPendingGridChanges();
 
protected:
    wxGridCellRenderer* m_stripedRenderer;
    wxGridCellRenderer* m_resolvedTextRenderer;
 
    bool             m_edited;
    int              m_sortColumn;
    bool             m_sortAscending;
    wxString         m_filter;
    BOM_FILTER_SCOPE m_filterScope;
    bool             m_groupingEnabled;
    bool             m_excludeDNP;
    bool             m_includeExcluded;
    bool             m_rebuildsEnabled;

    std::unordered_set<KIID_PATH> m_selectionItems;
 
    wxString              m_currentVariant;  
    std::vector<wxString> m_variantNames;    
 
 
    std::vector<DATA_MODEL_COL> m_cols;
 
    // Data store
    //
    // This is the storage of the dialog's currently edited field values for each item in the table.
    // Having a store separate from the live values in the symbols/footprints objects
    // allows us to keep our edits separate from the actual sch/pcb until the user explicitly
    // applies them.
    //
    // Items (symbols/footprints) are both identified by KIID_PATH, which for a symbol
    // is the sheet path + symbol UUID (symbols have multiple instances),
    // and for a footprint is the footprint UUID.
    //
    // m_rows and m_cols are just a generated view based on the data store,
    // and are rebuilt as the user changes grouping, sorting, filtering, etc.
    //
    // NOTE: be very careful about how you "read" this data store, you should
    // use getStoredFieldValue() to read values from the data store.
    //
    // Each field tracks shared presence and staged/baseline values for the visited variants.
    //
    // Use the get/set/clear/update/initialize functions to access the data store,
    // rather than accessing it directly, so that live baselines and field presence
    // are initialized consistently.
    std::map<KIID_PATH, std::map<wxString, FIELD_STORE_VALUE>> m_dataStore;
};
 

class GRID_CELL_RESOLVED_TEXT_RENDERER : public wxGridCellStringRenderer
{
public:
    GRID_CELL_RESOLVED_TEXT_RENDERER();
 
    void Draw( wxGrid& aGrid, wxGridCellAttr& aAttr, wxDC& aDC, const wxRect& aRect, int aRow, int aCol,
               bool isSelected ) override;
 
    wxSize GetBestSize( wxGrid& aGrid, wxGridCellAttr& aAttr, wxDC& aDC, int aRow, int aCol ) override;
 
    wxGridCellRenderer* Clone() const override;
};
 

template <typename ITEM_TYPE>
class FIELDS_TABLE_DATA_MODEL : public FIELDS_TABLE_DATA_MODEL_BASE
{
public:
    void SetCurrentVariant( const wxString& aVariantName ) override
    {
        commitPendingGridChanges();
        m_currentVariant = aVariantName.CmpNoCase( GetDefaultVariantName() ) == 0 ? wxString() : aVariantName;
 
        for( const ITEM_TYPE& item : getAllItems() )
            refreshDataStoreItem( item );
 
        updateEditedState();
    }
 
    void AddColumn( const wxString& aFieldName, const wxString& aLabel,
                    bool aAddedByUser ) override
    {
        // Don't add a field twice
        if( GetFieldNameCol( aFieldName ) != -1 )
            return;
 
        commitPendingGridChanges();
 
        m_cols.push_back( { aFieldName, aLabel, aAddedByUser, false, false } );
        FinalizeTextVarExpansion( m_cols.back().m_label, FOR_GUI );
 
        for( const ITEM_TYPE& item : getAllItems() )
            initializeDataStoreItemField( item, m_cols.back() );
 
        if( aAddedByUser )
            m_edited = true;
 
        if( wxGrid* grid = GetView() )
        {
            wxGridTableMessage msg( this, wxGRIDTABLE_NOTIFY_COLS_APPENDED, 1 );
            grid->ProcessTableMessage( msg );
        }
    }
 

    void ClearCell( int aRow, int aCol ) override
    {
        wxCHECK_RET( aRow >= 0 && aRow < static_cast<int>( m_rows.size() ), "Invalid Row Number" );
        wxCHECK_RET( aCol >= 0 && aCol < static_cast<int>( m_cols.size() ), "Invalid Column Number" );
 
        if( !CanClearCell( aRow, aCol ) )
            return;
 
        const wxString& fieldName = m_cols[aCol].m_fieldName;
 
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
            clearStoredField( item, fieldName );
 
        m_edited = true;
    }
 
 
    void CopyCellFromVariant( int aRow, int aCol, const wxString& aVariantName ) override
    {
        if( !CanCopyCellFromVariant( aRow, aCol ) )
            return;
 
        const wxString& fieldName = m_cols[aCol].m_fieldName;
        wxString        variant = aVariantName.CmpNoCase( GetDefaultVariantName() ) == 0 ? wxString() : aVariantName;
        std::vector<wxString> values;
 
        // Read every source before writing: grouped items may have different source values.
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
        {
            FIELD_STORE_VALUE& field = storedField( item, fieldName, variant );
            wxString           value = field.m_present ? field.m_variants[variant].m_value : wxString();
 
            if( ColIsAttribute( aCol ) && attributeForcedOnBySheet( item, fieldName, variant ) )
                value = wxS( "1" );
 
            values.push_back( value );
        }
 
        setCellValues( aRow, aCol, values );
        updateEditedState();
    }
 

    bool IsCellClear( int aRow, int aCol ) override
    {
        wxCHECK_MSG( aRow >= 0 && aRow < static_cast<int>( m_rows.size() ), false, "Invalid Row Number" );
        wxCHECK_MSG( aCol >= 0 && aCol < static_cast<int>( m_cols.size() ), false, "Invalid Column Number" );
 
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
        {
            wxString unused;
 
            if( getStoredFieldValue( item, m_cols[aCol].m_fieldName, unused ) )
                return false;
        }
 
        return true;
    }
 

    bool IsCellEdited( int aRow, int aCol ) override
    {
        wxCHECK_MSG( aRow >= 0 && aRow < static_cast<int>( m_rows.size() ), false, "Invalid Row Number" );
        wxCHECK_MSG( aCol >= 0 && aCol < static_cast<int>( m_cols.size() ), false, "Invalid Column Number" );
 
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
        {
            if( fieldIsModified( item, m_cols[aCol].m_fieldName ) )
                return true;
        }
 
        return false;
    }
 
 
    int       GetNumberRows() override { return (int) m_rows.size(); }
    ROW_STATE GetRowState( int aRow ) const override { return m_rows[aRow].m_state; }
 
 
    std::vector<KIID_PATH> GetRowItemKeys( int aRow ) const override
    {
        wxCHECK( aRow >= 0 && aRow < (int) m_rows.size(), std::vector<KIID_PATH>() );
 
        std::vector<KIID_PATH> keys;
 
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
            keys.push_back( getDataStoreKey( item ) );
 
        return keys;
    }
 
 
    std::vector<ITEM_TYPE> GetRowReferences( int aRow ) const
    {
        wxCHECK( aRow >= 0 && aRow < (int) m_rows.size(), std::vector<ITEM_TYPE>() );
        std::span<const ITEM_TYPE> items = m_rows[aRow].GetCellItems();
        return { items.begin(), items.end() };
    }
 

    void RevertRow( int aRow ) override
    {
        wxCHECK_RET( aRow >= 0 && aRow < static_cast<int>( m_rows.size() ), "Invalid Row Number" );
 
        for( const ITEM_TYPE& item : m_rows[aRow].GetCellItems() )
        {
            for( const DATA_MODEL_COL& col : m_cols )
                updateDataStoreItemFieldFromLive( item, col.m_fieldName );
        }
 
        updateEditedState();
    }
 
 
    void ExpandRow( int aRow )
    {
        std::vector<DATA_MODEL_ROW<ITEM_TYPE>> children;
        bool                                   isParent = IsParentRow( m_rows[aRow].m_state );
        std::span<const ITEM_TYPE>             items( m_rows[aRow].m_items );
 
        for( const ITEM_TYPE& ref : items.subspan( isParent ? 1 : 0 ) )
        {
            bool matchFound = false;
 
            // See if we already have a child group which this symbol fits into
            for( DATA_MODEL_ROW<ITEM_TYPE>& child : children )
            {
                // group members are by definition all matching, so just check
                // against the first member
                if( unitMatch( ref, child.m_items[0] ) )
                {
                    matchFound = true;
                    child.m_items.push_back( ref );
                    break;
                }
            }
 
            if( !matchFound )
                children.emplace_back( ref, ROW_STATE::EXPANDED_CHILD );
        }
 
        if( children.empty() || ( !isParent && children.size() < 2 ) )
            return;
 
        std::sort( children.begin(), children.end(),
                   [this]( const DATA_MODEL_ROW<ITEM_TYPE>& lhs,
                           const DATA_MODEL_ROW<ITEM_TYPE>& rhs ) -> bool
                   {
                       return cmpRows( lhs, rhs, m_sortColumn, m_sortAscending );
                   } );
 
        m_rows[aRow].m_state = isParent ? ROW_STATE::PARENT_EXPANDED : ROW_STATE::GROUP_EXPANDED;
        m_rows.insert( m_rows.begin() + aRow + 1, children.begin(), children.end() );
 
        if( GetView() )
        {
            wxGridTableMessage msg( this, wxGRIDTABLE_NOTIFY_ROWS_INSERTED, aRow + 1, children.size() );
            GetView()->ProcessTableMessage( msg );
        }
    }
 
 
    void CollapseRow( int aRow )
    {
        auto firstChild = m_rows.begin() + aRow + 1;
        auto afterLastChild = firstChild;
        int  deleted = 0;
 
        while( afterLastChild != m_rows.end() && afterLastChild->m_state == ROW_STATE::EXPANDED_CHILD )
        {
            deleted++;
            afterLastChild++;
        }
 
        m_rows[aRow].m_state =
                IsParentRow( m_rows[aRow].m_state ) ? ROW_STATE::PARENT_COLLAPSED : ROW_STATE::GROUP_COLLAPSED;
        m_rows.erase( firstChild, afterLastChild );
 
        if( GetView() )
        {
            wxGridTableMessage msg( this, wxGRIDTABLE_NOTIFY_ROWS_DELETED, aRow + 1, deleted );
            GetView()->ProcessTableMessage( msg );
        }
    }
 
 
    void ExpandCollapseRow( int aRow ) override
    {
        if( IsRowCollapsed( m_rows[aRow].m_state ) )
            ExpandRow( aRow );
        else if( IsRowExpanded( m_rows[aRow].m_state ) )
            CollapseRow( aRow );
    }
 
 
    void CollapseForSort()
    {
        for( size_t i = 0; i < m_rows.size(); ++i )
        {
            if( IsRowExpanded( m_rows[i].m_state ) )
            {
                CollapseRow( i );
                m_rows[i].m_state = IsParentRow( m_rows[i].m_state ) ? ROW_STATE::PARENT_COLLAPSED_DURING_SORT
                                                                     : ROW_STATE::GROUP_COLLAPSED_DURING_SORT;
            }
        }
    }
 
 
    void ExpandAfterSort()
    {
        for( size_t i = 0; i < m_rows.size(); ++i )
        {
            if( m_rows[i].m_state == ROW_STATE::GROUP_COLLAPSED_DURING_SORT
                || m_rows[i].m_state == ROW_STATE::PARENT_COLLAPSED_DURING_SORT )
                ExpandRow( i );
        }
    }
 
    wxString GetGroupedValue( const DATA_MODEL_ROW<ITEM_TYPE>& aRow, int aCol,
                              const wxString& refDelimiter = wxT( ", " ),
                              const wxString& refRangeDelimiter = wxT( "-" ), bool resolveVars = false,
                              bool forExport = false )
    {
        std::vector<ITEM_TYPE> items;
        std::set<wxString>     mixedValues;
        wxString               fieldValue;
 
        // Export always aggregates the full group, even when the grid displays its real parent.
        std::span<const ITEM_TYPE> rowItems =
                forExport ? std::span<const ITEM_TYPE>( aRow.m_items ) : aRow.GetCellItems();
 
        for( const ITEM_TYPE& item : rowItems )
        {
            if( ColIsItemIdentifier( aCol ) || ColIsQuantity( aCol ) || ColIsItemNumber( aCol ) )
            {
                items.push_back( item );
            }
            else // Other columns are either a single value or ROW_MULTI_ITEMS
            {
                wxString itemFieldValue;
                getStoredFieldValue( item, m_cols[aCol].m_fieldName, itemFieldValue );
 
                if( resolveVars )
                    getEffectiveFieldValue( item, m_cols[aCol].m_fieldName, itemFieldValue );
 
                // Show the effective state when a sheet forces it on, but do not change
                // the stored value so the symbol is never stamped on apply.
                if( ColIsAttribute( aCol ) && attributeForcedOnBySheet( item, m_cols[aCol].m_fieldName ) )
                    itemFieldValue = wxS( "1" );
 
                if( resolveVars )
                {
                    if( ColIsAttribute( aCol ) )
                    {
                        itemFieldValue =
                                getAttributeResolvedValue( m_cols[aCol].m_fieldName, itemFieldValue == wxS( "1" ) );
                    }
                    // Computed fields (e.g. ${FOOTPRINT_LIBRARY}) can't have un-applied values.
                    // Resolve them against the item.
                    else if( ColIsComputed( aCol ) )
                    {
                        itemFieldValue = getFieldResolvedLiveValue( item, m_cols[aCol].m_fieldName );
                    }
                    // We have a field that contains both non-variable text and a variable
                    else if( IsGeneratedValue( itemFieldValue ) )
                    {
                        // Resolve variables in the un-applied value using the parent symbol and instance
                        // data.
                        itemFieldValue = resolveTextVars( item, itemFieldValue );
                    }
                }
 
                if( forExport )
                    mixedValues.insert( itemFieldValue );
                else if( &item == &rowItems.front() )
                    fieldValue = itemFieldValue;
                else if( fieldValue != itemFieldValue )
                    return INDETERMINATE_STATE;
            }
        }
 
        if( forExport )
        {
            fieldValue = wxEmptyString;
 
            for( const wxString& value : mixedValues )
            {
                if( value.IsEmpty() )
                    continue;
                else if( fieldValue.IsEmpty() )
                    fieldValue = value;
                else
                    fieldValue += "," + value;
            }
        }
 
        if( ColIsItemIdentifier( aCol ) || ColIsQuantity( aCol ) || ColIsItemNumber( aCol ) )
        {
            // Remove duplicates (other units of multi-unit parts)
            std::sort( items.begin(), items.end(),
                    [this]( const ITEM_TYPE& lhs, const ITEM_TYPE& rhs ) -> bool
                    {
                        return cmpRowItems( lhs, rhs );
                    } );
 
            auto logicalEnd = std::unique( items.begin(), items.end(),
                    [this]( const ITEM_TYPE& lhs, const ITEM_TYPE& rhs ) -> bool
                    {
                        return unitMatch( lhs, rhs );
                    } );
 
            items.erase( logicalEnd, items.end() );
        }
 
        // References are the item-identifier column for schematic symbols and footprints and have
        // special formatting for representing reference ranges.  Library item identifiers are joined
        // without reference-range formatting.
        if( ColIsItemIdentifier( aCol ) )
        {
            std::vector<wxString> itemIdentifiers;
 
            for( const ITEM_TYPE& item : items )
                itemIdentifiers.push_back( getItemIdentifier( item ) );
 
            // C1-15, C17 kind of formatting
            if( ColIsReference( aCol ) )
            {
                fieldValue = UTIL::FormatRefDesRanges( itemIdentifiers, refDelimiter, refRangeDelimiter );
            }
            else
            {
                fieldValue.clear();
 
                for( const wxString& itemIdentifier : itemIdentifiers )
                {
                    if( !fieldValue.IsEmpty() )
                        fieldValue += refDelimiter;
 
                    fieldValue += itemIdentifier;
                }
            }
        }
        else if( ColIsQuantity( aCol ) )
            fieldValue = wxString::Format( wxT( "%d" ), (int) items.size() );
        else if( ColIsItemNumber( aCol ) && aRow.m_state != ROW_STATE::EXPANDED_CHILD )
            fieldValue = wxString::Format( wxT( "%d" ), aRow.m_itemNumber );
 
        return fieldValue;
    }
 
 
    wxString GetValue( int aRow, int aCol ) override
    {
        wxCHECK( aRow >= 0 && aRow < GetNumberRows(), wxEmptyString );
        wxCHECK( aCol >= 0 && aCol < GetNumberCols(), wxEmptyString );
 
        return GetGroupedValue( m_rows[aRow], aCol );
    }
 
    wxString GetResolvedValue( int aRow, int aCol ) override
    {
        return GetGroupedValue( m_rows[aRow], aCol, wxT( ", " ), wxT( "-" ), true, false );
    }
 
    wxString GetExportValue( int aRow, int aCol, const wxString& refDelimiter,
                             const wxString& refRangeDelimiter ) override
    {
        return GetGroupedValue( m_rows[aRow], aCol, refDelimiter, refRangeDelimiter, true, true );
    }
 
    bool IsCellReadOnly( int aRow, int aCol ) override
    {
        return FIELDS_TABLE_DATA_MODEL_BASE::IsCellReadOnly( aRow, aCol )
               || allRowItemsHaveAttributeForcedOnBySheet( m_rows[aRow], aCol );
    }
 
 
protected:
    bool MatchesFilter( const ITEM_TYPE& aItem, const wxString& aReference,
                        EDA_COMBINED_MATCHER& aMatcher )
    {
        if( m_filter.IsEmpty() )
            return true;
 
        // There is a slight difference between how the reference field is matched between
        // 'reference' mode and all other modes that only applies to multiunit symbols in the
        // symbol fields table.
        //
        // For compatibility, a filter like U1A will match a multiunit U1 that has a unit A
        // in 'reference' mode, but not in any other mode. It won't match a C1A to a C1 that
        // isn't mulitunit.
        //
        // In non-reference mode, because multiunit symbols are all grouped together,
        // e.g. U1A and U1B are in the same row as U1, the filter will only match if the
        // reference is exactly U1.
        //
        // None of this matters for the footprint fields table / other future tables.
        if( m_filterScope == BOM_FILTER_SCOPE::REFERENCE )
            return aMatcher.Find( aReference.Lower() );
 
        for( size_t i = 0; i < m_cols.size(); ++i )
        {
            const DATA_MODEL_COL& col = m_cols[i];
 
            if( m_filterScope == BOM_FILTER_SCOPE::VISIBLE && !col.m_show )
                continue;
 
            wxString value;
            getStoredFieldValue( aItem, col.m_fieldName, value );
 
            // We want to match on things like DNP and Excluded from BOM when the
            // checkbox is checked
            if( ColIsAttribute( static_cast<int>( i ) ) )
            {
                bool effectiveValue = value == wxS( "1" ) || attributeForcedOnBySheet( aItem, col.m_fieldName );
                value = getAttributeResolvedValue( col.m_fieldName, effectiveValue );
            }
            // Match computed fields on their resolved value rather than the stored variable.
            else if( ColIsComputed( static_cast<int>( i ) ) )
            {
                value = getFieldResolvedLiveValue( aItem, col.m_fieldName );
            }
            // Same as above, but for field values that contain a mix of text and variables, e.g. "Value: ${VALUE}"
            // The point is to match on what the user can see
            else if( IsGeneratedValue( value ) )
            {
                value = resolveTextVars( aItem, value );
            }
 
            if( aMatcher.Find( value.Lower() ) )
                return true;
        }
 
        return false;
    }
 

    bool attributeForcedOnBySheet( const ITEM_TYPE& aItem, const wxString& aAttributeName ) const
    {
        return attributeForcedOnBySheet( aItem, aAttributeName, m_currentVariant );
    }
 
    virtual bool attributeForcedOnBySheet( const ITEM_TYPE& aItem, const wxString& aAttributeName,
                                           const wxString& aVariantName ) const
    {
        return false;
    }
 
 
    bool allRowItemsHaveAttributeForcedOnBySheet( const DATA_MODEL_ROW<ITEM_TYPE>& aRow, int aCol )
    {
        if( !ColIsAttribute( aCol ) || aRow.m_items.empty() )
            return false;
 
        // Lock the cell only when every symbol in the row inherits it, so a mixed group
        // stays editable and shows the indeterminate state.
        for( const ITEM_TYPE& item : aRow.GetCellItems() )
        {
            if( !attributeForcedOnBySheet( item, m_cols[aCol].m_fieldName ) )
                return false;
        }
 
        return true;
    }
 
    bool cmpRows( const DATA_MODEL_ROW<ITEM_TYPE>& lhRow, const DATA_MODEL_ROW<ITEM_TYPE>& rhRow,
                  int aSortCol, bool aAscending )
    {
        // Empty rows always go to the bottom, whether ascending or descending
        if( lhRow.m_items.empty() )
            return false;
        else if( rhRow.m_items.empty() )
            return true;
 
        // N.B. To meet the iterator sort conditions, we cannot simply invert the truth
        // to get the opposite sort.  i.e. ~(a<b) != (a>b)
        auto local_cmp =
                [aAscending]( const auto a, const auto b )
                {
                    if( aAscending )
                        return a < b;
                    else
                        return a > b;
                };
 
        // Primary sort key is sortCol; secondary is always the item identifier.
        if( aSortCol < 0 || aSortCol >= this->GetNumberCols() )
            aSortCol = 0;
 
        wxString lhs = this->GetGroupedValue( lhRow, aSortCol, wxT( ", " ), wxT( "-" ), true )
                               .Trim( true )
                               .Trim( false );
        wxString rhs = this->GetGroupedValue( rhRow, aSortCol, wxT( ", " ), wxT( "-" ), true )
                               .Trim( true )
                               .Trim( false );
 
        if( lhs == rhs || this->ColIsItemIdentifier( aSortCol ) )
        {
            if( aAscending )
                return cmpRowItems( lhRow.m_items[0], rhRow.m_items[0] );
            else
                return cmpRowItems( rhRow.m_items[0], lhRow.m_items[0] );
        }
        else
        {
            return local_cmp( ValueStringCompare( lhs, rhs ), 0 );
        }
    }
 
    // Used for sorting items that are grouped within a single row by their item identifiers.
    virtual bool cmpRowItems( const ITEM_TYPE& lhItem, const ITEM_TYPE& rhItem )
    {
        return StrNumCmp( getItemIdentifier( lhItem ), getItemIdentifier( rhItem ), true ) < 0;
    }
 
    virtual bool unitMatch( const ITEM_TYPE& lhItem, const ITEM_TYPE& rhItem ) = 0;
 
    bool groupMatch( const ITEM_TYPE& lhItem, const ITEM_TYPE& rhItem )
    {
        bool matchFound = false;
 
        for( size_t i = 0; i < m_cols.size(); ++i )
        {
            if( !m_cols[i].m_group )
                continue;
 
            int col = static_cast<int>( i );
 
            // Schematic and PCB references are item identifiers and group by reference prefix.
            // Library item identifiers and library references use ordinary exact matching below.
            if( ColIsItemIdentifier( col ) && ColIsReference( col ) )
            {
                if( UTIL::GetRefDesPrefix( getItemIdentifier( lhItem ) )
                    != UTIL::GetRefDesPrefix( getItemIdentifier( rhItem ) ) )
                {
                    return false;
                }
 
                matchFound = true;
                continue;
            }
 
            const wxString& fieldName = m_cols[i].m_fieldName;
 
            wxString lh;
            wxString rh;
            getStoredFieldValue( lhItem, fieldName, lh );
            getStoredFieldValue( rhItem, fieldName, rh );
 
            // Normalize attributes so absent and explicitly false values group together, while
            // still honoring edits in the data store and effective values inherited from sheets.
            if( ColIsAttribute( col ) )
            {
                // Sheets can force values like DNP on, but it can't force them off
                bool lhEffectiveValue = lh == wxS( "1" ) || attributeForcedOnBySheet( lhItem, fieldName );
                bool rhEffectiveValue = rh == wxS( "1" ) || attributeForcedOnBySheet( rhItem, fieldName );
 
                if( lhEffectiveValue != rhEffectiveValue )
                    return false;
 
                matchFound = true;
                continue;
            }
            // Computed fields (e.g. ${QUANTITY}) are resolved through the item. Property-backed
            // fields use the store so regrouping includes changes that haven't been applied yet.
            else if( ColIsComputed( col ) )
            {
                lh = getFieldResolvedLiveValue( lhItem, fieldName );
                rh = getFieldResolvedLiveValue( rhItem, fieldName );
            }
            // If we're not generated, we might still have a variable reference in the value,
            // e.g. "10K ${TOLERANCE}", which still needs to be resolved
            else
            {
                if( IsGeneratedValue( lh ) )
                    lh = resolveTextVars( lhItem, lh );
                if( IsGeneratedValue( rh ) )
                    rh = resolveTextVars( rhItem, rh );
            }
 
            if( lh != rh )
                return false;
 
            matchFound = true;
        }
 
        return matchFound;
    }
 
    void Sort()
    {
        CollapseForSort();
 
        // Aggregate rows use their lowest item identifier; real parents must remain first.
        for( DATA_MODEL_ROW<ITEM_TYPE>& row : m_rows )
        {
            std::sort( row.m_items.begin() + ( IsParentRow( row.m_state ) ? 1 : 0 ), row.m_items.end(),
                       [this]( const ITEM_TYPE& lhs, const ITEM_TYPE& rhs ) -> bool
                       {
                           return cmpRowItems( lhs, rhs );
                       } );
        }
 
        std::sort( m_rows.begin(), m_rows.end(),
                   [this]( const DATA_MODEL_ROW<ITEM_TYPE>& lhs, const DATA_MODEL_ROW<ITEM_TYPE>& rhs ) -> bool
                   {
                       return cmpRows( lhs, rhs, m_sortColumn, m_sortAscending );
                   } );
 
        // Time to renumber the item numbers
        int itemNumber = 1;
 
        for( DATA_MODEL_ROW<ITEM_TYPE>& row : m_rows )
        {
            row.m_itemNumber = itemNumber++;
        }
 
        ExpandAfterSort();
    }
 

    virtual bool getLiveFieldValue( const ITEM_TYPE& aItem, const wxString& aFieldName, wxString& aValue ) = 0;
 
    virtual bool getLiveFieldValueForVariant( const ITEM_TYPE& aItem, const wxString& aFieldName,
                                              const wxString& aVariant, wxString& aValue )
    {
        return getLiveFieldValue( aItem, aFieldName, aValue );
    }

    std::map<wxString, wxString> getStoredFields( const ITEM_TYPE& aItem, const wxString& aVariant ) const
    {
        std::map<wxString, wxString> fields;
        auto itemIt = m_dataStore.find( getDataStoreKey( aItem ) );
 
        if( itemIt == m_dataStore.end() )
            return fields;
 
        for( const auto& [name, field] : itemIt->second )
        {
            auto valueIt = field.m_variants.find( aVariant );
 
            if( !field.m_present || valueIt == field.m_variants.end() )
                continue;
 
            const FIELD_VALUE_STATE& value = valueIt->second;
 
            if( !field.m_baselinePresent || value.m_value != value.m_baseline )
                fields[name] = value.m_value;
        }
 
        return fields;
    }

    bool getStoredFieldValue( const ITEM_TYPE& aItem, const wxString& aFieldName, wxString& aValue ) const
    {
        aValue.clear();
 
        auto itemIt = m_dataStore.find( getDataStoreKey( aItem ) );
 
        if( itemIt == m_dataStore.end() )
            return false;
 
        auto fieldIt = itemIt->second.find( aFieldName );
 
        if( fieldIt == itemIt->second.end() || !fieldIt->second.m_present )
            return false;
 
        auto valueIt = fieldIt->second.m_variants.find( fieldVariant( aFieldName, m_currentVariant ) );
 
        if( valueIt != fieldIt->second.m_variants.end() )
            aValue = valueIt->second.m_value;
 
        return true;
    }

    bool resolveStoredTextVar( const ITEM_TYPE& aItem, wxString* aToken, bool aCaseSensitive = false ) const
    {
        auto itemIt = m_dataStore.find( getDataStoreKey( aItem ) );
 
        if( itemIt == m_dataStore.end() )
            return false;
 
        const auto& fields = itemIt->second;
        auto        fieldIt = fields.find( *aToken );
 
        if( fieldIt == fields.end() && !aCaseSensitive )
        {
            fieldIt = std::find_if( fields.begin(), fields.end(),
                                    [&]( const auto& aEntry )
                                    {
                                        return aToken->IsSameAs( aEntry.first, false );
                                    } );
        }
 
        if( fieldIt == fields.end() || IsGeneratedField( fieldIt->first ) )
            return false;
 
        const wxString& name = fieldIt->first;
        int             col = GetFieldNameCol( name );
 
        // Identifiers can have instance-specific formatting, such as multi-unit references.
        if( col >= 0 && ColIsItemIdentifier( col ) )
            return false;
 
        const FIELD_STORE_VALUE& field = fieldIt->second;
 
        if( !field.m_present )
        {
            if( !field.m_baselinePresent )
                return false;
 
            // A pending deletion must not fall back to the old live field value.
            aToken->clear();
            return true;
        }
 
        wxString value;
        getStoredFieldValue( aItem, name, value );
        *aToken = UnescapeString( value );
        return true;
    }

    void setStoredFieldValue( const ITEM_TYPE& aItem, const wxString& aFieldName,
                              const wxString& aValue )
    {
        FIELD_STORE_VALUE& field = storedField( aItem, aFieldName );
        field.m_present = true;
        field.m_variants[fieldVariant( aFieldName, m_currentVariant )].m_value = aValue;
    }

    virtual void setCellValues( int aRow, int aCol, const std::vector<wxString>& aValues )
    {
        std::span<const ITEM_TYPE> items = m_rows[aRow].GetCellItems();
 
        for( size_t i = 0; i < items.size(); ++i )
            setStoredFieldValue( items[i], m_cols[aCol].m_fieldName, aValues[i] );
    }

    void ensureStoredFieldPresent( const ITEM_TYPE& aItem, const wxString& aFieldName )
    {
        storedField( aItem, aFieldName ).m_present = true;
    }

    void clearStoredField( const ITEM_TYPE& aItem, const wxString& aFieldName )
    {
        FIELD_STORE_VALUE& field = storedField( aItem, aFieldName );
        field.m_present = false;
 
        for( auto& [variant, state] : field.m_variants )
            state.m_value = state.m_baseline;
    }

    bool fieldIsModified( const ITEM_TYPE& aItem, const wxString& aFieldName )
    {
        wxString liveValue;
        wxString storedValue;
        bool     livePresent = getLiveFieldValue( aItem, aFieldName, liveValue );
        bool     storedPresent = getStoredFieldValue( aItem, aFieldName, storedValue );
 
        return livePresent != storedPresent || liveValue != storedValue;
    }
 

    void updateDataStoreItemFieldFromLive( const ITEM_TYPE& aItem, const wxString& aFieldName,
                                           bool aAllVariants = false )
    {
        FIELD_STORE_VALUE& field = storedField( aItem, aFieldName );
        wxString           currentVariant = fieldVariant( aFieldName, m_currentVariant );
        wxString           liveValue;
        bool               livePresent = getLiveFieldValueForVariant( aItem, aFieldName, currentVariant, liveValue );
 
        // Reverting a presence change affects every variant. Otherwise revert only the
        // displayed variant; successful Apply explicitly accepts all cached variants.
        bool allVariants = aAllVariants || field.m_present != livePresent;
 
        for( auto& [variant, state] : field.m_variants )
        {
            if( allVariants || variant == currentVariant )
            {
                getLiveFieldValueForVariant( aItem, aFieldName, variant, state.m_value );
                state.m_baseline = state.m_value;
            }
        }
 
        field.m_present = field.m_baselinePresent = livePresent;
    }

    void initializeDataStoreItemField( const ITEM_TYPE& aItem, const DATA_MODEL_COL& aCol )
    {
        FIELD_STORE_VALUE& field = storedField( aItem, aCol.m_fieldName );
 
        if( aCol.m_userAdded )
            field.m_present = true;
    }

    void initializeDataStoreItem( const ITEM_TYPE& aItem )
    {
        for( const DATA_MODEL_COL& col : m_cols )
            initializeDataStoreItemField( aItem, col );
    }
 
    FIELD_STORE_VALUE& storedField( const ITEM_TYPE& aItem, const wxString& aFieldName )
    {
        return storedField( aItem, aFieldName, m_currentVariant );
    }
 
    FIELD_STORE_VALUE& storedField( const ITEM_TYPE& aItem, const wxString& aFieldName, const wxString& aVariantName )
    {
        auto& fields = m_dataStore[getDataStoreKey( aItem )];
        auto [fieldIt, newField] = fields.try_emplace( aFieldName );
        FIELD_STORE_VALUE& field = fieldIt->second;
        wxString           variant = fieldVariant( aFieldName, aVariantName );
        auto [valueIt, newVariant] = field.m_variants.try_emplace( variant );
 
        if( newVariant )
        {
            FIELD_VALUE_STATE& state = valueIt->second;
            bool               present = getLiveFieldValueForVariant( aItem, aFieldName, variant, state.m_baseline );
            state.m_value = state.m_baseline;
 
            if( newField )
                field.m_present = field.m_baselinePresent = present;
        }
 
        return field;
    }
 
    void refreshDataStoreItem( const ITEM_TYPE& aItem )
    {
        commitPendingGridChanges();
 
        for( const DATA_MODEL_COL& col : m_cols )
        {
            auto& fields = m_dataStore[getDataStoreKey( aItem )];
 
            if( !fields.contains( col.m_fieldName ) )
                initializeDataStoreItemField( aItem, col );
            else
                storedField( aItem, col.m_fieldName );
        }
 
        for( auto& [name, field] : m_dataStore[getDataStoreKey( aItem )] )
        {
            bool livePresent = field.m_baselinePresent;
            bool liveChanged = false;
 
            for( auto& [variant, state] : field.m_variants )
            {
                wxString live;
                livePresent = getLiveFieldValueForVariant( aItem, name, variant, live );
                bool changed = livePresent != field.m_baselinePresent || live != state.m_baseline;
 
                // External changes to this field win; a move/rotation leaves staged edits alone.
                if( changed || state.m_value == state.m_baseline )
                    state.m_value = live;
 
                state.m_baseline = live;
                liveChanged |= changed;
            }
 
            if( liveChanged || field.m_present == field.m_baselinePresent )
                field.m_present = livePresent;
 
            field.m_baselinePresent = livePresent;
        }
    }
 
    bool storedFieldIsRemoved( const ITEM_TYPE& aItem, const wxString& aName ) const
    {
        auto itemIt = m_dataStore.find( getDataStoreKey( aItem ) );
 
        if( itemIt == m_dataStore.end() )
            return false;
 
        auto fieldIt = itemIt->second.find( aName );
        return fieldIt != itemIt->second.end() && !fieldIt->second.m_present && fieldIt->second.m_baselinePresent;
    }
 
    std::set<wxString> storedVariants( const ITEM_TYPE& aItem ) const
    {
        std::set<wxString> variants;
        auto               itemIt = m_dataStore.find( getDataStoreKey( aItem ) );
 
        if( itemIt != m_dataStore.end() )
        {
            for( const auto& [name, field] : itemIt->second )
            {
                for( const auto& [variant, state] : field.m_variants )
                    variants.insert( variant );
            }
        }
 
        return variants;
    }
 
    void acceptDataStoreItem( const ITEM_TYPE& aItem )
    {
        for( auto& [name, field] : m_dataStore[getDataStoreKey( aItem )] )
            updateDataStoreItemFieldFromLive( aItem, name, true );
    }
 
    // Display inherited values without turning them into locally stored overrides.
    virtual void getEffectiveFieldValue( const ITEM_TYPE& aItem, const wxString& aFieldName,
                                         wxString& aValue ) const
    {
        getStoredFieldValue( aItem, aFieldName, aValue );
    }
 
    virtual std::vector<ITEM_TYPE> getAllItems() const = 0;
 
public:
    virtual KIID_PATH getDataStoreKey( const ITEM_TYPE& aItem ) const = 0;
    virtual wxString  getItemIdentifier( const ITEM_TYPE& aItem ) const = 0;

    virtual wxString getFieldResolvedLiveValue( const ITEM_TYPE& aItem, const wxString& aFieldName ) = 0;
 
    virtual wxString resolveTextVars( const ITEM_TYPE& aItem, const wxString& aText ) = 0;
 
protected:
    std::vector<DATA_MODEL_ROW<ITEM_TYPE>> m_rows;
};