feat: consolidate card data into optimized format for faster queries and reduced file sizes

This commit is contained in:
matt 2025-10-15 11:04:49 -07:00
parent 5753bb19f8
commit f70ffca23e
24 changed files with 2903 additions and 135 deletions

View file

@ -94,6 +94,7 @@ MAIN_MENU_ITEMS: List[str] = ['Build A Deck', 'Setup CSV Files', 'Tag CSV Files'
SETUP_MENU_ITEMS: List[str] = ['Initial Setup', 'Regenerate CSV', 'Main Menu']
CSV_DIRECTORY: str = 'csv_files'
CARD_FILES_DIRECTORY: str = 'card_files' # Parquet files for consolidated card data
# Configuration for handling null/NA values in DataFrame columns
FILL_NA_COLUMNS: Dict[str, Optional[str]] = {
@ -101,6 +102,14 @@ FILL_NA_COLUMNS: Dict[str, Optional[str]] = {
'faceName': None # Use card's name column value when face name is not available
}
# ----------------------------------------------------------------------------------
# ALL CARDS CONSOLIDATION FEATURE FLAG
# ----------------------------------------------------------------------------------
# Enable use of consolidated all_cards.parquet file (default: True)
# Set to False to disable and fall back to individual CSV file loading
USE_ALL_CARDS_FILE = os.getenv('USE_ALL_CARDS_FILE', '1').lower() not in ('0', 'false', 'off', 'disabled')
# ----------------------------------------------------------------------------------
# TAGGING REFINEMENT FEATURE FLAGS (M1-M5)
# ----------------------------------------------------------------------------------