Refactored setup.py again, confirmed that all filters are now working as expected. Work will resume on main branch now

exceptions.py

@@ -83,3 +83,40 @@ class ColorFilterError(MTGSetupError):
         self.details = details
         error_info = f" - {details}" if details else ""
         super().__init__(f"{message} for color '{color}'{error_info}")
+
+
+class CommanderValidationError(MTGSetupError):
+    """Exception raised when commander validation fails.
+    
+    This exception is raised when there are issues validating commander cards,
+    such as non-legendary creatures, color identity mismatches, or banned cards.
+    
+    Args:
+        message: Explanation of the error
+        validation_type: Type of validation that failed (e.g., 'legendary_check', 'color_identity', 'banned_set')
+        details: Additional error details
+    
+    Examples:
+        >>> raise CommanderValidationError(
+        ...     "Card must be legendary",
+        ...     "legendary_check",
+        ...     "Lightning Bolt is not a legendary creature"
+        ... )
+        
+        >>> raise CommanderValidationError(
+        ...     "Commander color identity mismatch",
+        ...     "color_identity",
+        ...     "Omnath, Locus of Creation cannot be used in Golgari deck"
+        ... )
+        
+        >>> raise CommanderValidationError(
+        ...     "Commander banned in format",
+        ...     "banned_set",
+        ...     "Golos, Tireless Pilgrim is banned in Commander"
+        ... )
+    """
+    def __init__(self, message: str, validation_type: str, details: str = None) -> None:
+        self.validation_type = validation_type
+        self.details = details
+        error_info = f" - {details}" if details else ""
+        super().__init__(f"{message} [{validation_type}]{error_info}")

main.py

@@ -2,69 +2,104 @@ from __future__ import annotations
 
 import inquirer.prompt # type: ignore
 import sys
-
+import logging
 from pathlib import Path
+from typing import NoReturn, Optional
 
 import setup
 import card_info
+import tagger
 
-Path('csv_files').mkdir(parents=True, exist_ok=True)
+# Configure logging
-Path('staples').mkdir(parents=True, exist_ok=True)
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(),
+        logging.FileHandler('main.log', mode='w')
+    ]
+)
 
-while True:
+# Menu constants
-    print('What would you like to do?')
+MENU_SETUP = 'Setup'
-    choice = 'Menu'
+MENU_BUILD_DECK = 'Build a Deck'
-    while choice == 'Menu':
+MENU_CARD_INFO = 'Get Card Info'
+MAIN_TAG = 'Tag CSV Files'
+MENU_QUIT = 'Quit'
+
+MENU_CHOICES = [MENU_SETUP, MENU_BUILD_DECK, MENU_CARD_INFO, MAIN_TAG, MENU_QUIT]
+def get_menu_choice() -> Optional[str]:
+    """Display the main menu and get user choice.
+
+    Returns:
+        Optional[str]: The selected menu option or None if cancelled
+    """
     question = [
         inquirer.List('menu',
-                          choices=['Setup', 'Build a Deck', 'Get Card Info', 'Quit'],
+                      choices=MENU_CHOICES,
                       carousel=True)
     ]
     try:
         answer = inquirer.prompt(question)
-            if answer is None:
+        return answer['menu'] if answer else None
-                print("Operation cancelled. Returning to menu...")
+    except (KeyError, TypeError) as e:
-                choice = 'Menu'
+        logging.error(f"Error getting menu choice: {e}")
-                continue
+        return None
-            choice = answer['menu']
-        except (KeyError, TypeError):
-            print("Invalid input. Please try again.")
-            choice = 'Menu'
 
-    # Run through initial setup
+def handle_card_info() -> None:
-    while choice == 'Setup':
+    """Handle the card info menu option with proper error handling."""
-        setup.setup()
+    try:
-        choice = 'Menu'
+        while True:
-        
-
-    # Make a new deck
-    while choice == 'Build a Deck':
-        print('Deck building not yet implemented')
-        choice = 'Menu'
-        
-
-    # Get a cards info
-    while choice == 'Get Card Info':
             card_info.get_card_info()
             question = [
                 inquirer.Confirm('continue',
-                             message='Would you like to look up another card?'
+                                message='Would you like to look up another card?')
-                             )
             ]
             try:
                 answer = inquirer.prompt(question)
-            if answer is None:
+                if not answer or not answer['continue']:
-                print("Operation cancelled. Returning to menu...")
+                    break
-                choice = 'Menu'
+            except (KeyError, TypeError) as e:
+                logging.error(f"Error in card info continuation prompt: {e}")
+                break
+    except Exception as e:
+        logging.error(f"Error in card info handling: {e}")
+
+def run_menu() -> NoReturn:
+    """Main menu loop with improved error handling and logging."""
+    logging.info("Starting MTG Python Deckbuilder")
+    Path('csv_files').mkdir(parents=True, exist_ok=True)
+
+    while True:
+        try:
+            print('What would you like to do?')
+            choice = get_menu_choice()
+
+            if choice is None:
+                logging.info("Menu operation cancelled")
                 continue
-            new_card = answer['continue']
-            if new_card:
-                choice = 'Get Card Info'  # Fixed == to = for assignment
-        except (KeyError, TypeError):
-            print("Invalid input. Returning to menu...")
-            choice = 'Menu'
 
-    # Quit
+            logging.info(f"User selected: {choice}")
-    while choice == 'Quit':
-        sys.exit()
 
+            match choice:
+                case 'Setup':
+                    setup.setup()
+                    tagger.run_tagging()
+                case 'Build a Deck':
+                    logging.info("Deck building not yet implemented")
+                    print('Deck building not yet implemented')
+                case 'Get Card Info':
+                    handle_card_info()
+                case 'Tag CSV Files':
+                    tagger.run_tagging()
+                case 'Quit':
+                    logging.info("Exiting application")
+                    sys.exit(0)
+                case _:
+                    logging.warning(f"Invalid menu choice: {choice}")
+
+        except Exception as e:
+            logging.error(f"Unexpected error in main menu: {e}")
+
+if __name__ == "__main__":
+    run_menu()

settings.py

@@ -1,3 +1,9 @@
+"""Constants and configuration settings for the MTG Python Deckbuilder.
+
+This module contains all the constant values and configuration settings used throughout
+the application for card filtering, processing, and analysis. Constants are organized
+into logical sections with clear documentation.
+"""
 artifact_tokens = ['Blood', 'Clue', 'Food', 'Gold', 'Incubator',
                 'Junk','Map','Powerstone', 'Treasure']
 
@@ -793,21 +799,22 @@ CARD_TYPES_TO_EXCLUDE = [
     'Contraption'
 ]
 
+# Columns to keep when processing CSV files
 CSV_PROCESSING_COLUMNS = [
-    'name',
+    'name',        # Card name
-    'faceName',
+    'faceName',    # Name of specific face for multi-faced cards
-    'edhrecRank',
+    'edhrecRank',  # Card's rank on EDHREC
-    'colorIdentity',
+    'colorIdentity',  # Color identity for Commander format
-    'colors',
+    'colors',      # Actual colors in card's mana cost
-    'manaCost', 
+    'manaCost',    # Mana cost string
-    'manaValue',
+    'manaValue',   # Converted mana cost
-    'type',
+    'type',        # Card type line
-    'layout',
+    'layout',      # Card layout (normal, split, etc)
-    'text',
+    'text',        # Card text/rules
-    'power',
+    'power',       # Power (for creatures)
-    'toughness', 
+    'toughness',   # Toughness (for creatures)
-    'keywords',
+    'keywords',    # Card's keywords
-    'side'
+    'side'         # Side identifier for multi-faced cards
 ]
 
 SETUP_COLORS = ['colorless', 'white', 'blue', 'black', 'green', 'red',
@@ -824,3 +831,30 @@ COLOR_ABRV = ['Colorless', 'W', 'U', 'B', 'G', 'R',
               'B, G, W', 'R, U, W', 'B, R, W', 'B, G, U', 'G, R, U',
               'B, G, R, W', 'B, G, R, U', 'G, R, U, W', 'B, G, U, W',
               'B, R, U, W', 'B, G, R, U, W']
+
+# Configuration for handling null/NA values in DataFrame columns
+FILL_NA_COLUMNS = {
+    'colorIdentity': 'Colorless',  # Default color identity for cards without one
+    'faceName': None  # Use card's name column value when face name is not available
+}
+# Configuration for DataFrame sorting operations
+SORT_CONFIG = {
+    'columns': ['name', 'side'],  # Columns to sort by
+    'case_sensitive': False  # Ignore case when sorting
+}
+
+# Configuration for DataFrame filtering operations
+FILTER_CONFIG = {
+    'layout': {
+        'exclude': ['reversible_card']
+    },
+    'availability': {
+        'require': ['paper']
+    },
+    'promoTypes': {
+        'exclude': ['playtest']
+    },
+    'securityStamp': {
+        'exclude': ['Heart', 'Acorn']
+    }
+}

setup.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
+from enum import Enum
+
 import pandas as pd # type: ignore
-import requests # type: ignore
 import inquirer.prompt # type: ignore
 import logging
 
 from settings import banned_cards, csv_directory, SETUP_COLORS, COLOR_ABRV, MTGJSON_API_URL
-from setup_utils import download_cards_csv, filter_dataframe, process_legendary_cards
+from setup_utils import download_cards_csv, filter_dataframe, process_legendary_cards, filter_by_color_identity
+from exceptions import CSVFileNotFoundError, MTGJSONDownloadError, DataFrameProcessingError, ColorFilterError, CommanderValidationError
 
 # Configure logging
 logging.basicConfig(
@@ -16,77 +18,38 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-def filter_by_color(df, column_name, value, new_csv_name):
+def check_csv_exists(file_path: str) -> bool:
-    # Filter dataframe
+    """Check if a CSV file exists at the specified path.
-    filtered_df = df[df[column_name] == value]
+    
+    Args:
+        file_path: Path to the CSV file to check
+        
+    Returns:
+        bool: True if file exists, False otherwise
+        
+    Raises:
+        CSVFileNotFoundError: If there are issues accessing the file path
     """
-    Save the filtered dataframe to a new csv file, and narrow down/rearranges the columns it
-    keeps to increase readability/trim some extra data.
-    Additionally attempts to remove as many duplicates (including cards with reversible prints,
-    as well as taking out Arena-only cards.
-    """
-    filtered_df.sort_values('name')
-    filtered_df = filtered_df.loc[filtered_df['layout'] != 'reversible_card'] 
-    filtered_df = filtered_df[filtered_df['availability'].str.contains('paper')]
-    filtered_df = filtered_df.loc[filtered_df['promoTypes'] != 'playtest']
-    filtered_df = filtered_df.loc[filtered_df['securityStamp'] != 'heart']
-    filtered_df = filtered_df.loc[filtered_df['securityStamp'] != 'acorn']
-    
-    for card in banned_cards:
-        filtered_df = filtered_df[~filtered_df['name'].str.contains(card)]
-    
-    card_types = ['Plane —', 'Conspiracy', 'Vanguard', 'Scheme', 'Phenomenon', 'Stickers', 'Attraction', 'Hero', 'Contraption']
-    for card_type in card_types:
-        filtered_df = filtered_df[~filtered_df['type'].str.contains(card_type)]
-    filtered_df['faceName'] = filtered_df['faceName'].fillna(filtered_df['name'])
-    filtered_df.drop_duplicates(subset='faceName', keep='first', inplace=True)
-    columns_to_keep = ['name', 'faceName','edhrecRank','colorIdentity', 'colors', 'manaCost', 'manaValue', 'type', 'layout', 'text', 'power', 'toughness', 'keywords', 'side']
-    filtered_df = filtered_df[columns_to_keep]
-    filtered_df.sort_values(by=['name', 'side'], key=lambda col: col.str.lower(), inplace=True)
-        
-    
-    filtered_df.to_csv(new_csv_name, index=False)
-
-def determine_commanders():
-    print('Generating commander_cards.csv, containing all cards elligible to be commanders.')
     try:
-        # Check for cards.csv
+        with open(file_path, 'r', encoding='utf-8'):
-        cards_file = f'{csv_directory}/cards.csv'
+            return True
-        try:
-            with open(cards_file, 'r', encoding='utf-8'):
-                print('cards.csv exists.')
     except FileNotFoundError:
-            print('cards.csv not found, downloading from mtgjson')
+        return False
-            download_cards_csv(MTGJSON_API_URL, cards_file)
-            
-        # Load and process cards data
-        df = pd.read_csv(cards_file, low_memory=False)
-        df['colorIdentity'] = df['colorIdentity'].fillna('Colorless')
-        
-        # Process legendary cards
-        filtered_df = process_legendary_cards(df)
-        
-        # Apply standard filters
-        filtered_df = filter_dataframe(filtered_df, banned_cards)
-        
-        # Save commander cards
-        filtered_df.to_csv(f'{csv_directory}/commander_cards.csv', index=False)
-        print('commander_cards.csv file generated.')
-        
     except Exception as e:
-        print(f'Error generating commander cards: {str(e)}')
+        raise CSVFileNotFoundError(f'Error checking CSV file: {str(e)}')
-        raise
 
-def initial_setup():
+def initial_setup() -> None:
     """Perform initial setup by downloading card data and creating filtered CSV files.
     
-    This function:
+    Downloads the latest card data from MTGJSON if needed, creates color-filtered CSV files,
-    1. Downloads the latest card data from MTGJSON if needed
+    and generates commander-eligible cards list. Uses utility functions from setup_utils.py
-    2. Creates color-filtered CSV files
+    for file operations and data processing.
-    3. Generates commander-eligible cards list
     
-    Uses utility functions from setup_utils.py for file operations and data processing.
+    Raises:
-    Implements proper error handling for file operations and data processing.
+        CSVFileNotFoundError: If required CSV files cannot be found
+        MTGJSONDownloadError: If card data download fails
+        DataFrameProcessingError: If data processing fails
+        ColorFilterError: If color filtering fails
     """
     logger.info('Checking for cards.csv file')
     
@@ -120,103 +83,217 @@ def initial_setup():
         logger.error(f'Error during initial setup: {str(e)}')
         raise
 
-def regenerate_csvs_all():
+def filter_by_color(df: pd.DataFrame, column_name: str, value: str, new_csv_name: str) -> None:
-    """
+    """Filter DataFrame by color identity and save to CSV.
-    Pull the original cards.csv file and remake the {color}_cards.csv files.
-    This is useful if a new set has since come out to ensure the databases are up-to-date
-    """
-    print('Downloading cards.csv from mtgjson')
-    url = 'https://mtgjson.com/api/v5/csv/cards.csv'
-    r = requests.get(url)
-    with open('csv_files/cards.csv', 'wb') as outputfile:
-        outputfile.write(r.content)
     
-    # Load cards.csv file into pandas dataframe so it can be further broken down
+    Args:
-    df = pd.read_csv('csv_files/cards.csv', low_memory=False)#, converters={'printings': pd.eval})
+        df: DataFrame to filter
+        column_name: Column to filter on (should be 'colorIdentity')
+        value: Color identity value to filter for
+        new_csv_name: Path to save filtered CSV
         
-    # Set frames that have nothing for color identity to be 'Colorless' instead
+    Raises:
+        ColorFilterError: If filtering fails
+        DataFrameProcessingError: If DataFrame processing fails
+        CSVFileNotFoundError: If CSV file operations fail
+    """
+    try:
+        # Check if target CSV already exists
+        if check_csv_exists(new_csv_name):
+            logger.info(f'{new_csv_name} already exists, will be overwritten')
+            
+        filtered_df = filter_by_color_identity(df, value)
+        filtered_df.to_csv(new_csv_name, index=False)
+        logger.info(f'Successfully created {new_csv_name}')
+    except (ColorFilterError, DataFrameProcessingError, CSVFileNotFoundError) as e:
+        logger.error(f'Failed to filter by color {value}: {str(e)}')
+        raise
+
+def determine_commanders() -> None:
+    """Generate commander_cards.csv containing all cards eligible to be commanders.
+    
+    This function processes the card database to identify and validate commander-eligible cards,
+    applying comprehensive validation steps and filtering criteria.
+    
+    Raises:
+        CSVFileNotFoundError: If cards.csv is missing and cannot be downloaded
+        MTGJSONDownloadError: If downloading cards data fails
+        CommanderValidationError: If commander validation fails
+        DataFrameProcessingError: If data processing operations fail
+    """
+    logger.info('Starting commander card generation process')
+    
+    try:
+        # Check for cards.csv with progress tracking
+        cards_file = f'{csv_directory}/cards.csv'
+        if not check_csv_exists(cards_file):
+            logger.info('cards.csv not found, initiating download')
+            download_cards_csv(MTGJSON_API_URL, cards_file)
+        else:
+            logger.info('cards.csv found, proceeding with processing')
+            
+        # Load and process cards data
+        logger.info('Loading card data from CSV')
+        df = pd.read_csv(cards_file, low_memory=False)
         df['colorIdentity'] = df['colorIdentity'].fillna('Colorless')
         
-    rows_to_drop = []
+        # Process legendary cards with validation
-    non_legel_sets = ['PHTR', 'PH17', 'PH18' ,'PH19', 'PH20', 'PH21', 'UGL', 'UND', 'UNH', 'UST',]
+        logger.info('Processing and validating legendary cards')
-    for index, row in df.iterrows():
+        try:
-        for illegal_set in non_legel_sets:
+            filtered_df = process_legendary_cards(df)
-            if illegal_set in row['printings']:
+        except CommanderValidationError as e:
-                rows_to_drop.append(index)
+            logger.error(f'Commander validation failed: {str(e)}')
-    df = df.drop(rows_to_drop)
+            raise
         
-    # Color identity sorted cards
+        # Apply standard filters
-    print('Regenerating color identity sorted files.\n')
+        logger.info('Applying standard card filters')
+        filtered_df = filter_dataframe(filtered_df, banned_cards)
         
-    # For loop to iterate through the colors
+        # Save commander cards
-    for i in range(min(len(SETUP_COLORS), len(COLOR_ABRV))):
+        logger.info('Saving validated commander cards')
-        print(f'Regenerating {SETUP_COLORS[i]}_cards.csv.')
+        filtered_df.to_csv(f'{csv_directory}/commander_cards.csv', index=False)
-        filter_by_color(df, 'colorIdentity', COLOR_ABRV[i], f'csv_files/{SETUP_COLORS[i]}_cards.csv')
-        print(f'A new {SETUP_COLORS[i]}_cards.csv file has been made.\n')
         
-    # Once files are regenerated, create a new legendary list
+        logger.info('Commander card generation completed successfully')
-    determine_commanders()
         
-def regenerate_csv_by_color(color):
+    except (CSVFileNotFoundError, MTGJSONDownloadError) as e:
+        logger.error(f'File operation error: {str(e)}')
+        raise
+    except CommanderValidationError as e:
+        logger.error(f'Commander validation error: {str(e)}')
+        raise
+    except Exception as e:
+        logger.error(f'Unexpected error during commander generation: {str(e)}')
+        raise
+    
+def regenerate_csvs_all() -> None:
+    """Regenerate all color-filtered CSV files from latest card data.
+    
+    Downloads fresh card data and recreates all color-filtered CSV files.
+    Useful for updating the card database when new sets are released.
+    
+    Raises:
+        MTGJSONDownloadError: If card data download fails
+        DataFrameProcessingError: If data processing fails
+        ColorFilterError: If color filtering fails
     """
-    Pull the original cards.csv file and remake the {color}_cards.csv files
+    try:
-    """
+        logger.info('Downloading latest card data from MTGJSON')
-    # Determine the color_abv to use
+        download_cards_csv(MTGJSON_API_URL, f'{csv_directory}/cards.csv')
-    COLOR_ABRV_index = SETUP_COLORS.index(color)
+        
-    color_abv = COLOR_ABRV[COLOR_ABRV_index]
+        logger.info('Loading and processing card data')
-    print('Downloading cards.csv from mtgjson')
-    url = 'https://mtgjson.com/api/v5/csv/cards.csv'
-    r = requests.get(url)
-    with open(f'{csv_directory}/cards.csv', 'wb') as outputfile:
-        outputfile.write(r.content)
-    # Load cards.csv file into pandas dataframe so it can be further broken down
         df = pd.read_csv(f'{csv_directory}/cards.csv', low_memory=False)
-    
-    # Set frames that have nothing for color identity to be 'Colorless' instead
         df['colorIdentity'] = df['colorIdentity'].fillna('Colorless')
         
-    # Color identity sorted cards
+        logger.info('Regenerating color identity sorted files')
-    print(f'Regenerating {color}_cards.csv file.\n')
+        for i in range(min(len(SETUP_COLORS), len(COLOR_ABRV))):
+            color = SETUP_COLORS[i]
+            color_id = COLOR_ABRV[i]
+            logger.info(f'Processing {color} cards')
+            filter_by_color(df, 'colorIdentity', color_id, f'{csv_directory}/{color}_cards.csv')
             
-    # Regenerate the file
+        logger.info('Regenerating commander cards')
-    print(f'Regenerating {color}_cards.csv.')
+        determine_commanders()
-    filter_by_color(df, 'colorIdentity', color_abv, f'{csv_directory}/{color}_cards.csv')
-    print(f'A new {color}_cards.csv file has been made.\n')
         
+        logger.info('Card database regeneration complete')
+        
+    except Exception as e:
+        logger.error(f'Failed to regenerate card database: {str(e)}')
+        raise
     # Once files are regenerated, create a new legendary list
     determine_commanders()
 
-def add_tags():
+def regenerate_csv_by_color(color: str) -> None:
-    pass
+    """Regenerate CSV file for a specific color identity.
     
-def setup():
+    Args:
-    while True:
+        color: Color name to regenerate CSV for (e.g. 'white', 'blue')
+        
+    Raises:
+        ValueError: If color is not valid
+        MTGJSONDownloadError: If card data download fails
+        DataFrameProcessingError: If data processing fails
+        ColorFilterError: If color filtering fails
+    """
+    try:
+        if color not in SETUP_COLORS:
+            raise ValueError(f'Invalid color: {color}')
+            
+        color_abv = COLOR_ABRV[SETUP_COLORS.index(color)]
+        
+        logger.info(f'Downloading latest card data for {color} cards')
+        download_cards_csv(MTGJSON_API_URL, f'{csv_directory}/cards.csv')
+        
+        logger.info('Loading and processing card data')
+        df = pd.read_csv(f'{csv_directory}/cards.csv', low_memory=False)
+        df['colorIdentity'] = df['colorIdentity'].fillna('Colorless')
+        
+        logger.info(f'Regenerating {color} cards CSV')
+        filter_by_color(df, 'colorIdentity', color_abv, f'{csv_directory}/{color}_cards.csv')
+        
+        logger.info(f'Successfully regenerated {color} cards database')
+        
+    except Exception as e:
+        logger.error(f'Failed to regenerate {color} cards: {str(e)}')
+        raise
+
+class SetupOption(Enum):
+    """Enum for setup menu options."""
+    INITIAL_SETUP = 'Initial Setup'
+    REGENERATE_CSV = 'Regenerate CSV Files'
+    BACK = 'Back'
+
+def _display_setup_menu() -> SetupOption:
+    """Display the setup menu and return the selected option.
+    
+    Returns:
+        SetupOption: The selected menu option
+    """
+    question = [
+        inquirer.List('menu',
+                      choices=[option.value for option in SetupOption],
+                      carousel=True)
+    ]
+    answer = inquirer.prompt(question)
+    return SetupOption(answer['menu'])
+
+def setup() -> bool:
+    """Run the setup process for the MTG Python Deckbuilder.
+    
+    This function provides a menu-driven interface to:
+    1. Perform initial setup by downloading and processing card data
+    2. Regenerate CSV files with updated card data
+    3. Perform all tagging processes on the color-sorted csv files
+    
+    The function handles errors gracefully and provides feedback through logging.
+    
+    Returns:
+        bool: True if setup completed successfully, False otherwise
+    """
+    try:
         print('Which setup operation would you like to perform?\n'
               'If this is your first time setting up, do the initial setup.\n'
               'If you\'ve done the basic setup before, you can regenerate the CSV files\n')
         
-        choice = 'Menu'
+        choice = _display_setup_menu()
-        while choice == 'Menu':
-            question = [
-                inquirer.List('menu',
-                              choices=['Initial Setup', 'Regenerate CSV Files', 'Back'],
-                              carousel=True)
-            ]
-            answer = inquirer.prompt(question)
-            choice = answer['menu']
         
-        # Run through initial setup
+        if choice == SetupOption.INITIAL_SETUP:
-        while choice == 'Initial Setup':
+            logging.info('Starting initial setup')
             initial_setup()
-            break
+            logging.info('Initial setup completed successfully')
+            return True
             
-        # Regenerate CSV files
+        elif choice == SetupOption.REGENERATE_CSV:
-        while choice == 'Regenerate CSV Files':
+            logging.info('Starting CSV regeneration')
             regenerate_csvs_all()
-            break    
+            logging.info('CSV regeneration completed successfully')
-            # Go back
+            return True
-        while choice == 'Back':
-            break
-        break
             
-initial_setup()
+        elif choice == SetupOption.BACK:
+            logging.info('Setup cancelled by user')
+            return False
+            
+    except Exception as e:
+        logging.error(f'Error during setup: {e}')
+        raise
+    
+    return False

setup_utils.py

@@ -5,8 +5,18 @@ import requests
 import logging
 from tqdm import tqdm
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import List, Optional, Union, Dict, Any
 
+from settings import (
+    CSV_PROCESSING_COLUMNS,
+    CARD_TYPES_TO_EXCLUDE,
+    NON_LEGAL_SETS,
+    LEGENDARY_OPTIONS,
+    FILL_NA_COLUMNS,
+    SORT_CONFIG,
+    FILTER_CONFIG
+)
+from exceptions import CSVFileNotFoundError, MTGJSONDownloadError, DataFrameProcessingError, ColorFilterError, CommanderValidationError
 from settings import (
     CSV_PROCESSING_COLUMNS,
     CARD_TYPES_TO_EXCLUDE,
@@ -42,6 +52,7 @@ def download_cards_csv(url: str, output_path: Union[str, Path]) -> None:
             url,
             getattr(e.response, 'status_code', None) if hasattr(e, 'response') else None
         ) from e
+
 def check_csv_exists(filepath: Union[str, Path]) -> bool:
     """Check if a CSV file exists at the specified path.
 
@@ -54,7 +65,7 @@ def check_csv_exists(filepath: Union[str, Path]) -> bool:
     return Path(filepath).is_file()
 
 def filter_dataframe(df: pd.DataFrame, banned_cards: List[str]) -> pd.DataFrame:
-    """Apply standard filters to the cards DataFrame.
+    """Apply standard filters to the cards DataFrame using configuration from settings.
 
     Args:
         df: DataFrame to filter
@@ -67,40 +78,52 @@ def filter_dataframe(df: pd.DataFrame, banned_cards: List[str]) -> pd.DataFrame:
         DataFrameProcessingError: If filtering operations fail
     """
     try:
-        # Fill null color identities
+        logging.info('Starting standard DataFrame filtering')
-        df['colorIdentity'] = df['colorIdentity'].fillna('Colorless')
         
-        # Basic filters
+        # Fill null values according to configuration
-        filtered_df = df[
+        for col, fill_value in FILL_NA_COLUMNS.items():
-            (df['layout'] != 'reversible_card') &
+            if col == 'faceName':
-            (df['availability'].str.contains('paper', na=False)) &
+                fill_value = df['name']
-            (df['promoTypes'] != 'playtest') &
+            df[col] = df[col].fillna(fill_value)
-            (~df['securityStamp'].str.contains('Heart|Acorn', na=False))
+            logging.debug(f'Filled NA values in {col} with {fill_value}')
-        ]
+        
+        # Apply basic filters from configuration
+        filtered_df = df.copy()
+        for field, rules in FILTER_CONFIG.items():
+            for rule_type, values in rules.items():
+                if rule_type == 'exclude':
+                    for value in values:
+                        filtered_df = filtered_df[~filtered_df[field].str.contains(value, na=False)]
+                elif rule_type == 'require':
+                    for value in values:
+                        filtered_df = filtered_df[filtered_df[field].str.contains(value, na=False)]
+                logging.debug(f'Applied {rule_type} filter for {field}: {values}')
         
         # Remove illegal sets
         for set_code in NON_LEGAL_SETS:
-            filtered_df = filtered_df[
+            filtered_df = filtered_df[~filtered_df['printings'].str.contains(set_code, na=False)]
-                ~filtered_df['printings'].str.contains(set_code, na=False)
+        logging.debug('Removed illegal sets')
-            ]
 
         # Remove banned cards
         for card in banned_cards:
             filtered_df = filtered_df[~filtered_df['name'].str.contains(card, na=False)]
+        logging.debug('Removed banned cards')
 
         # Remove special card types
         for card_type in CARD_TYPES_TO_EXCLUDE:
             filtered_df = filtered_df[~filtered_df['type'].str.contains(card_type, na=False)]
+        logging.debug('Removed special card types')
 
-        # Handle face names and duplicates
+        # Select columns, sort, and drop duplicates
-        filtered_df['faceName'] = filtered_df['faceName'].fillna(filtered_df['name'])
-        filtered_df = filtered_df.drop_duplicates(subset='faceName', keep='first')
-
-        # Select and sort columns
         filtered_df = filtered_df[CSV_PROCESSING_COLUMNS]
+        filtered_df = filtered_df.sort_values(
+            by=SORT_CONFIG['columns'],
+            key=lambda col: col.str.lower() if not SORT_CONFIG['case_sensitive'] else col
+        )
+        filtered_df = filtered_df.drop_duplicates(subset='faceName', keep='first')
+        logging.info('Completed standard DataFrame filtering')
         
-        return filtered_df.sort_values(by=['name', 'side'], 
+        return filtered_df
-                                     key=lambda col: col.str.lower())
 
     except Exception as e:
         raise DataFrameProcessingError(
@@ -109,8 +132,78 @@ def filter_dataframe(df: pd.DataFrame, banned_cards: List[str]) -> pd.DataFrame:
             str(e)
         ) from e
 
+def filter_by_color_identity(df: pd.DataFrame, color_identity: str) -> pd.DataFrame:
+    """Filter DataFrame by color identity with additional color-specific processing.
+
+    This function extends the base filter_dataframe functionality with color-specific
+    filtering logic. It is used by setup.py's filter_by_color function but provides
+    a more robust and configurable implementation.
+
+    Args:
+        df: DataFrame to filter
+        color_identity: Color identity to filter by (e.g., 'W', 'U,B', 'Colorless')
+
+    Returns:
+        DataFrame filtered by color identity
+
+    Raises:
+        ColorFilterError: If color identity is invalid or filtering fails
+        DataFrameProcessingError: If general filtering operations fail
+    """
+    try:
+        logging.info(f'Filtering cards for color identity: {color_identity}')
+        
+        # Define processing steps for progress tracking
+        steps = [
+            'Validating color identity',
+            'Applying base filtering',
+            'Filtering by color identity',
+            'Performing color-specific processing'
+        ]
+        
+        # Validate color identity
+        with tqdm(total=1, desc='Validating color identity') as pbar:
+            if not isinstance(color_identity, str):
+                raise ColorFilterError(
+                    "Invalid color identity type",
+                    str(color_identity),
+                    "Color identity must be a string"
+                )
+            pbar.update(1)
+            
+        # Apply base filtering
+        with tqdm(total=1, desc='Applying base filtering') as pbar:
+            filtered_df = filter_dataframe(df, [])
+            pbar.update(1)
+            
+        # Filter by color identity
+        with tqdm(total=1, desc='Filtering by color identity') as pbar:
+            filtered_df = filtered_df[filtered_df['colorIdentity'] == color_identity]
+            logging.debug(f'Applied color identity filter: {color_identity}')
+            pbar.update(1)
+            
+        # Additional color-specific processing
+        with tqdm(total=1, desc='Performing color-specific processing') as pbar:
+            # Placeholder for future color-specific processing
+            pbar.update(1)
+        logging.info(f'Completed color identity filtering for {color_identity}')
+        return filtered_df
+        
+    except DataFrameProcessingError as e:
+        raise ColorFilterError(
+            "Color filtering failed",
+            color_identity,
+            str(e)
+        ) from e
+    except Exception as e:
+        raise ColorFilterError(
+            "Unexpected error during color filtering",
+            color_identity,
+            str(e)
+        ) from e
+        
 def process_legendary_cards(df: pd.DataFrame) -> pd.DataFrame:
-    """Process and filter legendary cards for commander eligibility.
+    """Process and filter legendary cards for commander eligibility with comprehensive validation.
 
     Args:
         df: DataFrame containing all cards
@@ -119,28 +212,75 @@ def process_legendary_cards(df: pd.DataFrame) -> pd.DataFrame:
         DataFrame containing only commander-eligible cards
 
     Raises:
-        DataFrameProcessingError: If processing fails
+        CommanderValidationError: If validation fails for legendary status, special cases, or set legality
+        DataFrameProcessingError: If general processing fails
     """
     try:
-        # Filter for legendary creatures and eligible cards
+        logging.info('Starting commander validation process')
-        mask = df['type'].str.contains('|'.join(LEGENDARY_OPTIONS), na=False)
+        validation_steps = [
+            'Checking legendary status',
+            'Validating special cases',
+            'Verifying set legality'
+        ]
 
-        # Add cards that can be commanders
+        filtered_df = df.copy()
-        can_be_commander = df['text'].str.contains(
+        # Step 1: Check legendary status
-            'can be your commander', 
+        try:
-            na=False
+            with tqdm(total=1, desc='Checking legendary status') as pbar:
+                mask = filtered_df['type'].str.contains('|'.join(LEGENDARY_OPTIONS), na=False)
+                if not mask.any():
+                    raise CommanderValidationError(
+                        "No legendary creatures found",
+                        "legendary_check",
+                        "DataFrame contains no cards matching legendary criteria"
                     )
+                filtered_df = filtered_df[mask].copy()
+                logging.debug(f'Found {len(filtered_df)} legendary cards')
+                pbar.update(1)
+        except Exception as e:
+            raise CommanderValidationError(
+                "Legendary status check failed",
+                "legendary_check",
+                str(e)
+            ) from e
 
-        filtered_df = df[mask | can_be_commander].copy()
+        # Step 2: Validate special cases
+        try:
+            with tqdm(total=1, desc='Validating special cases') as pbar:
+                special_cases = df['text'].str.contains('can be your commander', na=False)
+                special_commanders = df[special_cases].copy()
+                filtered_df = pd.concat([filtered_df, special_commanders]).drop_duplicates()
+                logging.debug(f'Added {len(special_commanders)} special commander cards')
+                pbar.update(1)
+        except Exception as e:
+            raise CommanderValidationError(
+                "Special case validation failed",
+                "special_cases",
+                str(e)
+            ) from e
 
-        # Remove illegal sets
+        # Step 3: Verify set legality
+        try:
+            with tqdm(total=1, desc='Verifying set legality') as pbar:
+                initial_count = len(filtered_df)
                 for set_code in NON_LEGAL_SETS:
                     filtered_df = filtered_df[
                         ~filtered_df['printings'].str.contains(set_code, na=False)
                     ]
-
+                removed_count = initial_count - len(filtered_df)
+                logging.debug(f'Removed {removed_count} cards from illegal sets')
+                pbar.update(1)
+        except Exception as e:
+            raise CommanderValidationError(
+                "Set legality verification failed",
+                "set_legality",
+                str(e)
+            ) from e
+        logging.info(f'Commander validation complete. {len(filtered_df)} valid commanders found')
         return filtered_df
 
+    except CommanderValidationError:
+        raise
     except Exception as e:
         raise DataFrameProcessingError(
             "Failed to process legendary cards",

tagger.py

@@ -6417,7 +6417,6 @@ def tag_for_removal(df: pd.DataFrame, color: str) -> None:
 
 
 #start_time = pd.Timestamp.now()
-#regenerate_csvs_all()
 #for color in settings.colors:
 #    load_dataframe(color)
 #duration = (pd.Timestamp.now() - start_time).total_seconds()