Makefile

.PHONY: all run list-configs clean-config show-config list navigate-directory help status validate logs models experiments setup info version train quick-train config sessions monitoring-up monitoring-down monitoring-restart prometheus grafana

# ================================================================================
# 🤖 FedPilot CLI TOOLKIT
# ================================================================================
# Professional command-line tool for federated learning experiments
# ================================================================================

# Framework metadata
FL_NAME := "FedPilot"
FL_VERSION := "v2.0.0"
FL_AUTHOR := ""

# Dynamic configuration discovery
CONFIG_DIR := templates
CONFIGS := $(shell find $(CONFIG_DIR) -name "*.yaml" -type f | sort)
CONFIG_COUNT := $(words $(CONFIGS))

# Directories
LOGS_DIR := logs
MODELS_DIR := saved_models
RESULTS_DIR := results
EXPERIMENTS_DIR := experiments

# Auto-incrementing federation id storage
FED_ID_FILE := .federation_id

# Monitoring and analysis tools
PROMETHEUS_CONFIG=prometheus.yaml
PROMETHEUS_PORT=9090
GRAFANA_PORT=3000

# Container names
PROMETHEUS_CONTAINER := prometheus
GRAFANA_CONTAINER := grafana

# Colors for output
RED := \033[31m
GREEN := \033[32m
YELLOW := \033[33m
BLUE := \033[34m
MAGENTA := \033[35m
CYAN := \033[36m
WHITE := \033[37m
BOLD := \033[1m
RESET := \033[0m

all: help

# ================================================================================
# 🎯 MAIN FEDERATED LEARNING COMMANDS
# ================================================================================

# Interactive configuration selection and training
train:
	@echo "$(BOLD)$(BLUE)🚀 $(FL_NAME) - Training Mode$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ $(CONFIG_COUNT) -eq 0 ]; then \
		echo "$(RED)❌ No configuration files found in $(CONFIG_DIR)$(RESET)"; \
		echo "$(YELLOW)💡 Use 'make setup' to initialize the framework$(RESET)"; \
		exit 1; \
	fi
	@echo "$(GREEN)✓ Found $(CONFIG_COUNT) configuration(s)$(RESET) - Starting interactive navigation..."
	@echo ""
	@$(MAKE) navigate-directory CURRENT_DIR=$(CONFIG_DIR) PARENT_DIR=""

# Quick training with existing config
quick-train:
	@echo "$(BOLD)$(MAGENTA)⚡ Quick Training Mode$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ ! -f ./config.yaml ]; then \
		echo "$(RED)❌ No config.yaml found!$(RESET)"; \
		echo "$(YELLOW)💡 Run 'make train' to select a configuration first$(RESET)"; \
		exit 1; \
	fi
	@echo "$(GREEN)🚀 Starting training with existing configuration...$(RESET)"
	@$(MAKE) run

# Help system - Main entry point
help:
	@echo "$(BOLD)$(BLUE)"
	@echo "🤖 =========================================================================="
	@echo "   $(FL_NAME) CLI Toolkit $(FL_VERSION)"
	@echo "   Professional Federated Learning Command-Line Interface"
	@echo "   Author: $(FL_AUTHOR)"
	@echo "===========================================================================$(RESET)"
	@echo ""
	@echo "$(BOLD)$(GREEN)🎯 MAIN COMMANDS:$(RESET)"
	@echo "  $(BOLD)make train$(RESET)               🚀 Interactive configuration selection + training"
	@echo "  $(BOLD)make quick-train$(RESET)         ⚡ Quick training with existing config"
	@echo "  $(BOLD)make run$(RESET)                 ▶️  Run with current configuration"
	@echo ""
	@echo "$(BOLD)$(CYAN)⚙️  CONFIGURATION MANAGEMENT:$(RESET)"
	@echo "  $(BOLD)make config$(RESET)              📁 Browse and select configurations"
	@echo "  $(BOLD)make show-config$(RESET)         📄 Show current active configuration"
	@echo "  $(BOLD)make validate$(RESET)            ✅ Validate current configuration"
	@echo "  $(BOLD)make clean-config$(RESET)        🗑️  Remove active configuration"
	@echo ""
	@echo "$(BOLD)$(YELLOW)📊 ANALYSIS & MONITORING:$(RESET)"
	@echo "  $(BOLD)make status$(RESET)              📈 Show framework status & statistics"
	@echo "  $(BOLD)make logs$(RESET)                📜 View training logs"
	@echo "  $(BOLD)make plot$(RESET)                📊 Plot training metrics (terminal)"
	@echo "  $(BOLD)make models$(RESET)              🧠 Manage saved models"
	@echo "  $(BOLD)make experiments$(RESET)         🔬 View experiment results"
	@echo "  $(BOLD)make sessions$(RESET)            🖥️  Manage tmux training sessions"
	@echo ""
	@echo "$(BOLD)$(MAGENTA)📡 MONITORING TOOLS:$(RESET)"
	@echo "  $(BOLD)make monitoring-up$(RESET)       📈 Start Prometheus & Grafana containers"
	@echo "  $(BOLD)make monitoring-down$(RESET)     🛑 Stop monitoring containers"
	@echo "  $(BOLD)make monitoring-restart$(RESET)  ♻️  Restart Prometheus & Grafana stack"
	@echo ""
	@echo "$(BOLD)$(MAGENTA)🧪 OPENTELEMETRY INTEGRATION:$(RESET)"
	@echo "  $(BOLD)@monitor_function$(RESET)         🔍 Decorate functions to track metrics"
	@echo "  $(BOLD)GET /metrics$(RESET)              📤 Expose metrics on port 9464 for Prometheus"
	@echo ""
	@echo "$(BOLD)$(MAGENTA)🛠️  UTILITIES:$(RESET)"
	@echo "  $(BOLD)make setup$(RESET)               🏗️  Setup framework environment"
	@echo "  $(BOLD)make list$(RESET)                📋 List all configurations (flat view)"
	@echo "  $(BOLD)make info$(RESET)                ℹ️  System information"
	@echo "  $(BOLD)make version$(RESET)             🏷️  Show version information"
	@echo "  $(BOLD)make help$(RESET)                ❓ Show this help message"
	@echo ""
	@echo "$(BOLD)$(BLUE)💡 EXAMPLES:$(RESET)"
	@echo "  $(GREEN)make train$(RESET)                      # Start interactive training"
	@echo "  $(GREEN)make config$(RESET)                     # Browse configurations only"
	@echo "  $(GREEN)make monitoring-up$(RESET)              # Start monitoring stack (Prometheus + Grafana)"
	@echo "  $(GREEN)curl localhost:9464/metrics$(RESET)     # Preview OpenTelemetry metrics"
	@echo "  $(GREEN)make logs | grep loss$(RESET)           # Filter logs for training loss"
	@echo "  $(GREEN)tmux attach -t fl-cnn-fmnist-143052$(RESET)  # Reconnect to training session"
	@echo ""
	@echo "$(BOLD)$(YELLOW)📚 For detailed documentation, visit: https://FedPilot.github.io$(RESET)"

# Configuration browsing without training
config:
	@echo "$(BOLD)$(CYAN)📁 Configuration Browser$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ $(CONFIG_COUNT) -eq 0 ]; then \
		echo "$(RED)❌ No configuration files found$(RESET)"; \
		exit 1; \
	fi
	@echo "$(GREEN)✓ Found $(CONFIG_COUNT) configuration(s)$(RESET) - Browse mode (no training)"
	@echo ""
	@$(MAKE) navigate-directory CURRENT_DIR=$(CONFIG_DIR) PARENT_DIR="" BROWSE_ONLY=true

# ================================================================================
# 📊 FEDERATED LEARNING ANALYSIS & MONITORING
# ================================================================================

# Framework status and statistics
status:
	@echo "$(BOLD)$(YELLOW)📈 FedPilot Status$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@echo "$(BOLD)🔧 Framework Information:$(RESET)"
	@echo "  Version: $(GREEN)$(FL_VERSION)$(RESET)"
	@echo "  Configurations: $(GREEN)$(CONFIG_COUNT) available$(RESET)"
	@if [ -f ./config.yaml ]; then \
		echo "  Active Config: $(GREEN)✓ Loaded$(RESET)"; \
		config_type=$$(grep -E '^(model_type|dataset_type):' ./config.yaml | head -2 | awk '{print $$2}' | tr -d '"' | paste -sd '/' -); \
		echo "  Config Type: $(CYAN)$$config_type$(RESET)"; \
	else \
		echo "  Active Config: $(RED)❌ None$(RESET)"; \
	fi
	@echo ""
	@echo "$(BOLD)📁 Directory Status:$(RESET)"
	@for dir in $(LOGS_DIR) $(MODELS_DIR) $(RESULTS_DIR) $(EXPERIMENTS_DIR); do \
		if [ -d "$$dir" ]; then \
			count=$$(find "$$dir" -type f 2>/dev/null | wc -l); \
			echo "  $$dir/: $(GREEN)✓ $$count files$(RESET)"; \
		else \
			echo "  $$dir/: $(YELLOW)⚠ Not found$(RESET)"; \
		fi; \
	done
	@echo ""
	@echo "$(BOLD)🖥️  System Resources:$(RESET)"
	@if command -v nvidia-smi >/dev/null 2>&1; then \
		gpu_count=$$(nvidia-smi --list-gpus | wc -l); \
		echo "  GPUs: $(GREEN)✓ $$gpu_count available$(RESET)"; \
	else \
		echo "  GPUs: $(YELLOW)⚠ Not detected$(RESET)"; \
	fi
	@echo "  Python: $(GREEN)$$(python --version 2>&1)$(RESET)"
	@echo "  PyTorch: $(GREEN)$$(python -c "import torch; print(f'v{torch.__version__}')" 2>/dev/null || echo "Not installed")$(RESET)"

# View training logs
logs:
	@echo "$(BOLD)$(BLUE)📜 Training Logs$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ -d "$(LOGS_DIR)" ]; then \
		log_files=$$(find $(LOGS_DIR) -name "*.log" -o -name "*.txt" | head -10); \
		if [ -n "$$log_files" ]; then \
			echo "$(GREEN)Recent log files:$(RESET)"; \
			echo "$$log_files" | nl -w2 -s'. '; \
			echo ""; \
			latest_log=$$(find $(LOGS_DIR) -name "*.log" -o -name "*.txt" | head -1); \
			if [ -n "$$latest_log" ]; then \
				echo "$(BOLD)📄 Latest log ($$latest_log):$(RESET)"; \
				echo "$(CYAN)----------------------------------------$(RESET)"; \
				tail -20 "$$latest_log"; \
			fi; \
		else \
			echo "$(YELLOW)⚠ No log files found in $(LOGS_DIR)$(RESET)"; \
		fi; \
	else \
		echo "$(RED)❌ Logs directory not found$(RESET)"; \
		echo "$(YELLOW)💡 Run training first to generate logs$(RESET)"; \
	fi

# Plot training metrics in terminal
plot:
	@echo "$(BOLD)$(CYAN)📊 Terminal Metrics Plotter$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ ! -d "$(RESULTS_DIR)" ]; then \
		echo "$(RED)❌ Results directory not found$(RESET)"; \
		echo "$(YELLOW)💡 Run training with 'mean_accuracy_to_csv: true' to generate CSV files$(RESET)"; \
		exit 1; \
	fi
	@csv_count=$$(find $(RESULTS_DIR) -name "*.csv" | wc -l); \
	if [ $$csv_count -eq 0 ]; then \
		echo "$(RED)❌ No CSV result files found$(RESET)"; \
		echo "$(YELLOW)💡 Run experiments with CSV logging enabled$(RESET)"; \
		echo "$(CYAN)   Set 'mean_accuracy_to_csv: true' in your config.yaml$(RESET)"; \
		exit 1; \
	fi
	@echo "$(GREEN)✓ Found $$csv_count CSV file(s) - Starting interactive plotter...$(RESET)"
	@echo ""
	@if ! python -c "import plotext" 2>/dev/null; then \
		echo "$(YELLOW)⚠ Installing plotext for terminal plotting...$(RESET)"; \
		pip install plotext>=5.2.0; \
	fi
	@python src/utils/terminal_plotter.py

# Manage saved models
models:
	@echo "$(BOLD)$(MAGENTA)🧠 Model Management$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ -d "$(MODELS_DIR)" ]; then \
		model_files=$$(find $(MODELS_DIR) -name "*.pth" -o -name "*.pt" -o -name "*.pkl" | head -10); \
		if [ -n "$$model_files" ]; then \
			echo "$(GREEN)Saved models:$(RESET)"; \
			for model in $$model_files; do \
				size=$$(du -h "$$model" | cut -f1); \
				echo "  📦 $$(basename "$$model") ($$size)"; \
			done; \
		else \
			echo "$(YELLOW)⚠ No saved models found$(RESET)"; \
		fi; \
	else \
		echo "$(RED)❌ Models directory not found$(RESET)"; \
		mkdir -p $(MODELS_DIR); \
		echo "$(GREEN)✓ Created $(MODELS_DIR) directory$(RESET)"; \
	fi

# View experiment results
experiments:
	@echo "$(BOLD)$(GREEN)🔬 Experiment Results$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ -d "$(RESULTS_DIR)" ]; then \
		result_files=$$(find $(RESULTS_DIR) -name "*.csv" -o -name "*.json" | head -10); \
		if [ -n "$$result_files" ]; then \
			echo "$(GREEN)Result files:$(RESET)"; \
			for result in $$result_files; do \
				echo "  📊 $$(basename "$$result")"; \
			done; \
			echo ""; \
			latest_csv=$$(find $(RESULTS_DIR) -name "*.csv" | head -1); \
			if [ -n "$$latest_csv" ]; then \
				echo "$(BOLD)📈 Latest results preview:$(RESET)"; \
				echo "$(CYAN)----------------------------------------$(RESET)"; \
				head -5 "$$latest_csv"; \
			fi; \
		else \
			echo "$(YELLOW)⚠ No result files found$(RESET)"; \
		fi; \
	else \
		echo "$(RED)❌ Results directory not found$(RESET)"; \
		mkdir -p $(RESULTS_DIR); \
		echo "$(GREEN)✓ Created $(RESULTS_DIR) directory$(RESET)"; \
	fi

# Manage tmux training sessions
sessions:
	@echo "$(BOLD)$(CYAN)🖥️  Tmux Session Management$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if ! command -v tmux >/dev/null 2>&1; then \
		echo "$(RED)❌ tmux not installed$(RESET)"; \
		echo "$(YELLOW)💡 Install: sudo apt install tmux$(RESET)"; \
		exit 1; \
	fi
	@echo "$(BOLD)🔍 Active Federated Learning Sessions:$(RESET)"
	@fl_sessions=$$(tmux list-sessions 2>/dev/null | grep "^fl-" || echo ""); \
	if [ -n "$$fl_sessions" ]; then \
		echo "$$fl_sessions" | while read session; do \
			session_name=$$(echo "$$session" | cut -d: -f1); \
			status=$$(echo "$$session" | grep -o "attached\|detached" || echo "unknown"); \
			if [ "$$status" = "attached" ]; then \
				echo "  $(GREEN)●$(RESET) $$session_name $(GREEN)(active)$(RESET)"; \
			else \
				echo "  $(YELLOW)●$(RESET) $$session_name $(YELLOW)(detached)$(RESET)"; \
			fi; \
		done; \
		echo ""; \
		echo "$(BOLD)🎮 Session Commands:$(RESET)"; \
		echo "  $(CYAN)tmux attach -t <session-name>$(RESET)  # Reconnect to session"; \
		echo "  $(CYAN)tmux kill-session -t <session-name>$(RESET)  # Stop session"; \
		echo "  $(CYAN)tmux list-sessions$(RESET)  # List all sessions"; \
		echo "  $(CYAN)Ctrl+B then D$(RESET)  # Detach from current session"; \
	else \
		echo "  $(YELLOW)⚠ No active FL sessions found$(RESET)"; \
		echo ""; \
		echo "$(CYAN)💡 Start a training session with 'make train'$(RESET)"; \
	fi
	@echo ""
	@all_sessions=$$(tmux list-sessions 2>/dev/null | wc -l); \
	if [ $$all_sessions -gt 0 ]; then \
		echo "$(BLUE)ℹ️  Total tmux sessions: $$all_sessions$(RESET)"; \
	fi

# ================================================================================
# 🛠️ UTILITIES & SETUP
# ================================================================================

# Setup framework environment
setup:
	@echo "$(BOLD)$(BLUE)🏗️  FedPilot Setup$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@echo "$(YELLOW)Setting up directories...$(RESET)"
	@for dir in $(LOGS_DIR) $(MODELS_DIR) $(RESULTS_DIR) $(EXPERIMENTS_DIR); do \
		if [ ! -d "$$dir" ]; then \
			mkdir -p "$$dir"; \
			echo "  $(GREEN)✓ Created $$dir/$(RESET)"; \
		else \
			echo "  $(BLUE)ℹ $$dir/ already exists$(RESET)"; \
		fi; \
	done
	@echo ""
	@echo "$(YELLOW)Checking dependencies...$(RESET)"
	@python -c "import torch, yaml, numpy" 2>/dev/null && echo "  $(GREEN)✓ Core dependencies installed$(RESET)" || echo "  $(RED)❌ Missing dependencies$(RESET)"
	@if command -v tmux >/dev/null 2>&1; then \
		echo "  $(GREEN)✓ tmux available ($$(tmux -V))$(RESET)"; \
	else \
		echo "  $(RED)❌ tmux not installed$(RESET)"; \
		echo "  $(YELLOW)💡 Install tmux: sudo apt install tmux$(RESET)"; \
	fi
	@echo ""
	@echo "$(GREEN)✅ Setup complete! Framework ready to use.$(RESET)"
	@echo "$(CYAN)💡 Run 'make train' to start your first experiment$(RESET)"

# System information
info:
	@echo "$(BOLD)$(CYAN)ℹ️  System Information$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@echo "$(BOLD)🤖 Framework:$(RESET)"
	@echo "  Name: $(FL_NAME)"
	@echo "  Version: $(FL_VERSION)"
	@echo "  Author: $(FL_AUTHOR)"
	@echo ""
	@echo "$(BOLD)🖥️  System:$(RESET)"
	@echo "  OS: $$(uname -s) $$(uname -r)"
	@echo "  Architecture: $$(uname -m)"
	@echo "  Python: $$(python --version 2>&1)"
	@echo "  tmux: $$(tmux -V 2>/dev/null || echo "❌ Not installed")"
	@echo ""
	@echo "$(BOLD)📦 Dependencies:$(RESET)"
	@echo "  PyTorch: $$(python -c "import torch; print(f'v{torch.__version__} (CUDA: {torch.cuda.is_available()})')" 2>/dev/null || echo "Not installed")"
	@echo "  NumPy: $$(python -c "import numpy; print(f'v{numpy.__version__}')" 2>/dev/null || echo "Not installed")"
	@echo "  YAML: $$(python -c "import yaml; print('✓ Available')" 2>/dev/null || echo "❌ Not installed")"

# Version information
version:
	@echo "$(BOLD)$(BLUE)$(FL_NAME) $(FL_VERSION)$(RESET)"
	@echo "Professional Federated Learning CLI Toolkit"
	@echo "Author: $(FL_AUTHOR)"

# Validate current configuration
validate:
	@echo "$(BOLD)$(GREEN)✅ Configuration Validation$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ ! -f ./config.yaml ]; then \
		echo "$(RED)❌ No config.yaml found$(RESET)"; \
		echo "$(YELLOW)💡 Run 'make config' to select a configuration$(RESET)"; \
		exit 1; \
	fi
	@echo "$(GREEN)✓ Configuration file exists$(RESET)"
	@echo "$(YELLOW)Validating YAML syntax...$(RESET)"
	@python -c "import yaml; yaml.safe_load(open('./config.yaml'))" 2>/dev/null && echo "$(GREEN)✓ Valid YAML syntax$(RESET)" || (echo "$(RED)❌ Invalid YAML syntax$(RESET)" && exit 1)

	@echo "$(YELLOW)Checking required fields...$(RESET)"
	@required_fields="device random_seed learning_rate model_type transformer_model_size pretrained_models dataset_type loss_function optimizer data_distribution_kind dirichlet_beta aggregation_strategy fed_avg distance_metric dynamic_sensitivity_percentage sensitivity_percentage remove_common_ids distance_metric_on_parameters number_of_epochs train_batch_size test_batch_size transform_input_size number_of_clients client_sampling_rate pre_computed_data_driven_clustering do_cluster clustering_period federated_learning_rounds stop_avg_accuracy save_before_aggregation_models save_global_models dp_enabled dp_epsilon dp_delta dp_clipping_norm dp_noise_multiplier chunking chunking_with_gradients chunking_parts chunking_random_section shapley shapley_type mean_accuracy_to_csv gpu_index"; \
	device=$$(grep '^device:' ./config.yaml | awk '{print $$2}' | tr -d '"'); \
	for field in $$required_fields; do \
		if grep -q "^$$field:" ./config.yaml; then \
			value=$$(grep "^$$field:" ./config.yaml | awk '{print $$2}' | tr -d '"'); \
			echo "  $(GREEN)✓ $$field: $$value$(RESET)"; \
		else \
			if [ "$$device" = "cpu" ] && [ "$$field" = "gpu_index" ]; then \
				echo "  $(YELLOW)⚠ gpu_index is not required for CPU device$(RESET)"; \
			else \
				echo "  $(RED)❌ Missing required field: $$field$(RESET)"; \
			fi; \
		fi; \
	done
	@echo "$(GREEN)🎉 Configuration validation complete!$(RESET)"

# Interactive directory navigation
navigate-directory:
	@current_dir=$(CURRENT_DIR); \
	parent_dir=$(PARENT_DIR); \
	browse_only=$(BROWSE_ONLY); \
	echo "$(BOLD)📁 Current directory: $(CYAN)$$current_dir$(RESET)"; \
	echo ""; \
	dirs=$$(find "$$current_dir" -mindepth 1 -maxdepth 1 -type d | sort); \
	yamls=$$(find "$$current_dir" -mindepth 1 -maxdepth 1 -name "*.yaml" | sort); \
	counter=1; \
	items=""; \
	types=""; \
	if [ -n "$$parent_dir" ]; then \
		echo "$$counter. $(YELLOW)🔙 [Go Back]$(RESET)"; \
		items="$$items$$counter:BACK "; \
		types="$$types$$counter:back "; \
		counter=$$((counter + 1)); \
	fi; \
	for dir in $$dirs; do \
		dir_name=$$(basename "$$dir"); \
		echo "$$counter. $(BLUE)📂 $$dir_name/$(RESET)"; \
		items="$$items$$counter:$$dir "; \
		types="$$types$$counter:dir "; \
		counter=$$((counter + 1)); \
	done; \
	for yaml in $$yamls; do \
		yaml_name=$$(basename "$$yaml"); \
		echo "$$counter. $(GREEN)📄 $$yaml_name$(RESET)"; \
		items="$$items$$counter:$$yaml "; \
		types="$$types$$counter:file "; \
		counter=$$((counter + 1)); \
	done; \
	if [ $$counter -eq 1 ]; then \
		echo "$(RED)❌ No directories or YAML files found in this location.$(RESET)"; \
		if [ -n "$$parent_dir" ]; then \
			echo "Going back to parent directory..."; \
			$(MAKE) navigate-directory CURRENT_DIR="$$parent_dir" PARENT_DIR="$$(dirname "$$parent_dir" 2>/dev/null || echo "")" BROWSE_ONLY="$$browse_only"; \
		else \
			echo "No configurations available."; \
			exit 1; \
		fi; \
	else \
		echo ""; \
		printf "$(BOLD)Enter your choice (1-$$((counter-1))) or 'q' to quit: $(RESET)"; \
		read choice; \
		if [ "$$choice" = "q" ] || [ "$$choice" = "Q" ]; then \
			echo "$(YELLOW)👋 Goodbye!$(RESET)"; \
			exit 0; \
		fi; \
		if echo "$$choice" | grep -q '^[0-9]\+$$' && [ "$$choice" -ge 1 ] && [ "$$choice" -lt "$$counter" ]; then \
			selected_item=$$(echo "$$items" | tr ' ' '\n' | grep "^$$choice:" | cut -d: -f2-); \
			selected_type=$$(echo "$$types" | tr ' ' '\n' | grep "^$$choice:" | cut -d: -f2-); \
			if [ "$$selected_type" = "back" ]; then \
				echo "$(YELLOW)🔙 Going back...$(RESET)"; \
				echo ""; \
				$(MAKE) navigate-directory CURRENT_DIR="$$parent_dir" PARENT_DIR="$$(dirname "$$parent_dir" 2>/dev/null || echo "")" BROWSE_ONLY="$$browse_only"; \
			elif [ "$$selected_type" = "dir" ]; then \
				echo "$(BLUE)📂 Entering directory: $$(basename "$$selected_item")$(RESET)"; \
				echo ""; \
				$(MAKE) navigate-directory CURRENT_DIR="$$selected_item" PARENT_DIR="$$current_dir" BROWSE_ONLY="$$browse_only"; \
			elif [ "$$selected_type" = "file" ]; then \
				echo ""; \
				echo "$(GREEN)📄 Selected: $$selected_item$(RESET)"; \
				echo "Preparing config.yaml..."; \
				tmp_config=$$(mktemp); \
				echo "# Source: $$selected_item" > "$$tmp_config"; \
				echo "" >> "$$tmp_config"; \
				echo "$(CYAN)🔧 Choose production mode: [prod/dev]$(RESET)"; \
				read -p "Enter mode (default: dev): " mode_choice; \
				echo ""; \
				echo "$(CYAN)💻 Choose device: [cpu/gpu]$(RESET)"; \
				read -p "Enter device type (default: cpu): " device_choice; \
				device_choice=$${device_choice:-cpu}; \
				if [ "$$device_choice" = "gpu" ]; then \
					device_line="device: cuda"; \
					gpu_count=$$(python -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo "0"); \
					if [ "$$gpu_count" -eq 0 ]; then \
						echo "$(RED)❌ No GPUs detected by PyTorch! Falling back to CPU.$(RESET)"; \
						echo "device: cpu" >> "$$tmp_config"; \
					else \
						if [ "$$gpu_count" -gt 1 ]; then \
							echo "$(CYAN)🖥️  Found $$gpu_count GPUs. Enter GPU index (0 to $$((gpu_count - 1))):$(RESET)"; \
							read -p "GPU index: " gpu_index; \
							gpu_index=$${gpu_index:-0}; \
						else \
							gpu_index=0; \
						fi; \
						echo "$$device_line" >> "$$tmp_config"; \
						echo "gpu_index: $$gpu_index" >> "$$tmp_config"; \
					fi; \
				else \
					if [ "$$device_choice" != "cpu" ]; then \
						echo "$(RED)❌ Invalid device choice! Defaulting to CPU.$(RESET)"; \
					fi; \
					echo "$(CYAN)💻 Using CPU for training$(RESET)"; \
					echo "device: cpu" >> "$$tmp_config"; \
				fi; \
				echo "" >> "$$tmp_config"; \
				if [ -f "$(FED_ID_FILE)" ]; then \
					last_id=$$(cat "$(FED_ID_FILE)"); \
				else \
					last_id="0.0.0"; \
				fi; \
				p1=$${last_id%%.*}; rest=$${last_id#*.}; \
				p2=$${rest%%.*};    p3=$${rest##*.}; \
				p3=$$((p3 + 1)); \
				if [ $$p3 -ge 100 ]; then \
					p3=0; p2=$$((p2 + 1)); \
					if [ $$p2 -ge 100 ]; then \
						p2=0; p1=$$((p1 + 1)); \
					fi; \
				fi; \
				new_id="$$p1.$$p2.$$p3"; \
				echo "$$new_id" > "$(FED_ID_FILE)"; \
				echo "federation_id: '$$new_id'" >> "$$tmp_config"; \
				echo ""; \
				echo "$(CYAN)Select federated_learning_schema:$(RESET)"; \
				echo "  1. TraditionalFederatedLearning"; \
				echo "  2. DecentralizedFederatedLearning"; \
				echo "  3. ClusterFederatedLearningSchema"; \
				read -p "Enter choice [1-3] (default: 2): " schema_choice; \
				echo ""; \
				case "$$schema_choice" in \
					1) schema="TraditionalFederatedLearning" ;; \
					2|"") schema="DecentralizedFederatedLearning" ;; \
					3) schema="ClusterFederatedLearningSchema" ;; \
					*) schema="DecentralizedFederatedLearning" ;; \
				esac; \
				echo "federated_learning_schema: '$$schema'" >> "$$tmp_config"; \
				read -p "Enable draw_topology? [y/N]: " draw_topo; \
				if [ "$$draw_topo" = "y" ] || [ "$$draw_topo" = "Y" ]; then \
					echo "draw_topology: true" >> "$$tmp_config"; \
				else \
					echo "draw_topology: false" >> "$$tmp_config"; \
				fi; \
				echo ""; \
				echo "$(CYAN)Select federated_learning_topology:$(RESET)"; \
				echo "  1. star"; \
				echo "  2. k_connect"; \
				echo "  3. ring"; \
				echo "  4. custom"; \
				read -p "Enter choice [1-4] (default: 2): " topo_choice; \
				echo ""; \
				case "$$topo_choice" in \
					1) topo="star" ;; \
					2|"") topo="k_connect" ;; \
					3) topo="ring" ;; \
					4) topo="custom" ;; \
					*) topo="k_connect" ;; \
				esac; \
				echo "federated_learning_topology: '$$topo'" >> "$$tmp_config"; \
				read -p "Enter adjacency_matrix_file_name (default: adjacency_matrix_2.csv): " matrix_file; \
				echo ""; \
				matrix_file=$${matrix_file:-adjacency_matrix_2.csv}; \
				echo "adjacency_matrix_file_name: '$$matrix_file'" >> "$$tmp_config"; \
				read -p "Enter client_k_neighbors (default: 2): " k_neighbors; \
				echo ""; \
				k_neighbors=$${k_neighbors:-2}; \
				echo "client_k_neighbors: $$k_neighbors" >> "$$tmp_config"; \
				echo "$(CYAN)Select client_role:$(RESET)"; \
				echo "  1. train"; \
				echo "  2. test"; \
				echo "  3. eval"; \
				echo "  4. train-test"; \
				echo "  5. train-eval"; \
				echo "  6. test-eval"; \
				echo "  7. train-test-eval"; \
				read -p "Enter choice [1-7] (default: 1): " role_choice; \
				echo ""; \
				case "$$role_choice" in \
					1|"") role="train" ;; \
					2) role="test" ;; \
					3) role="eval" ;; \
					4) role="train-test" ;; \
					5) role="train-eval" ;; \
					6) role="test-eval" ;; \
					7) role="train-test-eval" ;; \
					*) role="train" ;; \
				esac; \
				echo "client_role: '$$role'" >> "$$tmp_config"; \
				echo "" >> "$$tmp_config"; \
				grep -vE '^\s*(device:|gpu_index:|federation_id:|federated_learning_schema:|draw_topology:|federated_learning_topology:|adjacency_matrix_file_name:|client_k_neighbors:|client_role:|production_mode:)' "$$selected_item" >> "$$tmp_config"; \
				if [ "$$mode_choice" = "prod" ] || [ "$$mode_choice" = "production" ]; then \
					echo "$(GREEN)🏭 Production mode enabled$(RESET)"; \
					echo "production_mode: true" >> "$$tmp_config"; \
				else \
					if [ -n "$$mode_choice" ] && [ "$$mode_choice" != "dev" ] && [ "$$mode_choice" != "development" ]; then \
						echo "$(RED)❌ Invalid mode choice! Defaulting to development.$(RESET)"; \
					fi; \
					echo "$(CYAN)🔧 Development mode enabled$(RESET)"; \
					echo "production_mode: false" >> "$$tmp_config"; \
				fi; \
				mv "$$tmp_config" ./config.yaml; \
				echo "$(GREEN)✓ Clean configuration created: ./config.yaml$(RESET)"; \
				echo ""; \
				if [ "$$browse_only" = "true" ]; then \
					echo "$(CYAN)ℹ️  Configuration selected. Use 'make run' to start training.$(RESET)"; \
				else \
					$(MAKE) run; \
				fi; \
			fi; \
		else \
			echo ""; \
			echo "$(RED)❌ Invalid choice. Please enter a number from 1 to $$((counter-1)) or 'q' to quit.$(RESET)"; \
			echo ""; \
			$(MAKE) navigate-directory CURRENT_DIR="$$current_dir" PARENT_DIR="$$parent_dir" BROWSE_ONLY="$$browse_only"; \
		fi; \
	fi

list-configs:
	@counter=1; \
	for config in $(CONFIGS); do \
		relative_path=$${config#$(CONFIG_DIR)/}; \
		echo "$$counter. $$relative_path"; \
		counter=$$((counter + 1)); \
	done

# ================================================================================
# 📋 LEGACY & COMPATIBILITY COMMANDS
# ================================================================================

# Helper target to just list configurations without running (flat view)
list:
	@echo "$(BOLD)$(CYAN)📋 All Available Configurations (Flat View)$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@$(MAKE) list-configs
	@echo ""
	@echo "$(YELLOW)💡 Use 'make train' for interactive navigation$(RESET)"

# Clean up the copied configuration file
clean-config:
	@echo "$(BOLD)$(RED)🗑️  Configuration Cleanup$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ -f ./config.yaml ]; then \
		echo "$(YELLOW)Removing config.yaml...$(RESET)"; \
		rm ./config.yaml; \
		echo "$(GREEN)✓ config.yaml removed successfully!$(RESET)"; \
	else \
		echo "$(BLUE)ℹ️  No config.yaml file found to remove.$(RESET)"; \
	fi

# Show current active configuration
show-config:
	@echo "$(BOLD)$(BLUE)📄 Current Active Configuration$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ -f ./config.yaml ]; then \
		echo "$(GREEN)✓ Configuration loaded:$(RESET)"; \
		echo "$(CYAN)----------------------------------------$(RESET)"; \
		head -20 ./config.yaml; \
		echo "$(CYAN)... (showing first 20 lines)$(RESET)"; \
		echo ""; \
		echo "$(BLUE)📁 Full config available in ./config.yaml$(RESET)"; \
		echo ""; \
		config_type=$$(grep -E '^(model_type|dataset_type):' ./config.yaml | head -2 | awk '{print $$2}' | tr -d '"' | paste -sd '/' -); \
		echo "$(BOLD)Configuration Type: $(CYAN)$$config_type$(RESET)"; \
	else \
		echo "$(RED)❌ No active configuration found.$(RESET)"; \
		echo "$(YELLOW)💡 Run 'make train' to select a configuration.$(RESET)"; \
	fi

# ================================================================================
# 🚀 EXECUTION ENGINE
# ================================================================================

run:
	@echo "$(BOLD)$(GREEN)🚀 FedPilot - Execution$(RESET)"
	@echo "$(CYAN)========================================================$(RESET)"
	@if [ ! -f ./config.yaml ]; then \
		echo "$(RED)❌ No config.yaml found!$(RESET)"; \
		echo "$(YELLOW)💡 Please run 'make train' first to select a configuration.$(RESET)"; \
		exit 1; \
	fi

	@echo ""
	@echo "$(CYAN)🔍 Checking monitoring setup...$(RESET)"
	@if lsof -i :8080 >/dev/null 2>&1; then \
		echo "$(YELLOW)⚠️  Port 8080 is already in use$(RESET)"; \
		echo "$(YELLOW)⚠️  Monitoring may not work (Ray is running or port is blocked)$(RESET)"; \
	else \
		ray start --head --metrics-export-port=8080 >/dev/null 2>&1 && \
			echo "$(GREEN)✅ Monitoring is working (Ray started on port 8080)$(RESET)" || \
			echo "$(YELLOW)⚠️  Monitoring may not work (Ray is already running or another issue occurred)$(RESET)"; \
	fi
	@echo ""


	@if ! command -v tmux >/dev/null 2>&1; then \
		echo "$(RED)❌ tmux not found!$(RESET)"; \
		echo "$(YELLOW)💡 Install tmux: sudo apt install tmux$(RESET)"; \
		echo "$(CYAN)ℹ️  Running without tmux...$(RESET)"; \
		python ./main.py; \
		exit 0; \
	fi
	@config_source=$$(grep '^# Source:' ./config.yaml 2>/dev/null | sed 's/# Source: //' | sed 's|templates/||' | sed 's|\.yaml$$||' | tr '/' '-' | tr '.' '_' | tr ':' '_' || echo "unknown"); \
	config_type=$$(grep -E '^(model_type|dataset_type):' ./config.yaml | head -2 | awk '{print $$2}' | tr -d '"' | paste -sd '/' -); \
	timestamp=$$(date +"%H%M%S"); \
	session_name="fl-$$config_source-$$timestamp"; \
	echo "$(GREEN)✓ Configuration loaded: $(CYAN)$$config_type$(RESET)"; \
	echo "$(BLUE)📁 Config source: $(CYAN)$$config_source$(RESET)"; \
	echo "$(YELLOW)🖥️  Creating tmux session: $(MAGENTA)$$session_name$(RESET)"; \
	echo "$(CYAN)💡 Use 'tmux attach -t $$session_name' to reconnect$(RESET)"; \
	echo "$(CYAN)💡 Use 'tmux list-sessions' to see all sessions$(RESET)"; \
	echo "$(CYAN)💡 Use 'Ctrl+B then D' to detach from session$(RESET)"; \
	echo ""; \
	tmux new-session -d -s "$$session_name" -c "$$(pwd)" "echo '$(BOLD)$(GREEN)🚀 Federated Learning Training$(RESET)'; echo 'Session: $$session_name'; echo 'Config: $$config_type'; echo ''; python ./main.py; echo ''; echo '$(GREEN)✅ Training completed!$(RESET)'; echo 'Press Enter to exit session...'; read"; \
	echo "$(GREEN)✅ Training started in tmux session: $(MAGENTA)$$session_name$(RESET)"; \
	echo "$(YELLOW)📊 Attaching to session... (Press Ctrl+B then D to detach)$(RESET)"; \
	sleep 1; \
	tmux attach -t "$$session_name"


compose-file := docker-compose.yml

monitoring-up:
	@docker compose -f $(compose-file) up -d
	@echo ""
	@echo "✅ Services started:"
	@echo "- Prometheus: http://localhost:$(PROMETHEUS_PORT)"
	@echo "- Grafana:    http://localhost:$(GRAFANA_PORT)"
	@echo "- Jaeger UI:  http://localhost:16686"

monitoring-down:
	@docker compose -f $(compose-file) down

monitoring-restart: monitoring-down monitoring-up


.PHONY: monitoring-up monitoring-down monitoring-restart \
        grafana prometheus