From ed60e3ed32b1cc31c94b44491c2cadd36a9b8b13 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 8 Aug 2025 22:58:19 +0000 Subject: [PATCH 1/2] refactor: remove unnecessary whitespace Blank lines should not contain any tabs or spaces. --- debug_ci_robust.py | 1 - debug_ci_timing.py | 1 - debug_rate_limiter.py | 16 +- deployment/api_server.py | 20 +- deployment/cloud-run/robust_predict.py | 72 +++--- deployment/gcp/predict.py | 36 +-- deployment/inference.py | 30 +-- deployment/local/api_server.py | 86 +++---- deployment/secure_api_server.py | 108 ++++----- scripts/ci/model_calibration_test.py | 12 +- scripts/ci/run_full_ci_pipeline.py | 134 +++++------ scripts/ci/whisper_transcription_test.py | 4 +- .../deployment/complete_project_deployment.py | 62 ++--- .../create_model_deployment_package.py | 88 +++---- scripts/deployment/deploy_locally.py | 96 ++++---- scripts/deployment/deploy_to_gcp_vertex_ai.py | 144 +++++------ .../save_trained_model_for_deployment.py | 56 ++--- scripts/legacy/add_comprehensive_features.py | 10 +- scripts/legacy/add_wandb_setup.py | 14 +- .../legacy/comprehensive_model_validation.py | 120 +++++----- scripts/legacy/convert_to_onnx.py | 4 +- scripts/legacy/create_bulletproof_cell.py | 86 +++---- .../legacy/create_final_bulletproof_cell.py | 86 +++---- .../legacy/create_unique_fallback_dataset.py | 26 +- scripts/legacy/deep_model_analysis.py | 72 +++--- scripts/legacy/evaluate_whisper_wer.py | 2 +- scripts/legacy/expand_journal_dataset.py | 52 ++-- scripts/legacy/finalize_emotion_model.py | 78 +++--- scripts/legacy/improve_model_f1.py | 2 +- scripts/legacy/integrate_cmu_mosei.py | 84 +++---- scripts/legacy/reorganize_model_directory.py | 50 ++-- .../legacy/retrain_with_expanded_dataset.py | 110 ++++----- scripts/legacy/retrain_with_validation.py | 20 +- scripts/legacy/simple_cmu_mosei_download.py | 68 +++--- scripts/legacy/simple_f1_evaluation.py | 42 ++-- scripts/legacy/validate_model_performance.py | 102 ++++---- scripts/maintenance/emergency_f1_fix.py | 138 +++++------ scripts/maintenance/fix_code_quality.py | 14 +- scripts/maintenance/fix_import_paths.py | 24 +- scripts/maintenance/fix_label_mapping.py | 110 ++++----- .../fix_linting_issues_conservative.py | 2 +- .../fix_model_architecture_mismatch.py | 8 +- .../maintenance/fix_model_reconfiguration.py | 8 +- scripts/maintenance/fix_remaining_linting.py | 42 ++-- scripts/maintenance/quick_label_fix.py | 26 +- scripts/testing/check_model_health.py | 14 +- .../testing/create_journal_test_dataset.py | 44 ++-- scripts/testing/debug_dataset_structure.py | 2 +- scripts/testing/debug_go_emotions_labels.py | 26 +- scripts/testing/debug_label_mismatch.py | 74 +++--- scripts/testing/debug_model_loading.py | 10 +- scripts/testing/debug_rate_limiter_test.py | 1 - scripts/testing/final_temperature_test.py | 34 +-- .../testing/mega_comprehensive_model_test.py | 212 ++++++++--------- scripts/testing/mega_test_summary.py | 26 +- scripts/testing/setup_model_testing.py | 44 ++-- scripts/testing/simple_model_test.py | 40 ++-- scripts/testing/simple_rate_limiter_test.py | 1 - scripts/testing/simple_temperature_test.py | 10 +- scripts/training/SAMO_Colab_Setup.py | 84 +++---- .../add_advanced_features_to_notebook.py | 60 ++--- scripts/training/bulletproof_training.py | 174 +++++++------- scripts/training/complete_simple_notebook.py | 10 +- ...omprehensive_domain_adaptation_training.py | 224 +++++++++--------- .../create_bulletproof_colab_notebook.py | 6 +- .../create_colab_expanded_training.py | 6 +- scripts/training/create_colab_notebook.py | 6 +- .../training/create_comprehensive_notebook.py | 8 +- .../create_corrected_specialized_notebook.py | 6 +- .../create_emotion_specialized_notebook.py | 6 +- .../create_final_bulletproof_notebook.py | 6 +- .../training/create_final_colab_notebook.py | 10 +- .../create_fixed_bulletproof_notebook.py | 6 +- .../training/create_fixed_colab_notebook.py | 6 +- scripts/training/create_fixed_notebook.py | 6 +- ...ate_fixed_specialized_training_notebook.py | 8 +- .../create_improved_expanded_notebook.py | 6 +- .../create_minimal_working_notebook.py | 8 +- .../create_model_ensemble_notebook.py | 6 +- .../create_simple_ultimate_notebook.py | 8 +- .../create_ultimate_bulletproof_notebook.py | 8 +- scripts/training/debug_colab_compatibility.py | 86 +++---- scripts/training/debug_training_loss.py | 2 +- scripts/training/final_combined_training.py | 78 +++--- scripts/training/final_expanded_training.py | 20 +- scripts/training/fix_imports_in_notebook.py | 8 +- scripts/training/fix_notebook_json.py | 12 +- .../training/fix_preprocessing_in_notebook.py | 16 +- scripts/training/fix_training_arguments.py | 8 +- scripts/training/fixed_focal_training.py | 120 +++++----- .../training/full_dataset_focal_training.py | 4 +- scripts/training/full_focal_training.py | 2 +- scripts/training/full_scale_focal_training.py | 2 +- .../improve_expanded_training_notebook.py | 48 ++-- .../robust_domain_adaptation_training.py | 102 ++++---- scripts/training/setup_colab_environment.py | 44 ++-- .../summarize_comprehensive_notebook.py | 26 +- .../training/summarize_ultimate_notebook.py | 18 +- .../training/validate_improved_notebook.py | 34 +-- scripts/validation/check_dependencies.py | 46 ++-- .../validation/validate_security_config.py | 98 ++++---- src/api_rate_limiter.py | 134 +++++------ src/input_sanitizer.py | 130 +++++----- .../emotion_detection/dataset_loader.py | 38 +-- src/models/secure_loader/integrity_checker.py | 80 +++---- src/models/secure_loader/model_validator.py | 138 +++++------ src/models/secure_loader/sandbox_executor.py | 60 ++--- .../secure_loader/secure_model_loader.py | 128 +++++----- src/monitoring/dashboard.py | 114 ++++----- src/security/jwt_manager.py | 32 +-- src/security_headers.py | 112 ++++----- src/unified_ai_api.py | 178 +++++++------- tests/conftest.py | 4 +- 113 files changed, 2745 insertions(+), 2749 deletions(-) diff --git a/debug_ci_robust.py b/debug_ci_robust.py index 0519ecba6..e69de29bb 100644 --- a/debug_ci_robust.py +++ b/debug_ci_robust.py @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/debug_ci_timing.py b/debug_ci_timing.py index 0519ecba6..e69de29bb 100644 --- a/debug_ci_timing.py +++ b/debug_ci_timing.py @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/debug_rate_limiter.py b/debug_rate_limiter.py index b3082996c..7765f9698 100644 --- a/debug_rate_limiter.py +++ b/debug_rate_limiter.py @@ -13,15 +13,15 @@ def debug_rate_limiter(): """Debug the rate limiter behavior.""" print("๐Ÿ” Debugging Rate Limiter Issue") print("=" * 50) - + # Create config with minimal settings (same as test) config = RateLimitConfig(requests_per_minute=1, burst_size=1) print(f"Config: requests_per_minute={config.requests_per_minute}, burst_size={config.burst_size}") - + rate_limiter = TokenBucketRateLimiter(config) print(f"Initial buckets: {rate_limiter.buckets}") print(f"Initial last_refill: {rate_limiter.last_refill}") - + # Test first request print("\n๐Ÿš€ Testing First Request...") allowed1, reason1, meta1 = rate_limiter.allow_request("127.0.0.1") @@ -29,27 +29,27 @@ def debug_rate_limiter(): print(f"Meta: {meta1}") print(f"Buckets after first request: {rate_limiter.buckets}") print(f"Last refill after first request: {rate_limiter.last_refill}") - + # Test second request print("\n๐Ÿš€ Testing Second Request...") allowed2, reason2, meta2 = rate_limiter.allow_request("127.0.0.1") print(f"Second request - Allowed: {allowed2}, Reason: {reason2}") print(f"Meta: {meta2}") print(f"Buckets after second request: {rate_limiter.buckets}") - + # Check what's in the bucket for this client client_key = rate_limiter._get_client_key("127.0.0.1") print(f"\n๐Ÿ”‘ Client key: {client_key}") print(f"Bucket value for client: {rate_limiter.buckets[client_key]}") print(f"Last refill time for client: {rate_limiter.last_refill[client_key]}") - + # Check if client is blocked print(f"Client blocked: {rate_limiter._is_client_blocked(client_key)}") print(f"Blocked clients: {rate_limiter.blocked_clients}") - + # Check concurrent requests print(f"Concurrent requests: {rate_limiter.concurrent_requests}") - + # Check request history print(f"Request history: {list(rate_limiter.request_history[client_key])}") diff --git a/deployment/api_server.py b/deployment/api_server.py index d97caa20c..49199f993 100644 --- a/deployment/api_server.py +++ b/deployment/api_server.py @@ -37,17 +37,17 @@ def predict_emotion(): """Predict emotion for given text""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + try: data = request.get_json() text = data.get('text', '') - + if not text: return jsonify({'error': 'No text provided'}), 400 - + result = detector.predict(text) return jsonify(result) - + except Exception as e: logger.error(f"Prediction error: {e}") return jsonify({'error': str(e)}), 500 @@ -57,17 +57,17 @@ def predict_batch(): """Predict emotions for multiple texts""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + try: data = request.get_json() texts = data.get('texts', []) - + if not texts: return jsonify({'error': 'No texts provided'}), 400 - + results = detector.predict_batch(texts) return jsonify({'results': results}) - + except Exception as e: logger.error(f"Batch prediction error: {e}") return jsonify({'error': str(e)}), 500 @@ -77,7 +77,7 @@ def get_emotions(): """Get list of supported emotions""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + return jsonify({ 'emotions': list(detector.label_encoder.classes_), 'count': len(detector.label_encoder.classes_) @@ -94,5 +94,5 @@ def get_emotions(): print(" - POST /predict_batch - Batch prediction") print(" - GET /emotions - List emotions") print("=" * 50) - + app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/deployment/cloud-run/robust_predict.py b/deployment/cloud-run/robust_predict.py index fab62eab5..65e199108 100644 --- a/deployment/cloud-run/robust_predict.py +++ b/deployment/cloud-run/robust_predict.py @@ -41,42 +41,42 @@ def load_model(): """Load the emotion detection model""" global model, tokenizer, emotion_mapping, model_loading, model_loaded, model_lock - + with model_lock: if model_loading or model_loaded: return - + model_loading = True logger.info("๐Ÿ”„ Starting model loading...") - + try: # Get model path model_path = Path("/app/model") logger.info(f"๐Ÿ“ Loading model from: {model_path}") - + # Check if model files exist if not model_path.exists(): raise FileNotFoundError(f"Model directory not found: {model_path}") - + # Load tokenizer and model logger.info("๐Ÿ“ฅ Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained("roberta-base") - + logger.info("๐Ÿ“ฅ Loading model...") model = AutoModelForSequenceClassification.from_pretrained(str(model_path)) - + # Set device (CPU for Cloud Run) device = torch.device('cpu') model.to(device) model.eval() - + emotion_mapping = EMOTION_MAPPING model_loaded = True model_loading = False - + logger.info(f"โœ… Model loaded successfully on {device}") logger.info(f"๐ŸŽฏ Supported emotions: {emotion_mapping}") - + except Exception: model_loading = False logger.exception("โŒ Failed to load model") @@ -87,7 +87,7 @@ def load_model(): def predict_emotion(text): """Predict emotion for given text""" global model, tokenizer, emotion_mapping - + if not model_loaded: raise RuntimeError("Model not loaded") @@ -99,17 +99,17 @@ def predict_emotion(text): # Tokenize inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=MAX_INPUT_LENGTH, padding=True) - + # Predict with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + # Map to emotion name emotion = emotion_mapping[predicted_class] - + return { "emotion": emotion, "confidence": confidence, @@ -120,7 +120,7 @@ def ensure_model_loaded(): """Ensure model is loaded before processing requests""" if not model_loaded and not model_loading: load_model() - + if not model_loaded: raise RuntimeError("Model not loaded") @@ -159,27 +159,27 @@ def predict(): try: # Ensure model is loaded ensure_model_loaded() - + # Content-type validation if not request.is_json: return jsonify({'error': 'Content-Type must be application/json'}), 400 - + try: data = request.get_json() except Exception: return jsonify({'error': 'Invalid JSON data'}), 400 - + if not data: return jsonify({'error': 'No JSON data provided'}), 400 - + text = data.get('text', '') if not text: return jsonify({'error': 'No text provided'}), 400 - + # Make prediction result = predict_emotion(text) return jsonify(result) - + except Exception: return create_error_response('Prediction processing failed. Please try again later.') @@ -189,31 +189,31 @@ def predict_batch(): try: # Ensure model is loaded ensure_model_loaded() - + # Content-type validation if not request.is_json: return jsonify({'error': 'Content-Type must be application/json'}), 400 - + try: data = request.get_json() except Exception: return jsonify({'error': 'Invalid JSON data'}), 400 - + if not data: return jsonify({'error': 'No JSON data provided'}), 400 - + texts = data.get('texts', []) if not texts: return jsonify({'error': 'No texts provided'}), 400 - + # Make predictions results = [] for text in texts: result = predict_emotion(text) results.append(result) - + return jsonify({'results': results}) - + except Exception: return create_error_response('Batch prediction processing failed. Please try again later.') @@ -260,34 +260,34 @@ def initialize_model(): logger.info(" - GET /emotions - List emotions") logger.info(" - GET /model_status - Model status") logger.info("=" * 50) - + # Load model immediately try: load_model() except Exception: logger.exception("Failed to load model on startup") - + # Get port from environment (Cloud Run requirement) port = int(os.environ.get('PORT', 8080)) - + # Use production WSGI server for better performance and reliability import gunicorn.app.base - + class StandaloneApplication(gunicorn.app.base.BaseApplication): def __init__(self, app, options=None): self.options = options or {} self.application = app super().__init__() - + def load_config(self): config = {key: value for key, value in self.options.items() if key in self.cfg.settings and value is not None} for key, value in config.items(): self.cfg.set(key.lower(), value) - + def load(self): return self.application - + options = { 'bind': f'0.0.0.0:{port}', 'workers': 1, # Single worker for Cloud Run @@ -300,5 +300,5 @@ def load(self): 'error_logfile': '-', 'loglevel': 'info' } - + StandaloneApplication(app, options).run() \ No newline at end of file diff --git a/deployment/gcp/predict.py b/deployment/gcp/predict.py index 73fc60bff..3a795893d 100644 --- a/deployment/gcp/predict.py +++ b/deployment/gcp/predict.py @@ -18,44 +18,44 @@ def __init__(self): """Initialize the model.""" self.model_path = os.path.join(os.getcwd(), "model") print(f"Loading model from: {self.model_path}") - + try: self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + # Move to GPU if available if torch.cuda.is_available(): self.model = self.model.to('cuda') print("โœ… Model moved to GPU") else: print("โš ๏ธ CUDA not available, using CPU") - + self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] print("โœ… Model loaded successfully") - + except Exception as e: print(f"โŒ Failed to load model: {str(e)}") raise - + def predict(self, text): """Make a prediction.""" try: # Tokenize input inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512) - + if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + # Get prediction with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Get all probabilities all_probs = probabilities[0].cpu().numpy() - + # Get predicted emotion if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] @@ -63,7 +63,7 @@ def predict(self, text): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + # Create response response = { 'text': text, @@ -80,9 +80,9 @@ def predict(self, text): 'average_confidence': '83.9%' } } - + return response - + except Exception as e: print(f"Prediction error: {str(e)}") raise @@ -105,19 +105,19 @@ def predict(): """Prediction endpoint.""" try: data = request.get_json() - + if not data or 'text' not in data: return jsonify({'error': 'No text provided'}), 400 - + text = data['text'] if not text.strip(): return jsonify({'error': 'Empty text provided'}), 400 - + # Make prediction result = model.predict(text) - + return jsonify(result) - + except Exception as e: print(f"Prediction endpoint error: {str(e)}") return jsonify({'error': str(e)}), 500 @@ -152,6 +152,6 @@ def home(): print("") print("๐Ÿš€ Server starting on http://0.0.0.0:8080") print("") - + # Run the Flask app app.run(host='0.0.0.0', port=8080, debug=False) diff --git a/deployment/inference.py b/deployment/inference.py index 430f45042..f648554d3 100644 --- a/deployment/inference.py +++ b/deployment/inference.py @@ -15,44 +15,44 @@ def __init__(self, model_path=None): if model_path is None: # Use the model directory relative to this script model_path = Path(__file__).parent / "model" - + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - + print(f"๐Ÿ”ง Loading model from: {model_path}") - + # Load model and tokenizer self.tokenizer = AutoTokenizer.from_pretrained("roberta-base") self.model = AutoModelForSequenceClassification.from_pretrained(str(model_path)) self.model.to(self.device) self.model.eval() - + # Define emotion mapping based on training order self.emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - + print(f"โœ… Model loaded successfully on {self.device}") - + def predict(self, text): """Predict emotion for given text""" # Tokenize inputs = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(self.device) for k, v in inputs.items()} - + # Predict with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + # Map to emotion name emotion = self.emotion_mapping[predicted_class] - + return { "emotion": emotion, "confidence": confidence, "text": text } - + def predict_batch(self, texts): """Predict emotions for multiple texts""" results = [] @@ -64,20 +64,20 @@ def predict_batch(self, texts): def main(): """Main function for command line usage""" import sys - + if len(sys.argv) < 2: print("Usage: python inference.py 'Your text here'") print("Example: python inference.py 'I am feeling happy today!'") return - + text = sys.argv[1] - + # Initialize detector detector = EmotionDetector() - + # Make prediction result = detector.predict(text) - + print(f"\n๐ŸŽฏ EMOTION DETECTION RESULT") print(f"=" * 40) print(f"Text: {result['text']}") diff --git a/deployment/local/api_server.py b/deployment/local/api_server.py index 49661af14..5d8e83c16 100644 --- a/deployment/local/api_server.py +++ b/deployment/local/api_server.py @@ -57,12 +57,12 @@ def rate_limit(f): def decorated_function(*args, **kwargs): client_ip = request.remote_addr current_time = time.time() - + with rate_limit_lock: # Clean old requests while rate_limit_data[client_ip] and current_time - rate_limit_data[client_ip][0] > RATE_LIMIT_WINDOW: rate_limit_data[client_ip].popleft() - + # Check rate limit if len(rate_limit_data[client_ip]) >= RATE_LIMIT_MAX_REQUESTS: logger.warning(f"Rate limit exceeded for IP: {client_ip}") @@ -70,10 +70,10 @@ def decorated_function(*args, **kwargs): 'error': 'Rate limit exceeded', 'message': f'Maximum {RATE_LIMIT_MAX_REQUESTS} requests per {RATE_LIMIT_WINDOW} seconds' }), 429 - + # Add current request rate_limit_data[client_ip].append(current_time) - + return f(*args, **kwargs) return decorated_function @@ -82,7 +82,7 @@ def update_metrics(response_time, success=True, emotion=None, error_type=None): with metrics_lock: metrics['total_requests'] += 1 metrics['response_times'].append(response_time) - + if success: metrics['successful_requests'] += 1 if emotion: @@ -91,7 +91,7 @@ def update_metrics(response_time, success=True, emotion=None, error_type=None): metrics['failed_requests'] += 1 if error_type: metrics['error_counts'][error_type] += 1 - + # Update average response time if metrics['response_times']: metrics['average_response_time'] = sum(metrics['response_times']) / len(metrics['response_times']) @@ -101,46 +101,46 @@ def __init__(self): """Initialize the model.""" self.model_path = os.path.join(os.getcwd(), "model") logger.info(f"Loading model from: {self.model_path}") - + try: self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + # Move to GPU if available if torch.cuda.is_available(): self.model = self.model.to('cuda') logger.info("โœ… Model moved to GPU") else: logger.info("โš ๏ธ CUDA not available, using CPU") - + self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] logger.info("โœ… Model loaded successfully") - + except Exception as e: logger.error(f"โŒ Failed to load model: {str(e)}") raise - + def predict(self, text): """Make a prediction.""" start_time = time.time() - + try: # Tokenize input inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512) - + if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + # Get prediction with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Get all probabilities all_probs = probabilities[0].cpu().numpy() - + # Get predicted emotion if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] @@ -148,10 +148,10 @@ def predict(self, text): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + prediction_time = time.time() - start_time logger.info(f"Prediction completed in {prediction_time:.3f}s: '{text[:50]}...' โ†’ {predicted_emotion} (conf: {confidence:.3f})") - + # Create response response = { 'text': text, @@ -169,9 +169,9 @@ def predict(self, text): }, 'prediction_time_ms': round(prediction_time * 1000, 2) } - + return response - + except Exception as e: prediction_time = time.time() - start_time logger.error(f"Prediction failed after {prediction_time:.3f}s: {str(e)}") @@ -186,7 +186,7 @@ def predict(self, text): def health_check(): """Health check endpoint.""" start_time = time.time() - + try: response = { 'status': 'healthy', @@ -201,12 +201,12 @@ def health_check(): 'average_response_time_ms': round(metrics['average_response_time'] * 1000, 2) } } - + response_time = time.time() - start_time update_metrics(response_time, success=True) - + return jsonify(response) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='health_check_error') @@ -218,29 +218,29 @@ def health_check(): def predict(): """Prediction endpoint.""" start_time = time.time() - + try: data = request.get_json() - + if not data or 'text' not in data: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='missing_text') return jsonify({'error': 'No text provided'}), 400 - + text = data['text'] if not text.strip(): response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='empty_text') return jsonify({'error': 'Empty text provided'}), 400 - + # Make prediction result = model.predict(text) - + response_time = time.time() - start_time update_metrics(response_time, success=True, emotion=result['predicted_emotion']) - + return jsonify(result) - + except werkzeug.exceptions.BadRequest: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='invalid_json') @@ -257,36 +257,36 @@ def predict(): def predict_batch(): """Batch prediction endpoint.""" start_time = time.time() - + try: data = request.get_json() - + if not data or 'texts' not in data: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='missing_texts') return jsonify({'error': 'No texts provided'}), 400 - + texts = data['texts'] if not isinstance(texts, list): response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='invalid_texts_format') return jsonify({'error': 'Texts must be a list'}), 400 - + results = [] for text in texts: if text.strip(): result = model.predict(text) results.append(result) - + response_time = time.time() - start_time update_metrics(response_time, success=True) - + return jsonify({ 'predictions': results, 'count': len(results), 'batch_processing_time_ms': round(response_time * 1000, 2) }) - + except werkzeug.exceptions.BadRequest: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='invalid_json') @@ -325,7 +325,7 @@ def get_metrics(): def home(): """Home endpoint with API documentation.""" start_time = time.time() - + try: response = { 'message': 'Comprehensive Emotion Detection API', @@ -362,12 +362,12 @@ def home(): } } } - + response_time = time.time() - start_time update_metrics(response_time, success=True) - + return jsonify(response) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='documentation_error') @@ -399,5 +399,5 @@ def handle_bad_request(e): logger.info(f"๐Ÿ”’ Rate limiting: {RATE_LIMIT_MAX_REQUESTS} requests per {RATE_LIMIT_WINDOW} seconds") logger.info("๐Ÿ“Š Monitoring: Comprehensive metrics and logging enabled") logger.info("") - + app.run(host='0.0.0.0', port=8000, debug=False) diff --git a/deployment/secure_api_server.py b/deployment/secure_api_server.py index 361755c32..9f0548eb0 100644 --- a/deployment/secure_api_server.py +++ b/deployment/secure_api_server.py @@ -116,7 +116,7 @@ def update_metrics(response_time, success=True, emotion=None, error_type=None, r with metrics_lock: metrics['total_requests'] += 1 metrics['response_times'].append(response_time) - + if rate_limited: metrics['rate_limited_requests'] += 1 elif success: @@ -127,10 +127,10 @@ def update_metrics(response_time, success=True, emotion=None, error_type=None, r metrics['failed_requests'] += 1 if error_type: metrics['error_counts'][error_type] += 1 - + if sanitization_warnings > 0: metrics['sanitization_warnings'] += sanitization_warnings - + # Update average response time if metrics['response_times']: metrics['average_response_time'] = sum(metrics['response_times']) / len(metrics['response_times']) @@ -142,7 +142,7 @@ def decorated_function(*args, **kwargs): start_time = time.time() client_ip = request.remote_addr user_agent = request.headers.get('User-Agent', '') - + try: # Rate limiting allowed, reason, rate_limit_meta = rate_limiter.allow_request(client_ip, user_agent) @@ -155,7 +155,7 @@ def decorated_function(*args, **kwargs): 'message': reason, 'retry_after': rate_limit_config.window_size_seconds }), 429 - + # Content type validation if request.method == 'POST': content_type = request.headers.get('Content-Type', '') @@ -167,24 +167,24 @@ def decorated_function(*args, **kwargs): 'error': 'Invalid content type', 'message': 'Content-Type must be application/json' }), 400 - + # Process request result = f(*args, **kwargs) - + # Release rate limit slot rate_limiter.release_request(client_ip, user_agent) - + return result - + except Exception as e: # Release rate limit slot on error rate_limiter.release_request(client_ip, user_agent) - + response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='endpoint_error') logger.error(f"Endpoint error: {str(e)}") return jsonify({'error': str(e)}), 500 - + return decorated_function class SecureEmotionDetectionModel: @@ -192,48 +192,48 @@ def __init__(self): """Initialize the secure emotion detection model.""" self.model_path = os.path.join(os.path.dirname(__file__), '..', 'model') logger.info(f"Loading secure model from: {self.model_path}") - + try: self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + # Move to GPU if available if torch.cuda.is_available(): self.model = self.model.to('cuda') logger.info("โœ… Model moved to GPU") else: logger.info("โš ๏ธ CUDA not available, using CPU") - + self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] logger.info("โœ… Secure model loaded successfully") - + except Exception as e: logger.error(f"โŒ Failed to load secure model: {str(e)}") raise - + def predict(self, text, confidence_threshold=None): """Make a secure prediction.""" start_time = time.time() - + try: # Sanitize input text sanitized_text, warnings = input_sanitizer.sanitize_text(text, "emotion") if warnings: logger.warning(f"Sanitization warnings: {warnings}") - + # Tokenize input inputs = self.tokenizer(sanitized_text, return_tensors='pt', truncation=True, padding=True, max_length=512) - + if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + # Get prediction with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Apply confidence threshold if specified if confidence_threshold and confidence < confidence_threshold: predicted_emotion = "uncertain" @@ -246,13 +246,13 @@ def predict(self, text, confidence_threshold=None): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + # Get all probabilities all_probs = probabilities[0].cpu().numpy() - + prediction_time = time.time() - start_time logger.info(f"Secure prediction completed in {prediction_time:.3f}s: '{sanitized_text[:50]}...' โ†’ {predicted_emotion} (conf: {confidence:.3f})") - + # Create secure response response = { 'text': sanitized_text, @@ -275,9 +275,9 @@ def predict(self, text, confidence_threshold=None): 'correlation_id': getattr(g, 'correlation_id', None) } } - + return response - + except Exception as e: prediction_time = time.time() - start_time logger.error(f"Secure prediction failed after {prediction_time:.3f}s: {str(e)}") @@ -306,7 +306,7 @@ def decorated_function(*args, **kwargs): def health_check(): """Secure health check endpoint.""" start_time = time.time() - + try: response = { 'status': 'healthy', @@ -328,12 +328,12 @@ def health_check(): 'average_response_time_ms': round(metrics['average_response_time'] * 1000, 2) } } - + response_time = time.time() - start_time update_metrics(response_time, success=True) - + return jsonify(response) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='health_check_error') @@ -345,7 +345,7 @@ def health_check(): def predict(): """Secure prediction endpoint.""" start_time = time.time() - + try: # Parse and validate request data try: @@ -355,12 +355,12 @@ def predict(): update_metrics(response_time, success=False, error_type='invalid_json') logger.error(f"Invalid JSON in request from {request.remote_addr}") return jsonify({'error': 'Invalid JSON format'}), 400 - + if not data: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='missing_data') return jsonify({'error': 'No data provided'}), 400 - + # Sanitize and validate request try: sanitized_data, warnings = input_sanitizer.validate_emotion_request(data) @@ -369,23 +369,23 @@ def predict(): update_metrics(response_time, success=False, error_type='validation_error') logger.warning(f"Validation error: {str(e)} from {request.remote_addr}") return jsonify({'error': str(e)}), 400 - + # Detect anomalies anomalies = input_sanitizer.detect_anomalies(data) if anomalies: logger.warning(f"Security anomalies detected: {anomalies}") metrics['security_violations'] += 1 - + # Make secure prediction result = secure_model.predict( sanitized_data['text'], confidence_threshold=sanitized_data.get('confidence_threshold') ) - + # Add sanitization warnings to response if warnings: result['security']['sanitization_warnings'] = warnings - + response_time = time.time() - start_time update_metrics( response_time, @@ -393,9 +393,9 @@ def predict(): emotion=result['predicted_emotion'], sanitization_warnings=len(warnings) ) - + return jsonify(result) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='prediction_error') @@ -407,7 +407,7 @@ def predict(): def predict_batch(): """Secure batch prediction endpoint.""" start_time = time.time() - + try: # Parse and validate request data try: @@ -417,12 +417,12 @@ def predict_batch(): update_metrics(response_time, success=False, error_type='invalid_json') logger.error(f"Invalid JSON in batch request from {request.remote_addr}") return jsonify({'error': 'Invalid JSON format'}), 400 - + if not data: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='missing_data') return jsonify({'error': 'No data provided'}), 400 - + # Sanitize and validate request try: sanitized_data, warnings = input_sanitizer.validate_batch_request(data) @@ -431,13 +431,13 @@ def predict_batch(): update_metrics(response_time, success=False, error_type='validation_error') logger.warning(f"Batch validation error: {str(e)} from {request.remote_addr}") return jsonify({'error': str(e)}), 400 - + # Detect anomalies anomalies = input_sanitizer.detect_anomalies(data) if anomalies: logger.warning(f"Security anomalies detected in batch: {anomalies}") metrics['security_violations'] += 1 - + # Make secure batch predictions results = [] for text in sanitized_data['texts']: @@ -447,14 +447,14 @@ def predict_batch(): confidence_threshold=sanitized_data.get('confidence_threshold') ) results.append(result) - + response_time = time.time() - start_time update_metrics( response_time, success=True, sanitization_warnings=len(warnings) ) - + return jsonify({ 'predictions': results, 'count': len(results), @@ -465,7 +465,7 @@ def predict_batch(): 'correlation_id': getattr(g, 'correlation_id', None) } }) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='batch_prediction_error') @@ -506,7 +506,7 @@ def add_to_blacklist(): data = request.get_json() if not data or 'ip' not in data: return jsonify({'error': 'IP address required'}), 400 - + ip = data['ip'] rate_limiter.add_to_blacklist(ip) logger.info(f"Added {ip} to blacklist") @@ -523,7 +523,7 @@ def add_to_whitelist(): data = request.get_json() if not data or 'ip' not in data: return jsonify({'error': 'IP address required'}), 400 - + ip = data['ip'] rate_limiter.add_to_whitelist(ip) logger.info(f"Added {ip} to whitelist") @@ -537,7 +537,7 @@ def add_to_whitelist(): def home(): """Secure home endpoint with API documentation.""" start_time = time.time() - + try: response = { 'message': 'Secure Emotion Detection API', @@ -580,12 +580,12 @@ def home(): } } } - + response_time = time.time() - start_time update_metrics(response_time, success=True) - + return jsonify(response) - + except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='documentation_error') @@ -641,5 +641,5 @@ def handle_internal_error(e): logger.info(f"๐Ÿ”’ Rate limiting: {rate_limit_config.requests_per_minute} requests per minute") logger.info("๐Ÿ›ก๏ธ Security monitoring: Comprehensive logging and metrics enabled") logger.info("=" * 60) - + app.run(host='0.0.0.0', port=8000, debug=False) \ No newline at end of file diff --git a/scripts/ci/model_calibration_test.py b/scripts/ci/model_calibration_test.py index f1d0fd5fe..7bc21e95f 100644 --- a/scripts/ci/model_calibration_test.py +++ b/scripts/ci/model_calibration_test.py @@ -88,10 +88,10 @@ def create_test_data(): # Create tokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") - + # Basic validation assert len(test_texts) == len(emotions), "Texts and emotions must have same length" - + return test_texts, emotions, emotion_to_idx, tokenizer @@ -102,7 +102,7 @@ def test_model_calibration(): # Create test data test_texts, emotions, emotion_to_idx, tokenizer = create_test_data() - + # Create model model = SimpleBERTClassifier("bert-base-uncased", num_emotions=28) model.eval() @@ -119,11 +119,11 @@ def test_model_calibration(): truncation=True, max_length=512 ) - + # Get predictions (only pass required arguments) outputs = model(inputs["input_ids"], inputs["attention_mask"]) probabilities = torch.sigmoid(outputs) - + logger.info(f"โœ… Model inference successful, output shape: {outputs.shape}") # Test temperature setting @@ -141,7 +141,7 @@ def test_model_calibration(): labels = torch.zeros(1, 28) # Match the single prediction shape if emotions[0] in emotion_to_idx: labels[0, emotion_to_idx[emotions[0]]] = 1.0 - + # Calculate F1 score f1 = f1_score(labels.flatten(), predictions.flatten(), average='micro') logger.info(f"โœ… Metrics calculation successful, F1: {f1:.3f}") diff --git a/scripts/ci/run_full_ci_pipeline.py b/scripts/ci/run_full_ci_pipeline.py index 3eba0fe67..8f0af80d4 100644 --- a/scripts/ci/run_full_ci_pipeline.py +++ b/scripts/ci/run_full_ci_pipeline.py @@ -34,7 +34,7 @@ class CIPipelineRunner: """Comprehensive CI Pipeline Runner.""" - + def __init__(self): self.results = {} self.start_time = time.time() @@ -46,11 +46,11 @@ def __init__(self): "scripts/ci/model_calibration_test.py", "scripts/ci/onnx_conversion_test.py", ] - + def detect_environment(self) -> Dict[str, str]: """Detect the current environment (local vs Colab).""" logger.info("๐Ÿ” Detecting environment...") - + env_info = { "platform": sys.platform, "python_version": sys.version, @@ -58,7 +58,7 @@ def detect_environment(self) -> Dict[str, str]: "gpu_available": False, "conda_env": os.environ.get("CONDA_DEFAULT_ENV", "unknown"), } - + # Check for GPU try: import torch @@ -68,26 +68,26 @@ def detect_environment(self) -> Dict[str, str]: env_info["gpu_name"] = torch.cuda.get_device_name(0) except ImportError: logger.warning("โš ๏ธ PyTorch not available for GPU detection") - + # Check for Colab if env_info["is_colab"]: logger.info("๐ŸŽฏ Running in Google Colab environment") env_info["colab_gpu"] = os.environ.get("COLAB_GPU", "unknown") else: logger.info("๐Ÿ’ป Running in local environment") - + logger.info(f"๐Ÿ“Š Environment: {env_info}") return env_info - + def validate_dependencies(self) -> bool: """Validate that all required dependencies are available.""" logger.info("๐Ÿ“ฆ Validating dependencies...") - + required_packages = [ "torch", "transformers", "fastapi", "pydantic", "datasets", "tokenizers", "numpy", "pandas" ] - + missing_packages = [] for package in required_packages: try: @@ -96,22 +96,22 @@ def validate_dependencies(self) -> bool: except ImportError: missing_packages.append(package) logger.error(f"โŒ {package} missing") - + if missing_packages: logger.error(f"โŒ Missing packages: {missing_packages}") return False - + logger.info("โœ… All dependencies validated") return True - + def run_ci_script(self, script_path: str) -> Tuple[bool, str]: """Run a single CI script and return success status and output.""" logger.info(f"๐Ÿš€ Running {script_path}...") - + try: # Use the correct Python interpreter python_executable = sys.executable - + # Run the script result = subprocess.run( [python_executable, script_path], @@ -119,7 +119,7 @@ def run_ci_script(self, script_path: str) -> Tuple[bool, str]: text=True, timeout=300 # 5 minute timeout ) - + if result.returncode == 0: logger.info(f"โœ… {script_path} PASSED") return True, result.stdout @@ -127,18 +127,18 @@ def run_ci_script(self, script_path: str) -> Tuple[bool, str]: logger.error(f"โŒ {script_path} FAILED") logger.error(f"Error output: {result.stderr}") return False, result.stderr - + except subprocess.TimeoutExpired: logger.error(f"โฐ {script_path} TIMEOUT") return False, "Script timed out after 5 minutes" except Exception as e: logger.error(f"๐Ÿ’ฅ {script_path} ERROR: {e}") return False, str(e) - + def run_unit_tests(self) -> bool: """Run unit tests.""" logger.info("๐Ÿงช Running unit tests...") - + try: result = subprocess.run( [sys.executable, "-m", "pytest", "tests/unit/", "-v"], @@ -146,7 +146,7 @@ def run_unit_tests(self) -> bool: text=True, timeout=1200 # 20 minute timeout (increased from 10) ) - + if result.returncode == 0: logger.info("โœ… Unit tests PASSED") return True @@ -156,18 +156,18 @@ def run_unit_tests(self) -> bool: logger.error(f"Error output: {result.stderr}") logger.error(f"Standard output: {result.stdout}") return False - + except subprocess.TimeoutExpired: logger.error("โฐ Unit tests TIMEOUT") return False except Exception as e: logger.error(f"๐Ÿ’ฅ Unit tests ERROR: {e}") return False - + def run_e2e_tests(self) -> bool: """Run end-to-end tests.""" logger.info("๐ŸŽฏ Running E2E tests...") - + try: result = subprocess.run( [sys.executable, "-m", "pytest", "tests/e2e/", "-v"], @@ -175,7 +175,7 @@ def run_e2e_tests(self) -> bool: text=True, timeout=900 # 15 minute timeout ) - + if result.returncode == 0: logger.info("โœ… E2E tests PASSED") return True @@ -183,87 +183,87 @@ def run_e2e_tests(self) -> bool: logger.error("โŒ E2E tests FAILED") logger.error(f"Error output: {result.stderr}") return False - + except Exception as e: logger.error(f"๐Ÿ’ฅ E2E tests ERROR: {e}") return False - + def test_gpu_compatibility(self) -> bool: """Test GPU compatibility if available.""" logger.info("๐Ÿ–ฅ๏ธ Testing GPU compatibility...") - + try: import torch - + if not torch.cuda.is_available(): logger.info("โ„น๏ธ No GPU available, skipping GPU tests") return True - + logger.info(f"๐ŸŽฎ GPU detected: {torch.cuda.get_device_name(0)}") - + # Test GPU model loading device = torch.device("cuda") - + # Add src to path for imports import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - + # Test BERT on GPU try: from models.emotion_detection.bert_classifier import BERTEmotionClassifier except ImportError: from src.models.emotion_detection.bert_classifier import BERTEmotionClassifier model = BERTEmotionClassifier().to(device) - + # Test forward pass import torch dummy_input = torch.randint(0, 1000, (2, 512)).to(device) with torch.no_grad(): output = model(dummy_input, torch.ones_like(dummy_input)) - + logger.info(f"โœ… GPU forward pass successful, output shape: {output.shape}") return True - + except Exception as e: logger.error(f"โŒ GPU compatibility test failed: {e}") return False - + def run_performance_benchmarks(self) -> bool: """Run performance benchmarks.""" logger.info("โšก Running performance benchmarks...") - + try: # Simple performance test - model loading speed import time import torch - + # Test BERT model loading speed start_time = time.time() - + # Add src to path import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - + try: from models.emotion_detection.bert_classifier import BERTEmotionClassifier except ImportError: from src.models.emotion_detection.bert_classifier import BERTEmotionClassifier - + model = BERTEmotionClassifier() loading_time = time.time() - start_time - + # Test inference speed start_time = time.time() dummy_input = torch.randint(0, 1000, (1, 512)) with torch.no_grad(): output = model(dummy_input, torch.ones_like(dummy_input)) inference_time = time.time() - start_time - + logger.info(f"โœ… Model loading time: {loading_time:.2f}s") logger.info(f"โœ… Inference time: {inference_time:.2f}s") - + # Check if times are reasonable if loading_time < 10.0 and inference_time < 5.0: # Increased threshold for CPU environments logger.info("โœ… Performance benchmarks passed") @@ -271,54 +271,54 @@ def run_performance_benchmarks(self) -> bool: else: logger.error(f"โŒ Performance too slow - loading: {loading_time:.2f}s, inference: {inference_time:.2f}s") return False - + except Exception as e: logger.error(f"โŒ Performance benchmark failed: {e}") return False - + def run_full_pipeline(self) -> Dict[str, bool]: """Run the complete CI pipeline.""" logger.info("๐Ÿš€ Starting Comprehensive CI Pipeline") logger.info("=" * 60) - + # Environment detection env_info = self.detect_environment() self.results["environment"] = env_info - + # Dependency validation self.results["dependencies"] = self.validate_dependencies() - + # Run individual CI scripts for script in self.ci_scripts: script_name = Path(script).stem success, output = self.run_ci_script(script) self.results[script_name] = success - + if not success: logger.error(f"โŒ {script_name} failed, but continuing...") - + # Run unit tests self.results["unit_tests"] = self.run_unit_tests() - + # Run E2E tests self.results["e2e_tests"] = self.run_e2e_tests() - + # Test GPU compatibility self.results["gpu_compatibility"] = self.test_gpu_compatibility() - + # Run performance benchmarks self.results["performance"] = self.run_performance_benchmarks() - + return self.results - + def generate_report(self) -> str: """Generate a comprehensive CI report.""" logger.info("๐Ÿ“Š Generating CI Report") logger.info("=" * 60) - + total_tests = len(self.results) passed_tests = sum(1 for result in self.results.values() if isinstance(result, bool) and result) - + report = f""" ๐ŸŽฏ COMPREHENSIVE CI PIPELINE REPORT {'=' * 60} @@ -331,20 +331,20 @@ def generate_report(self) -> str: ๐Ÿ” DETAILED RESULTS: """ - + for test_name, result in self.results.items(): if isinstance(result, bool): status = "โœ… PASSED" if result else "โŒ FAILED" report += f"- {test_name}: {status}\n" elif isinstance(result, dict): report += f"- {test_name}: {result}\n" - + report += f""" โฑ๏ธ EXECUTION TIME: {time.time() - self.start_time:.1f}s ๐ŸŽฏ RECOMMENDATIONS: """ - + if passed_tests == total_tests: report += "๐ŸŽ‰ All tests passed! Pipeline is ready for deployment.\n" else: @@ -352,35 +352,35 @@ def generate_report(self) -> str: if isinstance(result, bool) and not result] report += f"โš ๏ธ Failed tests: {', '.join(failed_tests)}\n" report += "๐Ÿ”ง Please fix the failed tests before deployment.\n" - + return report def main(): """Main function to run the CI pipeline.""" runner = CIPipelineRunner() - + try: results = runner.run_full_pipeline() report = runner.generate_report() - + print(report) - + # Write report to file with open("ci_pipeline_report.txt", "w") as f: f.write(report) - + # Exit with appropriate code total_tests = len([r for r in results.values() if isinstance(r, bool)]) passed_tests = sum(1 for r in results.values() if isinstance(r, bool) and r) - + if passed_tests == total_tests: logger.info("๐ŸŽ‰ CI Pipeline completed successfully!") sys.exit(0) else: logger.error("โŒ CI Pipeline failed!") sys.exit(1) - + except KeyboardInterrupt: logger.info("โน๏ธ CI Pipeline interrupted by user") sys.exit(1) diff --git a/scripts/ci/whisper_transcription_test.py b/scripts/ci/whisper_transcription_test.py index 03ea37767..574c373e7 100644 --- a/scripts/ci/whisper_transcription_test.py +++ b/scripts/ci/whisper_transcription_test.py @@ -125,7 +125,7 @@ def test_audio_preprocessor(): try: preprocessor = AudioPreprocessor() - + # Test audio validation is_valid, error_msg = preprocessor.validate_audio_file(test_audio_path) if not is_valid: @@ -184,7 +184,7 @@ def test_minimal_transcription(): # Test transcription result = transcriber.transcribe(test_audio_path) - + if result and result.text: logger.info(f"โœ… Transcription successful: {result.text[:50]}...") return True diff --git a/scripts/deployment/complete_project_deployment.py b/scripts/deployment/complete_project_deployment.py index dcff44f77..ac39f2b9e 100644 --- a/scripts/deployment/complete_project_deployment.py +++ b/scripts/deployment/complete_project_deployment.py @@ -25,7 +25,7 @@ def check_project_status(): """Check the current project status""" print("๐Ÿ“Š CHECKING PROJECT STATUS") print("=" * 40) - + # Check for trained models model_paths = [ "./emotion_model_ensemble_final", @@ -33,18 +33,18 @@ def check_project_status(): "./emotion_model_fixed_bulletproof_final", "./emotion_model" ] - + found_models = [] for path in model_paths: if os.path.exists(path): found_models.append(path) print(f"โœ… Found model: {path}") - + if not found_models: print("โŒ No trained models found!") print("Please train a model first using the Colab notebooks.") return False - + print(f"๐Ÿ“Š Found {len(found_models)} trained model(s)") return True @@ -52,13 +52,13 @@ def save_model_for_deployment(): """Save the trained model for deployment""" print("\n๐Ÿš€ SAVING MODEL FOR DEPLOYMENT") print("=" * 40) - + try: # Run the model saving script result = subprocess.run([ sys.executable, "scripts/save_trained_model_for_deployment.py" ], capture_output=True, text=True) - + if result.returncode == 0: print("โœ… Model saved successfully!") print(result.stdout) @@ -67,7 +67,7 @@ def save_model_for_deployment(): print("โŒ Failed to save model!") print(result.stderr) return False - + except Exception as e: print(f"โŒ Error saving model: {e}") return False @@ -76,17 +76,17 @@ def test_deployment_package(): """Test the deployment package""" print("\n๐Ÿงช TESTING DEPLOYMENT PACKAGE") print("=" * 40) - + if not os.path.exists("deployment/model"): print("โŒ Model not found in deployment directory!") return False - + try: # Test the model result = subprocess.run([ sys.executable, "deployment/test_examples.py" ], capture_output=True, text=True) - + if result.returncode == 0: print("โœ… Deployment package test passed!") print(result.stdout) @@ -95,7 +95,7 @@ def test_deployment_package(): print("โŒ Deployment package test failed!") print(result.stderr) return False - + except Exception as e: print(f"โŒ Error testing deployment: {e}") return False @@ -104,7 +104,7 @@ def create_final_documentation(): """Create final project documentation""" print("\n๐Ÿ“š CREATING FINAL DOCUMENTATION") print("=" * 40) - + # Create project summary summary = { "project_name": "SAMO Emotion Detection", @@ -136,23 +136,23 @@ def create_final_documentation(): "Test API at http://localhost:5000" ] } - + # Save summary with open("deployment/project_summary.json", 'w') as f: json.dump(summary, f, indent=2) - + print("โœ… Final documentation created!") print("๐Ÿ“ Files created:") print(" - deployment/project_summary.json") print(" - docs/PROJECT_COMPLETION_SUMMARY.md") - + return True def create_deployment_instructions(): """Create deployment instructions""" print("\n๐Ÿ“‹ CREATING DEPLOYMENT INSTRUCTIONS") print("=" * 40) - + instructions = """# ๐Ÿš€ EMOTION DETECTION MODEL - DEPLOYMENT INSTRUCTIONS ## ๐ŸŽ‰ PROJECT COMPLETION STATUS @@ -232,10 +232,10 @@ def create_deployment_instructions(): **MISSION ACCOMPLISHED!** ๐Ÿš€ """ - + with open("deployment/DEPLOYMENT_INSTRUCTIONS.md", 'w') as f: f.write(instructions) - + print("โœ… Deployment instructions created!") return True @@ -243,16 +243,16 @@ def run_final_tests(): """Run final comprehensive tests""" print("\n๐Ÿงช RUNNING FINAL TESTS") print("=" * 40) - + tests = [ ("Model Loading", "python3.12 -c \"from deployment.inference import EmotionDetector; d = EmotionDetector(); print('โœ… Model loaded successfully!')\""), ("API Health", "curl -s http://localhost:5000/health | grep -q 'healthy' && echo 'โœ… API health check passed' || echo 'โŒ API health check failed'"), ("Single Prediction", "curl -s -X POST http://localhost:5000/predict -H 'Content-Type: application/json' -d '{\"text\": \"I am happy\"}' | grep -q 'emotion' && echo 'โœ… Single prediction passed' || echo 'โŒ Single prediction failed'"), ] - + passed = 0 total = len(tests) - + for test_name, command in tests: try: result = subprocess.run(command, shell=True, capture_output=True, text=True) @@ -263,33 +263,33 @@ def run_final_tests(): print(f"โŒ {test_name}: FAILED") except Exception as e: print(f"โŒ {test_name}: ERROR - {e}") - + print(f"\n๐Ÿ“Š Test Results: {passed}/{total} tests passed") return passed == total def main(): """Main deployment process""" print_banner() - + # Check project status if not check_project_status(): print("\nโŒ Project not ready for deployment!") return False - + # Save model for deployment if not save_model_for_deployment(): print("\nโŒ Failed to save model!") return False - + # Test deployment package if not test_deployment_package(): print("\nโŒ Deployment package test failed!") return False - + # Create documentation create_final_documentation() create_deployment_instructions() - + # Final success message print("\n๐ŸŽ‰" * 50) print("๐Ÿ† PROJECT DEPLOYMENT COMPLETE!") @@ -297,7 +297,7 @@ def main(): print("๐Ÿ† ACHIEVED: 99.48% F1 Score") print("โœ… STATUS: TARGET CRUSHED!") print("๐ŸŽ‰" * 50) - + print("\n๐Ÿ“ DEPLOYMENT PACKAGE READY:") print(" - deployment/model/ (trained model)") print(" - deployment/inference.py (inference script)") @@ -305,16 +305,16 @@ def main(): print(" - deployment/test_examples.py (test script)") print(" - deployment/deploy.sh (deployment script)") print(" - deployment/DEPLOYMENT_INSTRUCTIONS.md (instructions)") - + print("\n๐Ÿš€ NEXT STEPS:") print(" 1. cd deployment") print(" 2. ./deploy.sh") print(" 3. Test API at: http://localhost:5000") - + print("\n๐ŸŽฏ MODEL PERFORMANCE: 99.48% F1 Score!") print("๐Ÿ† TARGET ACHIEVED: โœ… YES!") print("๐ŸŽ‰ MISSION ACCOMPLISHED!") - + return True if __name__ == "__main__": diff --git a/scripts/deployment/create_model_deployment_package.py b/scripts/deployment/create_model_deployment_package.py index 8014f5b6e..32e5c3b6f 100644 --- a/scripts/deployment/create_model_deployment_package.py +++ b/scripts/deployment/create_model_deployment_package.py @@ -9,7 +9,7 @@ def create_model_deployment_package(): """Create the deployment package content""" - + # Create deployment directory structure deployment_files = { "README.md": """# ๐Ÿš€ EMOTION DETECTION MODEL - DEPLOYMENT PACKAGE @@ -56,7 +56,7 @@ def create_model_deployment_package(): - **Improvement**: 1,813% increase - **Target**: 75-85% F1 (CRUSHED!) """, - + "requirements.txt": """transformers==4.35.0 torch==2.1.0 scikit-learn==1.3.0 @@ -65,7 +65,7 @@ def create_model_deployment_package(): flask==2.3.3 requests==2.31.0 """, - + "inference.py": '''#!/usr/bin/env python3 """ ๐Ÿš€ EMOTION DETECTION INFERENCE SCRIPT @@ -83,23 +83,23 @@ class EmotionDetector: def __init__(self, model_path="./model"): """Initialize the emotion detector""" self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - + # Load model and tokenizer self.tokenizer = AutoTokenizer.from_pretrained(model_path) self.model = AutoModelForSequenceClassification.from_pretrained(model_path) self.model.to(self.device) self.model.eval() - + # Load label encoder with open(f"{model_path}/label_encoder.json", 'r') as f: label_data = json.load(f) self.label_encoder = LabelEncoder() self.label_encoder.classes_ = np.array(label_data['classes']) - + print(f"โœ… Model loaded successfully!") print(f"๐ŸŽฏ Device: {self.device}") print(f"๐Ÿ“Š Emotions: {list(self.label_encoder.classes_)}") - + def predict(self, text, return_confidence=True): """Predict emotion for given text""" # Tokenize input @@ -109,30 +109,30 @@ def predict(self, text, return_confidence=True): padding=True, return_tensors='pt' ).to(self.device) - + # Get predictions with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + # Decode prediction predicted_emotion = self.label_encoder.inverse_transform([predicted_class])[0] - + if return_confidence: return { 'text': text, 'emotion': predicted_emotion, 'confidence': confidence, 'probabilities': { - emotion: prob.item() + emotion: prob.item() for emotion, prob in zip(self.label_encoder.classes_, probabilities[0]) } } else: return predicted_emotion - + def predict_batch(self, texts): """Predict emotions for multiple texts""" results = [] @@ -149,7 +149,7 @@ def main(): except Exception: print("โŒ Failed to load model") return - + # Test examples test_texts = [ "I'm feeling really happy today!", @@ -158,10 +158,10 @@ def main(): "I'm grateful for all the support.", "I'm feeling overwhelmed with tasks." ] - + print("๐Ÿงช Testing Emotion Detection Model") print("=" * 50) - + for text in test_texts: result = detector.predict(text) print(f"Text: {text}") @@ -175,7 +175,7 @@ def main(): if __name__ == "__main__": main() ''', - + "test_examples.py": '''#!/usr/bin/env python3 """ ๐Ÿงช TEST EMOTION DETECTION MODEL @@ -189,7 +189,7 @@ def test_model(): """Test the emotion detection model""" print("๐Ÿงช EMOTION DETECTION MODEL TESTING") print("=" * 50) - + # Initialize detector try: detector = EmotionDetector() @@ -197,7 +197,7 @@ def test_model(): except Exception: print("โŒ Failed to load model") return - + # Test cases test_cases = [ # Happy emotions @@ -205,44 +205,44 @@ def test_model(): "I'm excited about the new opportunities ahead.", "I'm grateful for all the support I've received.", "I'm proud of what I've accomplished so far.", - + # Negative emotions "I'm so frustrated with this project. Nothing is working.", "I feel anxious about the upcoming presentation.", "I'm feeling sad and lonely today.", "I'm feeling overwhelmed with all these tasks.", - + # Neutral emotions "I feel calm and peaceful right now.", "I'm content with how things are going.", "I'm hopeful that things will get better.", "I'm tired and need some rest." ] - + print("\\n๐Ÿ“Š Testing Results:") print("=" * 50) - + correct_predictions = 0 total_predictions = len(test_cases) - + for i, text in enumerate(test_cases, 1): result = detector.predict(text) - + print(f"{i:2d}. Text: {text}") print(f" Predicted: {result['emotion']} (confidence: {result['confidence']:.3f})") - + # Show top 3 predictions sorted_probs = sorted(result['probabilities'].items(), key=lambda x: x[1], reverse=True) print(f" Top 3: {', '.join([f'{emotion}({prob:.3f})' for emotion, prob in sorted_probs[:3]])}") print() - + print("๐ŸŽ‰ Testing completed!") print(f"๐Ÿ“Š Model confidence range: {min([detector.predict(text)['confidence'] for text in test_cases]):.3f} - {max([detector.predict(text)['confidence'] for text in test_cases]):.3f}") if __name__ == "__main__": test_model() ''', - + "api_server.py": '''#!/usr/bin/env python3 """ ๐Ÿš€ EMOTION DETECTION API SERVER @@ -282,17 +282,17 @@ def predict_emotion(): """Predict emotion for given text""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + try: data = request.get_json() text = data.get('text', '') - + if not text: return jsonify({'error': 'No text provided'}), 400 - + result = detector.predict(text) return jsonify(result) - + except Exception: import uuid request_id = str(uuid.uuid4()) @@ -307,17 +307,17 @@ def predict_batch(): """Predict emotions for multiple texts""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + try: data = request.get_json() texts = data.get('texts', []) - + if not texts: return jsonify({'error': 'No texts provided'}), 400 - + results = detector.predict_batch(texts) return jsonify({'results': results}) - + except Exception: import uuid request_id = str(uuid.uuid4()) @@ -332,7 +332,7 @@ def get_emotions(): """Get list of supported emotions""" if detector is None: return jsonify({'error': 'Model not loaded'}), 500 - + return jsonify({ 'emotions': list(detector.label_encoder.classes_), 'count': len(detector.label_encoder.classes_) @@ -349,10 +349,10 @@ def get_emotions(): print(" - POST /predict_batch - Batch prediction") print(" - GET /emotions - List emotions") print("=" * 50) - + app.run(host='0.0.0.0', port=5000, debug=False) ''', - + "deploy.sh": """#!/bin/bash # ๐Ÿš€ DEPLOYMENT SCRIPT # ==================== @@ -380,7 +380,7 @@ def get_emotions(): echo "Server will be available at: http://localhost:5000" python api_server.py """, - + "dockerfile": """# ๐Ÿš€ EMOTION DETECTION MODEL DOCKERFILE # ===================================== @@ -409,7 +409,7 @@ def get_emotions(): # Run the application CMD ["python", "api_server.py"] """, - + "docker-compose.yml": """version: '3.8' services: @@ -430,20 +430,20 @@ def get_emotions(): start_period: 40s """ } - + # Create deployment directory deployment_dir = "deployment" os.makedirs(deployment_dir, exist_ok=True) - + # Write all files for filename, content in deployment_files.items(): filepath = os.path.join(deployment_dir, filename) with open(filepath, 'w') as f: f.write(content) - + # Make shell script executable os.chmod(os.path.join(deployment_dir, "deploy.sh"), 0o755) - + print("โœ… Deployment package created: deployment/") print("๐Ÿ“ฆ Files included:") for filename in deployment_files.keys(): diff --git a/scripts/deployment/deploy_locally.py b/scripts/deployment/deploy_locally.py index 6545d4dc6..ed509ee57 100644 --- a/scripts/deployment/deploy_locally.py +++ b/scripts/deployment/deploy_locally.py @@ -18,27 +18,27 @@ def deploy_locally(): print("=" * 50) print(f"โฐ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print() - + # Check if model exists model_path = "deployment/models/default" if not os.path.exists(model_path): print(f"โŒ Model not found at: {model_path}") return False - + print("โœ… Model found") - + # Create local deployment directory local_deployment_dir = "local_deployment" if os.path.exists(local_deployment_dir): import shutil shutil.rmtree(local_deployment_dir) os.makedirs(local_deployment_dir) - + # Copy model files import shutil shutil.copytree(model_path, os.path.join(local_deployment_dir, "model")) print("โœ… Model files copied") - + # Create local API server api_server_script = '''#!/usr/bin/env python3 """ @@ -61,38 +61,38 @@ def __init__(self): """Initialize the model.""" self.model_path = os.path.join(os.getcwd(), "model") print(f"Loading model from: {self.model_path}") - + self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + # Move to GPU if available if torch.cuda.is_available(): self.model = self.model.to('cuda') print("โœ… Model moved to GPU") else: print("โš ๏ธ CUDA not available, using CPU") - + self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] print("โœ… Model loaded successfully") - + def predict(self, text): """Make a prediction.""" # Tokenize input inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512) - + if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + # Get prediction with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Get all probabilities all_probs = probabilities[0].cpu().numpy() - + # Get predicted emotion if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] @@ -100,7 +100,7 @@ def predict(self, text): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + # Create response response = { 'text': text, @@ -117,7 +117,7 @@ def predict(self, text): 'average_confidence': '83.9%' } } - + return response # Initialize model @@ -139,19 +139,19 @@ def predict(): """Prediction endpoint.""" try: data = request.get_json() - + if not data or 'text' not in data: return jsonify({'error': 'No text provided'}), 400 - + text = data['text'] if not text.strip(): return jsonify({'error': 'Empty text provided'}), 400 - + # Make prediction result = model.predict(text) - + return jsonify(result) - + except Exception as e: return jsonify({'error': str(e)}), 500 @@ -160,25 +160,25 @@ def predict_batch(): """Batch prediction endpoint.""" try: data = request.get_json() - + if not data or 'texts' not in data: return jsonify({'error': 'No texts provided'}), 400 - + texts = data['texts'] if not isinstance(texts, list): return jsonify({'error': 'Texts must be a list'}), 400 - + results = [] for text in texts: if text.strip(): result = model.predict(text) results.append(result) - + return jsonify({ 'predictions': results, 'count': len(results) }) - + except Exception as e: return jsonify({'error': str(e)}), 500 @@ -228,25 +228,25 @@ def home(): print(" -H 'Content-Type: application/json' \\") print(" -d '{\\"text\\": \\"I am feeling happy today!\\"}'") print() - + app.run(host='0.0.0.0', port=5000, debug=False) ''' - + with open(os.path.join(local_deployment_dir, "api_server.py"), 'w') as f: f.write(api_server_script) print("โœ… API server script created") - + # Create requirements.txt requirements = '''flask>=2.0.0 torch>=2.0.0 transformers>=4.30.0 numpy>=1.21.0 ''' - + with open(os.path.join(local_deployment_dir, "requirements.txt"), 'w') as f: f.write(requirements) print("โœ… Requirements file created") - + # Create test script test_script = '''#!/usr/bin/env python3 """ @@ -263,10 +263,10 @@ def home(): def test_api(): """Test the local API server.""" base_url = "http://localhost:5000" - + print("๐Ÿงช TESTING LOCAL API SERVER") print("=" * 50) - + # Test health check print("1. Testing health check...") try: @@ -280,7 +280,7 @@ def test_api(): except Exception as e: print(f"โŒ Health check error: {e}") return False - + # Test single prediction print("\\n2. Testing single prediction...") test_cases = [ @@ -290,7 +290,7 @@ def test_api(): "I feel anxious about the test", "I am calm and relaxed" ] - + for i, text in enumerate(test_cases, 1): try: response = requests.post( @@ -298,16 +298,16 @@ def test_api(): json={"text": text}, headers={"Content-Type": "application/json"} ) - + if response.status_code == 200: result = response.json() print(f"โœ… Test {i}: '{text}' โ†’ {result['predicted_emotion']} (conf: {result['confidence']:.3f})") else: print(f"โŒ Test {i} failed: {response.status_code}") - + except Exception as e: print(f"โŒ Test {i} error: {e}") - + # Test batch prediction print("\\n3. Testing batch prediction...") try: @@ -316,7 +316,7 @@ def test_api(): json={"texts": test_cases}, headers={"Content-Type": "application/json"} ) - + if response.status_code == 200: result = response.json() print(f"โœ… Batch prediction successful: {result['count']} predictions") @@ -324,10 +324,10 @@ def test_api(): print(f" {i+1}. '{pred['text']}' โ†’ {pred['predicted_emotion']} (conf: {pred['confidence']:.3f})") else: print(f"โŒ Batch prediction failed: {response.status_code}") - + except Exception as e: print(f"โŒ Batch prediction error: {e}") - + print("\\n๐ŸŽ‰ API testing completed!") return True @@ -335,14 +335,14 @@ def test_api(): # Wait a bit for server to start print("โณ Waiting for server to start...") time.sleep(3) - + test_api() ''' - + with open(os.path.join(local_deployment_dir, "test_api.py"), 'w') as f: f.write(test_script) print("โœ… Test script created") - + # Create start script start_script = '''#!/bin/bash # Start local deployment @@ -362,12 +362,12 @@ def test_api(): python api_server.py ''' - + with open(os.path.join(local_deployment_dir, "start.sh"), 'w') as f: f.write(start_script) os.chmod(os.path.join(local_deployment_dir, "start.sh"), 0o755) print("โœ… Start script created") - + # Create deployment summary deployment_summary = { 'status': 'ready', @@ -386,11 +386,11 @@ def test_api(): 'manual_test': 'curl -X POST http://localhost:5000/predict -H "Content-Type: application/json" -d \'{"text": "I am happy"}\'' } } - + with open(os.path.join(local_deployment_dir, "deployment_info.json"), 'w') as f: json.dump(deployment_summary, f, indent=2) print("โœ… Deployment info created") - + print(f"\nโœ… LOCAL DEPLOYMENT READY!") print("=" * 50) print(f"๐Ÿ“ Deployment directory: {local_deployment_dir}") @@ -413,7 +413,7 @@ def test_api(): print(' curl -X POST http://localhost:5000/predict \\') print(' -H "Content-Type: application/json" \\') print(' -d \'{"text": "I am feeling happy today!"}\'') - + return True if __name__ == "__main__": diff --git a/scripts/deployment/deploy_to_gcp_vertex_ai.py b/scripts/deployment/deploy_to_gcp_vertex_ai.py index 34798f4d1..89570011d 100644 --- a/scripts/deployment/deploy_to_gcp_vertex_ai.py +++ b/scripts/deployment/deploy_to_gcp_vertex_ai.py @@ -17,7 +17,7 @@ def check_prerequisites(): """Check if all prerequisites are met for GCP deployment.""" print("๐Ÿ” CHECKING DEPLOYMENT PREREQUISITES") print("=" * 50) - + # Check if gcloud is installed try: result = subprocess.run(['gcloud', '--version'], capture_output=True, text=True) @@ -30,7 +30,7 @@ def check_prerequisites(): print("โŒ gcloud CLI is not installed") print(" Install from: https://cloud.google.com/sdk/docs/install") return False - + # Check if user is authenticated try: result = subprocess.run(['gcloud', 'auth', 'list', '--filter=status:ACTIVE'], capture_output=True, text=True) @@ -43,7 +43,7 @@ def check_prerequisites(): except Exception as e: print(f"โŒ Error checking authentication: {e}") return False - + # Check if project is set try: result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True) @@ -57,7 +57,7 @@ def check_prerequisites(): except Exception as e: print(f"โŒ Error checking project: {e}") return False - + # Check if Vertex AI API is enabled try: result = subprocess.run(['gcloud', 'services', 'list', '--enabled', '--filter=name:aiplatform.googleapis.com'], capture_output=True, text=True) @@ -70,7 +70,7 @@ def check_prerequisites(): except Exception as e: print(f"โŒ Error checking Vertex AI API: {e}") return False - + print("โœ… All prerequisites are met!") return True @@ -78,27 +78,27 @@ def prepare_model_for_deployment(): """Prepare the model for deployment.""" print("\n๐Ÿ“ฆ PREPARING MODEL FOR DEPLOYMENT") print("=" * 50) - + # Check if default model exists default_model_path = "deployment/models/default" if not os.path.exists(default_model_path): print(f"โŒ Default model not found at: {default_model_path}") return False - + # Check model files required_files = ['config.json', 'model.safetensors', 'tokenizer.json', 'vocab.json'] missing_files = [] - + for file in required_files: if not os.path.exists(os.path.join(default_model_path, file)): missing_files.append(file) - + if missing_files: print(f"โŒ Missing model files: {missing_files}") return False - + print("โœ… Model files are complete") - + # Read model metadata metadata_path = os.path.join(default_model_path, "model_metadata.json") if os.path.exists(metadata_path): @@ -108,29 +108,29 @@ def prepare_model_for_deployment(): print(f" Performance: {metadata.get('performance', {}).get('test_accuracy', 'Unknown')}") else: print("โš ๏ธ No model metadata found") - + return True def create_deployment_package(): """Create a deployment package for Vertex AI.""" print("\n๐Ÿ“ฆ CREATING DEPLOYMENT PACKAGE") print("=" * 50) - + # Create deployment directory deployment_dir = "gcp_deployment" if os.path.exists(deployment_dir): import shutil shutil.rmtree(deployment_dir) os.makedirs(deployment_dir) - + # Copy model files model_source = "deployment/models/default" model_dest = os.path.join(deployment_dir, "model") - + import shutil shutil.copytree(model_source, model_dest) print(f"โœ… Model copied to: {model_dest}") - + # Create prediction script prediction_script = '''#!/usr/bin/env python3 """ @@ -152,31 +152,31 @@ def __init__(self): self.model_path = os.path.join(os.getcwd(), "model") self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + # Move to GPU if available if torch.cuda.is_available(): self.model = self.model.to('cuda') - + self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - + def predict(self, text): """Make a prediction.""" # Tokenize input inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=512) - + if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + # Get prediction with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Get all probabilities all_probs = probabilities[0].cpu().numpy() - + # Get predicted emotion if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] @@ -184,7 +184,7 @@ def predict(self, text): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + # Create response response = { 'text': text, @@ -196,7 +196,7 @@ def predict(self, text): 'model_version': '2.0', 'model_type': 'comprehensive_emotion_detection' } - + return response # Initialize model @@ -210,35 +210,35 @@ def predict(request): request_json = json.loads(request) else: request_json = request - + # Get text from request text = request_json.get('text', '') if not text: return json.dumps({'error': 'No text provided'}) - + # Make prediction result = model.predict(text) - + return json.dumps(result) - + except Exception as e: return json.dumps({'error': str(e)}) ''' - + with open(os.path.join(deployment_dir, "predict.py"), 'w') as f: f.write(prediction_script) print("โœ… Prediction script created") - + # Create requirements.txt requirements = '''torch>=2.0.0 transformers>=4.30.0 numpy>=1.21.0 ''' - + with open(os.path.join(deployment_dir, "requirements.txt"), 'w') as f: f.write(requirements) print("โœ… Requirements file created") - + # Create Dockerfile dockerfile = '''FROM python:3.9-slim @@ -268,11 +268,11 @@ def predict(request): # Run the prediction service CMD ["python", "predict.py"] ''' - + with open(os.path.join(deployment_dir, "Dockerfile"), 'w') as f: f.write(dockerfile) print("โœ… Dockerfile created") - + # Create deployment configuration deployment_config = { 'model_info': { @@ -291,11 +291,11 @@ def predict(request): 'deployment_package': deployment_dir } } - + with open(os.path.join(deployment_dir, "deployment_config.json"), 'w') as f: json.dump(deployment_config, f, indent=2) print("โœ… Deployment configuration created") - + print(f"โœ… Deployment package created at: {deployment_dir}") return deployment_dir @@ -303,55 +303,55 @@ def deploy_to_vertex_ai(deployment_dir): """Deploy the model to Vertex AI.""" print("\n๐Ÿš€ DEPLOYING TO VERTEX AI") print("=" * 50) - + # Get project ID result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True) project_id = result.stdout.strip() - + # Set region region = "us-central1" # You can change this - + # Create model name model_name = "comprehensive-emotion-detection" endpoint_name = "emotion-detection-endpoint" - + print(f"๐Ÿ“‹ Deployment Configuration:") print(f" Project ID: {project_id}") print(f" Region: {region}") print(f" Model Name: {model_name}") print(f" Endpoint Name: {endpoint_name}") print() - + # Build and push Docker image print("๐Ÿณ Building and pushing Docker image...") - + # Create repository name repository_name = "emotion-detection" - + # Configure Docker for gcloud subprocess.run(['gcloud', 'auth', 'configure-docker'], check=True) - + # Build and push image image_uri = f"gcr.io/{project_id}/{repository_name}:latest" - + try: # Build image subprocess.run([ 'docker', 'build', '-t', image_uri, deployment_dir ], check=True) print("โœ… Docker image built") - + # Push image subprocess.run(['docker', 'push', image_uri], check=True) print("โœ… Docker image pushed to Container Registry") - + except subprocess.CalledProcessError as e: print(f"โŒ Error building/pushing Docker image: {e}") return False - + # Create Vertex AI model print("\n๐Ÿค– Creating Vertex AI model...") - + try: # Create model subprocess.run([ @@ -363,14 +363,14 @@ def deploy_to_vertex_ai(deployment_dir): '--container-health-route', '/health' ], check=True) print("โœ… Vertex AI model created") - + except subprocess.CalledProcessError as e: print(f"โŒ Error creating Vertex AI model: {e}") return False - + # Create endpoint print("\n๐ŸŒ Creating endpoint...") - + try: subprocess.run([ 'gcloud', 'ai', 'endpoints', 'create', @@ -378,14 +378,14 @@ def deploy_to_vertex_ai(deployment_dir): '--display-name', endpoint_name ], check=True) print("โœ… Endpoint created") - + except subprocess.CalledProcessError as e: print(f"โŒ Error creating endpoint: {e}") return False - + # Deploy model to endpoint print("\n๐Ÿš€ Deploying model to endpoint...") - + try: # Get model ID result = subprocess.run([ @@ -394,9 +394,9 @@ def deploy_to_vertex_ai(deployment_dir): '--filter', f'displayName={model_name}', '--format', 'value(name)' ], capture_output=True, text=True, check=True) - + model_id = result.stdout.strip() - + # Get endpoint ID result = subprocess.run([ 'gcloud', 'ai', 'endpoints', 'list', @@ -404,9 +404,9 @@ def deploy_to_vertex_ai(deployment_dir): '--filter', f'displayName={endpoint_name}', '--format', 'value(name)' ], capture_output=True, text=True, check=True) - + endpoint_id = result.stdout.strip() - + # Deploy model subprocess.run([ 'gcloud', 'ai', 'endpoints', 'deploy-model', endpoint_id, @@ -418,16 +418,16 @@ def deploy_to_vertex_ai(deployment_dir): '--max-replica-count', '10' ], check=True) print("โœ… Model deployed to endpoint") - + except subprocess.CalledProcessError as e: print(f"โŒ Error deploying model: {e}") return False - + print(f"\n๐ŸŽ‰ DEPLOYMENT COMPLETE!") print(f"๐Ÿ“‹ Endpoint ID: {endpoint_id}") print(f"๐ŸŒ Region: {region}") print(f"๐Ÿค– Model: {model_name}") - + # Create deployment summary deployment_summary = { 'status': 'success', @@ -439,12 +439,12 @@ def deploy_to_vertex_ai(deployment_dir): 'image_uri': image_uri, 'deployment_dir': deployment_dir } - + with open(os.path.join(deployment_dir, "deployment_summary.json"), 'w') as f: json.dump(deployment_summary, f, indent=2) - + print(f"\n๐Ÿ“ Deployment summary saved to: {deployment_dir}/deployment_summary.json") - + return True def main(): @@ -453,33 +453,33 @@ def main(): print("=" * 60) print(f"โฐ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print() - + # Check prerequisites if not check_prerequisites(): print("\nโŒ Prerequisites not met. Please fix the issues above.") return False - + # Prepare model if not prepare_model_for_deployment(): print("\nโŒ Model preparation failed.") return False - + # Create deployment package deployment_dir = create_deployment_package() if not deployment_dir: print("\nโŒ Failed to create deployment package.") return False - + # Deploy to Vertex AI if not deploy_to_vertex_ai(deployment_dir): print("\nโŒ Deployment to Vertex AI failed.") return False - + print("\n๐ŸŽ‰ DEPLOYMENT SUCCESSFUL!") print("=" * 60) print("Your comprehensive emotion detection model is now deployed on GCP/Vertex AI!") print("You can now make predictions using the Vertex AI endpoint.") - + return True if __name__ == "__main__": diff --git a/scripts/deployment/save_trained_model_for_deployment.py b/scripts/deployment/save_trained_model_for_deployment.py index 8ef6a37d4..2fe646cba 100644 --- a/scripts/deployment/save_trained_model_for_deployment.py +++ b/scripts/deployment/save_trained_model_for_deployment.py @@ -13,10 +13,10 @@ def save_model_for_deployment(): """Save the trained model for deployment""" - + print("๐Ÿš€ SAVING TRAINED MODEL FOR DEPLOYMENT") print("=" * 50) - + # Define model paths model_paths = [ "./emotion_model_ensemble_final", # Latest ensemble model @@ -24,7 +24,7 @@ def save_model_for_deployment(): "./emotion_model_fixed_bulletproof_final", # Bulletproof model "./emotion_model", # Generic model path ] - + # Find the best model best_model_path = None for path in model_paths: @@ -32,50 +32,50 @@ def save_model_for_deployment(): print(f"โœ… Found model at: {path}") best_model_path = path break - + if not best_model_path: print("โŒ No trained model found!") print("๐Ÿ“‹ Available paths checked:") for path in model_paths: print(f" - {path}: {'โœ… EXISTS' if os.path.exists(path) else 'โŒ NOT FOUND'}") return False - + print(f"๐ŸŽฏ Using model: {best_model_path}") - + # Create deployment model directory deployment_model_dir = "deployment/model" os.makedirs(deployment_model_dir, exist_ok=True) - + try: # Load the model and tokenizer print("๐Ÿ”ง Loading model and tokenizer...") tokenizer = AutoTokenizer.from_pretrained(best_model_path) model = AutoModelForSequenceClassification.from_pretrained(best_model_path) - + # Save model and tokenizer print("๐Ÿ’พ Saving model and tokenizer...") model.save_pretrained(deployment_model_dir) tokenizer.save_pretrained(deployment_model_dir) - + # Create label encoder (12 emotions) print("๐Ÿท๏ธ Creating label encoder...") emotions = [ 'anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired' ] - + label_encoder = LabelEncoder() label_encoder.fit(emotions) - + # Save label encoder label_encoder_data = { 'classes': label_encoder.classes_.tolist(), 'n_classes': len(label_encoder.classes_) } - + with open(f"{deployment_model_dir}/label_encoder.json", 'w') as f: json.dump(label_encoder_data, f, indent=2) - + # Create model info file model_info = { 'model_name': best_model_path, @@ -96,23 +96,23 @@ def save_model_for_deployment(): 'deployment_ready': True, 'created_at': '2025-08-03' } - + with open(f"{deployment_model_dir}/model_info.json", 'w') as f: json.dump(model_info, f, indent=2) - + print("โœ… Model saved successfully!") print(f"๐Ÿ“ Deployment directory: {deployment_model_dir}") print(f"๐Ÿ“Š Model info:") print(f" - Emotions: {len(emotions)} classes") print(f" - F1 Score: 99.48%") print(f" - Target Achieved: โœ… YES!") - + # Test the saved model print("๐Ÿงช Testing saved model...") test_saved_model(deployment_model_dir) - + return True - + except Exception as e: print(f"โŒ Error saving model: {e}") return False @@ -121,10 +121,10 @@ def test_saved_model(model_dir): """Test the saved model""" try: from inference import EmotionDetector - + # Initialize detector with saved model detector = EmotionDetector(model_dir) - + # Test cases test_texts = [ "I'm feeling really happy today!", @@ -133,24 +133,24 @@ def test_saved_model(model_dir): "I'm grateful for all the support.", "I'm feeling overwhelmed with tasks." ] - + print("๐Ÿ“Š Testing saved model:") print("-" * 30) - + for text in test_texts: result = detector.predict(text) print(f"Text: {text}") print(f"Emotion: {result['emotion']} (confidence: {result['confidence']:.3f})") print() - + print("โœ… Saved model test completed!") - + except Exception as e: print(f"โš ๏ธ Could not test saved model: {e}") def create_deployment_script(): """Create a deployment script""" - + deployment_script = """#!/bin/bash # ๐Ÿš€ EMOTION DETECTION MODEL DEPLOYMENT # ===================================== @@ -186,17 +186,17 @@ def create_deployment_script(): echo "Press Ctrl+C to stop the server" python api_server.py """ - + with open("deployment/deploy.sh", 'w') as f: f.write(deployment_script) - + # Make executable os.chmod("deployment/deploy.sh", 0o755) print("โœ… Deployment script updated!") if __name__ == "__main__": success = save_model_for_deployment() - + if success: create_deployment_script() print("\n๐ŸŽ‰ DEPLOYMENT PACKAGE READY!") diff --git a/scripts/legacy/add_comprehensive_features.py b/scripts/legacy/add_comprehensive_features.py index a4fc9c308..8b2161458 100644 --- a/scripts/legacy/add_comprehensive_features.py +++ b/scripts/legacy/add_comprehensive_features.py @@ -11,11 +11,11 @@ def add_comprehensive_features(): """Add all advanced features to the comprehensive notebook.""" - + # Read the existing notebook with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Add all the advanced features as new cells advanced_cells = [ { @@ -535,14 +535,14 @@ def add_comprehensive_features(): ] } ] - + # Add all the advanced cells to the notebook notebook['cells'].extend(advanced_cells) - + # Save the updated notebook with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Added all comprehensive features!') print('๐Ÿ“‹ Advanced features added:') print(' โœ… Model setup with architecture fixes') diff --git a/scripts/legacy/add_wandb_setup.py b/scripts/legacy/add_wandb_setup.py index 35c8bb753..668879792 100644 --- a/scripts/legacy/add_wandb_setup.py +++ b/scripts/legacy/add_wandb_setup.py @@ -11,11 +11,11 @@ def add_wandb_setup(): """Add wandb setup to the minimal notebook.""" - + # Read the existing notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Add wandb setup cell after the imports wandb_setup_cell = { "cell_type": "markdown", @@ -24,7 +24,7 @@ def add_wandb_setup(): "## ๐Ÿ”‘ WANDB API KEY SETUP" ] } - + wandb_setup_code = { "cell_type": "code", "execution_count": None, @@ -96,7 +96,7 @@ def add_wandb_setup(): "print('\\nโœ… WandB setup completed')" ] } - + # Find the imports cell and add wandb setup after it for i, cell in enumerate(notebook['cells']): if cell['cell_type'] == 'code' and 'import torch' in ''.join(cell['source']): @@ -104,7 +104,7 @@ def add_wandb_setup(): notebook['cells'].insert(i + 2, wandb_setup_cell) notebook['cells'].insert(i + 3, wandb_setup_code) break - + # Also update the training arguments to disable wandb if no API key for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'TrainingArguments(' in ''.join(cell['source']): @@ -130,11 +130,11 @@ def add_wandb_setup(): " print('โš ๏ธ WandB logging disabled (no API key)')" ] break - + # Save the updated notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Added WandB setup to minimal notebook!') print('๐Ÿ“‹ Changes made:') print(' โœ… Added WandB API key setup from Colab secrets') diff --git a/scripts/legacy/comprehensive_model_validation.py b/scripts/legacy/comprehensive_model_validation.py index 61aecd9a7..c25d8dead 100644 --- a/scripts/legacy/comprehensive_model_validation.py +++ b/scripts/legacy/comprehensive_model_validation.py @@ -14,19 +14,19 @@ def comprehensive_validation(): """Comprehensive validation of the emotion detection model""" - + print("๐Ÿ”ฌ COMPREHENSIVE MODEL VALIDATION") print("=" * 60) print("๐ŸŽฏ Goal: Verify 99.54% F1 score reliability") print("=" * 60) - + # Check model files model_dir = Path(__file__).parent.parent / 'deployment' / 'model' required_files = ['config.json', 'model.safetensors', 'training_args.bin'] - + print(f"\n๐Ÿ“ MODEL FILE VALIDATION") print("-" * 40) - + missing_files = [] for file in required_files: file_path = model_dir / file @@ -36,58 +36,58 @@ def comprehensive_validation(): else: print(f"โŒ {file}: MISSING") missing_files.append(file) - + if missing_files: print(f"\nโŒ CRITICAL: Missing files: {missing_files}") return False - + print(f"โœ… All model files present and valid") - + # Load model configuration print(f"\n๐Ÿ”ง MODEL CONFIGURATION VALIDATION") print("-" * 40) - + with open(model_dir / 'config.json', 'r') as f: config = json.load(f) - + print(f"Model Type: {config.get('model_type', 'unknown')}") print(f"Architecture: {config.get('architectures', ['unknown'])[0]}") print(f"Hidden Size: {config.get('hidden_size', 'unknown')}") print(f"Number of Labels: {len(config.get('id2label', {}))}") print(f"Vocab Size: {config.get('vocab_size', 'unknown')}") - + # Define emotion mapping emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] print(f"Emotion Classes: {len(emotion_mapping)}") - + # Load model and tokenizer print(f"\n๐Ÿ”ง MODEL LOADING VALIDATION") print("-" * 40) - + try: start_time = time.time() tokenizer = AutoTokenizer.from_pretrained("roberta-base") load_time = time.time() - start_time print(f"โœ… Tokenizer loaded: {load_time:.2f}s") - + start_time = time.time() model = AutoModelForSequenceClassification.from_pretrained(str(model_dir)) load_time = time.time() - start_time print(f"โœ… Model loaded: {load_time:.2f}s") - + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model.to(device) model.eval() print(f"โœ… Model moved to {device}") - + except Exception as e: print(f"โŒ Model loading failed: {str(e)}") return False - + # Test 1: Basic Functionality print(f"\n๐Ÿงช TEST 1: BASIC FUNCTIONALITY") print("-" * 40) - + test_cases = [ ("I'm feeling really happy today!", "happy"), ("I'm so frustrated with this project.", "frustrated"), @@ -102,77 +102,77 @@ def comprehensive_validation(): ("I feel content with my life.", "content"), ("I'm hopeful for the future.", "hopeful") ] - + correct_predictions = 0 total_predictions = len(test_cases) - + for text, expected_emotion in test_cases: try: # Tokenize inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + # Predict with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + predicted_emotion = emotion_mapping[predicted_class] is_correct = predicted_emotion == expected_emotion - + if is_correct: correct_predictions += 1 status = "โœ…" else: status = "โŒ" - + print(f"{status} '{text}' โ†’ {predicted_emotion} (expected: {expected_emotion}, confidence: {confidence:.3f})") - + except Exception as e: print(f"โŒ Error predicting '{text}': {str(e)}") return False - + accuracy = correct_predictions / total_predictions print(f"\n๐Ÿ“Š Basic Functionality Results:") print(f" Correct: {correct_predictions}/{total_predictions}") print(f" Accuracy: {accuracy:.1%}") - + if accuracy < 0.8: print(f"โŒ CRITICAL: Basic accuracy too low ({accuracy:.1%})") return False - + # Test 2: Confidence Distribution print(f"\n๐Ÿงช TEST 2: CONFIDENCE DISTRIBUTION") print("-" * 40) - + confidence_scores = [] for text, _ in test_cases: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) confidence = torch.max(probabilities, dim=1)[0].item() confidence_scores.append(confidence) - + avg_confidence = np.mean(confidence_scores) min_confidence = np.min(confidence_scores) max_confidence = np.max(confidence_scores) - + print(f"Average Confidence: {avg_confidence:.3f}") print(f"Min Confidence: {min_confidence:.3f}") print(f"Max Confidence: {max_confidence:.3f}") - + if avg_confidence < 0.5: print(f"โš ๏ธ WARNING: Low average confidence ({avg_confidence:.3f})") - + # Test 3: Edge Cases print(f"\n๐Ÿงช TEST 3: EDGE CASES") print("-" * 40) - + edge_cases = [ "", # Empty string "a", # Single character @@ -183,89 +183,89 @@ def comprehensive_validation(): "I'M FEELING HAPPY TODAY!", # All caps "i am feeling happy today", # All lowercase ] - + edge_case_success = 0 for text in edge_cases: try: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + predicted_emotion = emotion_mapping[predicted_class] edge_case_success += 1 print(f"โœ… Edge case handled: '{text[:30]}...' โ†’ {predicted_emotion} ({confidence:.3f})") - + except Exception as e: print(f"โŒ Edge case failed: '{text[:30]}...' - {str(e)}") - + print(f"\n๐Ÿ“Š Edge Case Results: {edge_case_success}/{len(edge_cases)} successful") - + # Test 4: Performance Benchmark print(f"\n๐Ÿงช TEST 4: PERFORMANCE BENCHMARK") print("-" * 40) - + benchmark_text = "I'm feeling really happy today!" num_iterations = 100 - + start_time = time.time() for _ in range(num_iterations): inputs = tokenizer(benchmark_text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) - + total_time = time.time() - start_time avg_time = total_time / num_iterations throughput = num_iterations / total_time - + print(f"Total Time: {total_time:.2f}s") print(f"Average Time per Prediction: {avg_time:.4f}s") print(f"Throughput: {throughput:.1f} predictions/second") - + if avg_time > 1.0: print(f"โš ๏ธ WARNING: Slow inference time ({avg_time:.4f}s)") - + # Test 5: Consistency Check print(f"\n๐Ÿงช TEST 5: CONSISTENCY CHECK") print("-" * 40) - + consistency_text = "I'm feeling happy today!" predictions = [] - + for _ in range(10): inputs = tokenizer(consistency_text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + predictions.append((emotion_mapping[predicted_class], confidence)) - + # Check if all predictions are the same unique_predictions = set(pred[0] for pred in predictions) is_consistent = len(unique_predictions) == 1 - + if is_consistent: emotion, avg_conf = unique_predictions.pop(), np.mean([p[1] for p in predictions]) print(f"โœ… Consistent predictions: {emotion} (avg confidence: {avg_conf:.3f})") else: print(f"โŒ Inconsistent predictions: {unique_predictions}") return False - + # Final Validation Summary print(f"\n๐ŸŽฏ FINAL VALIDATION SUMMARY") print("=" * 60) - + validation_results = { "model_files": True, "model_loading": True, @@ -274,21 +274,21 @@ def comprehensive_validation(): "performance": avg_time < 1.0, "consistency": is_consistent } - + all_passed = all(validation_results.values()) - + for test, passed in validation_results.items(): status = "โœ… PASS" if passed else "โŒ FAIL" print(f"{status} {test.replace('_', ' ').title()}") - + print(f"\n{'๐ŸŽ‰ ALL TESTS PASSED!' if all_passed else 'โŒ SOME TESTS FAILED'}") - + if all_passed: print(f"โœ… Your 99.54% F1 score model is 100% RELIABLE!") print(f"๐Ÿš€ Ready for production deployment!") else: print(f"โš ๏ธ Model needs further validation before deployment") - + return all_passed if __name__ == "__main__": diff --git a/scripts/legacy/convert_to_onnx.py b/scripts/legacy/convert_to_onnx.py index 7ac653e8b..d1b3a86f8 100755 --- a/scripts/legacy/convert_to_onnx.py +++ b/scripts/legacy/convert_to_onnx.py @@ -139,7 +139,7 @@ def wrapper_function(input_ids, attention_mask, token_type_ids): def benchmark_pytorch_inference(model, input_ids, attention_mask, num_runs=50): """Benchmark PyTorch model inference time.""" model.eval() - + # Warm up with torch.no_grad(): for _ in range(10): @@ -161,7 +161,7 @@ def benchmark_onnx_inference(model_path, input_ids, attention_mask, token_type_i # Create ONNX session session = ort.InferenceSession(model_path) - + # Prepare inputs input_feed = { "input_ids": input_ids.numpy(), diff --git a/scripts/legacy/create_bulletproof_cell.py b/scripts/legacy/create_bulletproof_cell.py index 4fa79be07..b93f983c9 100644 --- a/scripts/legacy/create_bulletproof_cell.py +++ b/scripts/legacy/create_bulletproof_cell.py @@ -5,7 +5,7 @@ def create_bulletproof_cell(): """Create a bulletproof training cell.""" - + cell_code = '''# ๐Ÿš€ BULLETPROOF TRAINING CELL - RUN IN FRESH KERNEL # Runtime โ†’ Change runtime type โ†’ GPU (T4 or V100) # Kernel โ†’ Restart and run all @@ -138,30 +138,30 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + # Validate data if len(texts) != len(labels): raise ValueError(f"Texts and labels have different lengths: {len(texts)} vs {len(labels)}") - + # Validate labels for i, label in enumerate(labels): if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {i}: {label}") - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] - + # Validate inputs if not isinstance(text, str) or not text.strip(): raise ValueError(f"Invalid text at index {idx}") - + if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {idx}: {label}") - + encoding = self.tokenizer( text, truncation=True, @@ -169,7 +169,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -180,33 +180,33 @@ def __getitem__(self, idx): class SimpleEmotionClassifier(nn.Module): def __init__(self, model_name="bert-base-uncased", num_labels=None): super().__init__() - + if num_labels is None or num_labels <= 0: raise ValueError(f"Invalid num_labels: {num_labels}") - + self.num_labels = num_labels self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(0.3) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + print(f"โœ… Model initialized with {num_labels} labels") - + def forward(self, input_ids, attention_mask): # Validate inputs if input_ids.dim() != 2: raise ValueError(f"Expected input_ids to be 2D, got {input_ids.dim()}D") - + if attention_mask.dim() != 2: raise ValueError(f"Expected attention_mask to be 2D, got {attention_mask.dim()}D") - + outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.pooler_output logits = self.classifier(self.dropout(pooled_output)) - + # Validate outputs if logits.shape[-1] != self.num_labels: raise ValueError(f"Expected {self.num_labels} output classes, got {logits.shape[-1]}") - + return logits # Step 7: Setup training @@ -252,12 +252,12 @@ def forward(self, input_ids, attention_mask): for epoch in range(num_epochs): print(f"\\n๐Ÿ”„ Epoch {epoch + 1}/{num_epochs}") - + # Training model.train() total_loss = 0 num_batches = 0 - + # Train on GoEmotions print(" ๐Ÿ“š Training on GoEmotions...") for i, batch in enumerate(go_loader): @@ -266,34 +266,34 @@ def forward(self, input_ids, attention_mask): if 'input_ids' not in batch or 'attention_mask' not in batch or 'labels' not in batch: print(f"โš ๏ธ Invalid batch structure at batch {i}") continue - + # Move to device with validation input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + # Validate labels if torch.any(labels >= num_labels) or torch.any(labels < 0): print(f"โš ๏ธ Invalid labels in batch {i}: {labels}") continue - + # Forward pass optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 50 == 0: print(f" Batch {i}/{len(go_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in batch {i}: {e}") continue - + # Train on journal data print(" ๐Ÿ“ Training on journal data...") for i, batch in enumerate(journal_train_loader): @@ -301,67 +301,67 @@ def forward(self, input_ids, attention_mask): input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + if torch.any(labels >= num_labels) or torch.any(labels < 0): continue - + optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 10 == 0: print(f" Batch {i}/{len(journal_train_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in journal batch {i}: {e}") continue - + # Validation print(" ๐ŸŽฏ Validating...") model.eval() all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in journal_val_loader: try: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) preds = torch.argmax(outputs, dim=1) - + all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + except Exception as e: print(f"โŒ Error in validation batch: {e}") continue - + # Calculate metrics if all_preds and all_labels: f1_macro = f1_score(all_labels, all_preds, average='macro') accuracy = accuracy_score(all_labels, all_preds) - + avg_loss = total_loss / num_batches if num_batches > 0 else 0 - + print(f" ๐Ÿ“Š Epoch {epoch + 1} Results:") print(f" Average Loss: {avg_loss:.4f}") print(f" Validation F1 (Macro): {f1_macro:.4f}") print(f" Validation Accuracy: {accuracy:.4f}") - + # Save best model if f1_macro > best_f1: best_f1 = f1_macro torch.save(model.state_dict(), 'best_simple_model.pth') print(f" ๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") - + # Clear GPU cache if torch.cuda.is_available(): torch.cuda.empty_cache() @@ -391,11 +391,11 @@ def forward(self, input_ids, attention_mask): print("\\n๐ŸŽ‰ BULLETPROOF TRAINING COMPLETED!") print("๐Ÿ“ Files downloaded: best_simple_model.pth, simple_training_results.json")''' - + # Write to file with open('bulletproof_training_cell.py', 'w') as f: f.write(cell_code) - + print("โœ… Created bulletproof training cell: bulletproof_training_cell.py") print("๐Ÿ“‹ Instructions:") print("1. Copy the code from bulletproof_training_cell.py") diff --git a/scripts/legacy/create_final_bulletproof_cell.py b/scripts/legacy/create_final_bulletproof_cell.py index 499fb7be0..34c508074 100644 --- a/scripts/legacy/create_final_bulletproof_cell.py +++ b/scripts/legacy/create_final_bulletproof_cell.py @@ -5,7 +5,7 @@ def create_final_bulletproof_cell(): """Create the final bulletproof cell with proper label mapping.""" - + cell_code = '''# ๐Ÿš€ FINAL BULLETPROOF TRAINING CELL - PROPER LABEL MAPPING # Runtime โ†’ Change runtime type โ†’ GPU (T4 or V100) # Kernel โ†’ Restart and run all @@ -174,30 +174,30 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + # Validate data if len(texts) != len(labels): raise ValueError(f"Texts and labels have different lengths: {len(texts)} vs {len(labels)}") - + # Validate labels for i, label in enumerate(labels): if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {i}: {label}") - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] - + # Validate inputs if not isinstance(text, str) or not text.strip(): raise ValueError(f"Invalid text at index {idx}") - + if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {idx}: {label}") - + encoding = self.tokenizer( text, truncation=True, @@ -205,7 +205,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -216,33 +216,33 @@ def __getitem__(self, idx): class SimpleEmotionClassifier(nn.Module): def __init__(self, model_name="bert-base-uncased", num_labels=None): super().__init__() - + if num_labels is None or num_labels <= 0: raise ValueError(f"Invalid num_labels: {num_labels}") - + self.num_labels = num_labels self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(0.3) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + print(f"โœ… Model initialized with {num_labels} labels") - + def forward(self, input_ids, attention_mask): # Validate inputs if input_ids.dim() != 2: raise ValueError(f"Expected input_ids to be 2D, got {input_ids.dim()}D") - + if attention_mask.dim() != 2: raise ValueError(f"Expected attention_mask to be 2D, got {attention_mask.dim()}D") - + outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.pooler_output logits = self.classifier(self.dropout(pooled_output)) - + # Validate outputs if logits.shape[-1] != self.num_labels: raise ValueError(f"Expected {self.num_labels} output classes, got {logits.shape[-1]}") - + return logits # Step 9: Setup training @@ -288,12 +288,12 @@ def forward(self, input_ids, attention_mask): for epoch in range(num_epochs): print(f"\\n๐Ÿ”„ Epoch {epoch + 1}/{num_epochs}") - + # Training model.train() total_loss = 0 num_batches = 0 - + # Train on GoEmotions print(" ๐Ÿ“š Training on GoEmotions...") for i, batch in enumerate(go_loader): @@ -302,34 +302,34 @@ def forward(self, input_ids, attention_mask): if 'input_ids' not in batch or 'attention_mask' not in batch or 'labels' not in batch: print(f"โš ๏ธ Invalid batch structure at batch {i}") continue - + # Move to device with validation input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + # Validate labels if torch.any(labels >= num_labels) or torch.any(labels < 0): print(f"โš ๏ธ Invalid labels in batch {i}: {labels}") continue - + # Forward pass optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 50 == 0: print(f" Batch {i}/{len(go_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in batch {i}: {e}") continue - + # Train on journal data print(" ๐Ÿ“ Training on journal data...") for i, batch in enumerate(journal_train_loader): @@ -337,67 +337,67 @@ def forward(self, input_ids, attention_mask): input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + if torch.any(labels >= num_labels) or torch.any(labels < 0): continue - + optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 10 == 0: print(f" Batch {i}/{len(journal_train_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in journal batch {i}: {e}") continue - + # Validation print(" ๐ŸŽฏ Validating...") model.eval() all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in journal_val_loader: try: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) preds = torch.argmax(outputs, dim=1) - + all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + except Exception as e: print(f"โŒ Error in validation batch: {e}") continue - + # Calculate metrics if all_preds and all_labels: f1_macro = f1_score(all_labels, all_preds, average='macro') accuracy = accuracy_score(all_labels, all_preds) - + avg_loss = total_loss / num_batches if num_batches > 0 else 0 - + print(f" ๐Ÿ“Š Epoch {epoch + 1} Results:") print(f" Average Loss: {avg_loss:.4f}") print(f" Validation F1 (Macro): {f1_macro:.4f}") print(f" Validation Accuracy: {accuracy:.4f}") - + # Save best model if f1_macro > best_f1: best_f1 = f1_macro torch.save(model.state_dict(), 'best_simple_model.pth') print(f" ๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") - + # Clear GPU cache if torch.cuda.is_available(): torch.cuda.empty_cache() @@ -432,11 +432,11 @@ def forward(self, input_ids, attention_mask): print("\\n๐Ÿ”ฅ THIS VERSION HAS PROPER INTEGER-TO-EMOTION MAPPING!") print("๐Ÿ”ฅ NO MORE ZERO SAMPLES ISSUE!") print("๐Ÿ”ฅ READY TO ACHIEVE 70% F1 SCORE!")''' - + # Write to file with open('final_bulletproof_training_cell.py', 'w') as f: f.write(cell_code) - + print("โœ… Created FINAL bulletproof training cell: final_bulletproof_training_cell.py") print("๐Ÿ“‹ This version has PROPER INTEGER-TO-EMOTION MAPPING!") print("๐ŸŽฏ This will solve the zero samples issue!") diff --git a/scripts/legacy/create_unique_fallback_dataset.py b/scripts/legacy/create_unique_fallback_dataset.py index 8386cc31d..95af366f6 100644 --- a/scripts/legacy/create_unique_fallback_dataset.py +++ b/scripts/legacy/create_unique_fallback_dataset.py @@ -11,7 +11,7 @@ def create_unique_fallback_dataset(): """Create a unique fallback dataset with no duplicates""" - + # Define unique templates for each emotion with variations emotion_templates = { 'happy': [ @@ -183,10 +183,10 @@ def create_unique_fallback_dataset(): "I feel satisfied with the growth experienced." ] } - + # Create unique samples unique_samples = [] - + for emotion, templates in emotion_templates.items(): for i, template in enumerate(templates): unique_samples.append({ @@ -194,40 +194,40 @@ def create_unique_fallback_dataset(): 'emotion': emotion, 'sample_id': f"{emotion}_{i+1}" }) - + # Shuffle the samples for better training random.shuffle(unique_samples) - + print(f"โœ… Created {len(unique_samples)} UNIQUE samples") print(f"๐Ÿ“Š Samples per emotion: {len(unique_samples) // 12}") - + # Verify no duplicates texts = [sample['text'] for sample in unique_samples] unique_texts = set(texts) print(f"๐Ÿ” Duplicate check: {len(texts)} total, {len(unique_texts)} unique") - + if len(texts) != len(unique_texts): print("โŒ WARNING: DUPLICATES FOUND!") return None - + print("โœ… All samples are unique!") - + # Save the dataset with open('data/unique_fallback_dataset.json', 'w') as f: json.dump(unique_samples, f, indent=2) - + print("๐Ÿ’พ Saved unique fallback dataset to data/unique_fallback_dataset.json") - + # Show emotion distribution emotion_counts = {} for sample in unique_samples: emotion = sample['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - + print("\n๐Ÿ“Š Emotion Distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + return unique_samples if __name__ == "__main__": diff --git a/scripts/legacy/deep_model_analysis.py b/scripts/legacy/deep_model_analysis.py index c1683680a..db1f0e950 100644 --- a/scripts/legacy/deep_model_analysis.py +++ b/scripts/legacy/deep_model_analysis.py @@ -11,12 +11,12 @@ def deep_model_analysis(): """Deep analysis of the model's behavior""" - + print("๐Ÿ” DEEP MODEL ANALYSIS") print("=" * 50) print("๐ŸŽฏ Goal: Understand 99.54% F1 vs 58.3% basic accuracy") print("=" * 50) - + # Load model model_dir = Path(__file__).parent.parent / 'deployment' / 'model' tokenizer = AutoTokenizer.from_pretrained("roberta-base") @@ -24,20 +24,20 @@ def deep_model_analysis(): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model.to(device) model.eval() - + # Define emotion mapping emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - + print(f"\n๐Ÿ“Š EMOTION MAPPING ANALYSIS") print("-" * 40) print("Current mapping (LABEL_0 to LABEL_11):") for i, emotion in enumerate(emotion_mapping): print(f" LABEL_{i} โ†’ {emotion}") - + # Test with different variations print(f"\n๐Ÿงช DETAILED PREDICTION ANALYSIS") print("-" * 40) - + test_cases = [ ("I'm grateful for all the support.", "grateful"), ("I'm feeling overwhelmed with tasks.", "overwhelmed"), @@ -45,82 +45,82 @@ def deep_model_analysis(): ("I'm excited about the new opportunity.", "excited"), ("I'm hopeful for the future.", "hopeful"), ] - + for text, expected_emotion in test_cases: print(f"\n๐Ÿ“ Text: '{text}'") print(f"๐ŸŽฏ Expected: {expected_emotion}") - + # Tokenize inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + # Get all probabilities with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) - + # Get top 3 predictions top_probs, top_indices = torch.topk(probabilities[0], 3) - + print(f"๐Ÿ” Top 3 predictions:") for i, (prob, idx) in enumerate(zip(top_probs, top_indices)): emotion = emotion_mapping[idx.item()] print(f" {i+1}. {emotion}: {prob.item():.3f}") - + # Check if expected emotion is in top 3 expected_idx = emotion_mapping.index(expected_emotion) expected_prob = probabilities[0][expected_idx].item() print(f"๐Ÿ“Š Expected emotion '{expected_emotion}' probability: {expected_prob:.3f}") - + # Analyze model confidence patterns print(f"\n๐Ÿ“ˆ CONFIDENCE PATTERN ANALYSIS") print("-" * 40) - + confidence_by_emotion = {emotion: [] for emotion in emotion_mapping} - + # Test with simple emotion words simple_tests = [ "happy", "sad", "angry", "excited", "calm", "anxious", "proud", "grateful", "hopeful", "tired", "content", "overwhelmed" ] - + for word in simple_tests: inputs = tokenizer(word, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + predicted_emotion = emotion_mapping[predicted_class] confidence_by_emotion[predicted_emotion].append(confidence) - + print(f"'{word}' โ†’ {predicted_emotion} (confidence: {confidence:.3f})") - + # Check for bias towards certain emotions print(f"\n๐ŸŽฏ EMOTION BIAS ANALYSIS") print("-" * 40) - + emotion_counts = {} for emotion in emotion_mapping: emotion_counts[emotion] = len(confidence_by_emotion[emotion]) - + print("Prediction frequency by emotion:") for emotion, count in sorted(emotion_counts.items(), key=lambda x: x[1], reverse=True): print(f" {emotion}: {count} predictions") - + # Check if model is biased towards certain emotions most_common = max(emotion_counts.items(), key=lambda x: x[1]) print(f"\nโš ๏ธ Most predicted emotion: {most_common[0]} ({most_common[1]} times)") - + if most_common[1] > len(simple_tests) * 0.3: print(f"โŒ WARNING: Model shows bias towards '{most_common[0]}'") - + # Test with training-like data print(f"\n๐ŸŽ“ TRAINING-LIKE DATA TEST") print("-" * 40) - + # These should be more similar to what the model was trained on training_like_tests = [ "I am feeling really happy today!", @@ -136,27 +136,27 @@ def deep_model_analysis(): "I feel content with my life.", "I am hopeful for the future." ] - + correct_training_like = 0 for text in training_like_tests: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512, padding=True) inputs = {k: v.to(device) for k, v in inputs.items()} - + with torch.no_grad(): outputs = model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_class].item() - + predicted_emotion = emotion_mapping[predicted_class] - + # Extract expected emotion from text expected_emotion = None for emotion in emotion_mapping: if emotion in text.lower(): expected_emotion = emotion break - + if expected_emotion: is_correct = predicted_emotion == expected_emotion if is_correct: @@ -164,16 +164,16 @@ def deep_model_analysis(): status = "โœ…" else: status = "โŒ" - + print(f"{status} '{text}' โ†’ {predicted_emotion} (expected: {expected_emotion}, confidence: {confidence:.3f})") - + training_like_accuracy = correct_training_like / len(training_like_tests) print(f"\n๐Ÿ“Š Training-like accuracy: {training_like_accuracy:.1%}") - + # Final analysis print(f"\n๐Ÿ” ANALYSIS SUMMARY") print("=" * 50) - + if training_like_accuracy > 0.8: print(f"โœ… Model performs well on training-like data ({training_like_accuracy:.1%})") print(f"โš ๏ธ Issue: Model may be overfitting to specific training patterns") @@ -182,7 +182,7 @@ def deep_model_analysis(): print(f"โŒ Model performs poorly even on training-like data ({training_like_accuracy:.1%})") print(f"โš ๏ธ Issue: Fundamental problem with model training or label mapping") print(f"๐Ÿ’ก Solution: Retrain model with better data or check label mapping") - + return training_like_accuracy > 0.8 if __name__ == "__main__": diff --git a/scripts/legacy/evaluate_whisper_wer.py b/scripts/legacy/evaluate_whisper_wer.py index 453cc96ce..57f58f61f 100644 --- a/scripts/legacy/evaluate_whisper_wer.py +++ b/scripts/legacy/evaluate_whisper_wer.py @@ -142,7 +142,7 @@ def evaluate_wer(api: TranscriptionAPI, samples: list[dict], model_size: str) -> if results: avg_wer = sum(r["wer"] for r in results) / len(results) avg_time = total_time / len(results) - + return { "model_size": model_size, "num_samples": len(results), diff --git a/scripts/legacy/expand_journal_dataset.py b/scripts/legacy/expand_journal_dataset.py index 0786d8f7d..cf92ba84e 100644 --- a/scripts/legacy/expand_journal_dataset.py +++ b/scripts/legacy/expand_journal_dataset.py @@ -21,60 +21,60 @@ def save_expanded_dataset(data, filename='data/expanded_journal_dataset.json'): def create_balanced_dataset(target_size=1000): """Create a balanced expanded dataset.""" print("๐Ÿ”ง Creating balanced expanded dataset...") - + # Load current data current_data = load_current_dataset() - + # Analyze current distribution emotion_counts = {} for entry in current_data: emotion = entry['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - + print(f"๐Ÿ“Š Current emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + # Calculate target per emotion target_per_emotion = target_size // len(emotion_counts) print(f"\n๐ŸŽฏ Target: {target_per_emotion} samples per emotion") - + # Create expanded dataset expanded_data = [] - + for emotion in emotion_counts.keys(): # Get existing samples for this emotion existing_samples = [entry for entry in current_data if entry['emotion'] == emotion] current_count = len(existing_samples) - + print(f"\n๐Ÿ“ Expanding '{emotion}' from {current_count} to {target_per_emotion} samples...") - + # Add existing samples expanded_data.extend(existing_samples) - + # Generate additional samples needed_samples = target_per_emotion - current_count - + if needed_samples > 0: # Create variations of existing samples for i in range(needed_samples): # Pick a random existing sample to base variation on base_sample = random.choice(existing_samples) - + # Create variation variation = create_variation(base_sample, emotion) expanded_data.append(variation) - + print(f"\nโœ… Expanded dataset created:") print(f" Original samples: {len(current_data)}") print(f" Expanded samples: {len(expanded_data)}") print(f" Target size: {target_size}") - + return expanded_data def create_variation(base_sample: Dict, emotion: str) -> Dict: """Create a variation of a base sample.""" - + # Templates for different emotions emotion_templates = { 'happy': [ @@ -222,22 +222,22 @@ def create_variation(base_sample: Dict, emotion: str) -> Dict: "I'm really tired of dealing with this." ] } - + # Get templates for this emotion templates = emotion_templates.get(emotion, [f"I'm feeling {emotion}."]) - + # Create variation template = random.choice(templates) - + # Add some variety to the content variations = [ f"{template} {random.choice(['It\'s been a long day.', 'Things are going well.', 'I need to process this.', 'This is important to me.'])}", f"{template} {random.choice(['I hope this continues.', 'I wonder what\'s next.', 'This feels right.', 'I\'m processing this.'])}", f"{template} {random.choice(['I should reflect on this.', 'This is meaningful.', 'I appreciate this moment.', 'I\'m learning from this.'])}" ] - + content = random.choice(variations) - + return { 'content': content, 'emotion': emotion, @@ -248,16 +248,16 @@ def analyze_expanded_dataset(data): """Analyze the expanded dataset.""" print("\n๐Ÿ“Š Expanded Dataset Analysis:") print("=" * 40) - + emotion_counts = {} for entry in data: emotion = entry['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - + print("Emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + print(f"\nTotal samples: {len(data)}") print(f"Unique emotions: {len(emotion_counts)}") @@ -265,16 +265,16 @@ def main(): """Main function to expand the dataset.""" print("๐Ÿš€ JOURNAL DATASET EXPANSION") print("=" * 50) - + # Create expanded dataset expanded_data = create_balanced_dataset(target_size=1000) - + # Analyze expanded dataset analyze_expanded_dataset(expanded_data) - + # Save expanded dataset save_expanded_dataset(expanded_data) - + print("\n๐ŸŽ‰ Dataset expansion completed!") print("๐Ÿ“‹ Next steps:") print(" 1. Review expanded dataset") diff --git a/scripts/legacy/finalize_emotion_model.py b/scripts/legacy/finalize_emotion_model.py index 014101800..8f61e3741 100755 --- a/scripts/legacy/finalize_emotion_model.py +++ b/scripts/legacy/finalize_emotion_model.py @@ -134,10 +134,10 @@ def forward(self, **kwargs) -> torch.Tensor: # Weighted average of predictions weighted_pred = sum(w * p for w, p in zip(self.weights, predictions)) - + # Apply temperature scaling scaled_pred = weighted_pred / self.temperature - + return scaled_pred def set_temperature(self, temperature: float) -> None: @@ -160,7 +160,7 @@ def create_augmented_dataset(data_loader: GoEmotionsDataLoader, tokenizer: AutoT Augmented dataset """ logger.info("Creating augmented dataset with back-translation...") - + # For now, return the original dataset # TODO: Implement back-translation augmentation return data_loader.get_train_data() @@ -180,7 +180,7 @@ def train_final_model( Training metrics """ logger.info(f"Training final model for {epochs} epochs with batch size {batch_size}") - + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") logger.info(f"Using device: {device}") @@ -205,45 +205,45 @@ def train_final_model( best_f1 = 0.0 for epoch in range(epochs): logger.info(f"Epoch {epoch + 1}/{epochs}") - + # Training model.train() total_loss = 0.0 - + for batch in train_data: optimizer.zero_grad() - + # Forward pass outputs = model(batch["input_ids"], batch["attention_mask"]) loss = focal_loss(outputs, batch["labels"]) - + # Backward pass loss.backward() optimizer.step() - + total_loss += loss.item() - + # Validation model.eval() val_predictions = [] val_labels = [] - + with torch.no_grad(): for batch in val_data: outputs = model(batch["input_ids"], batch["attention_mask"]) predictions = (torch.sigmoid(outputs) > OPTIMAL_THRESHOLD).float() - + val_predictions.append(predictions.cpu()) val_labels.append(batch["labels"].cpu()) - + # Calculate F1 score val_predictions = torch.cat(val_predictions, dim=0) val_labels = torch.cat(val_labels, dim=0) - + f1 = f1_score(val_labels, val_predictions, average='micro', zero_division=0) - + logger.info(f"Epoch {epoch + 1}: Loss = {total_loss:.4f}, F1 = {f1:.4f}") - + # Save best model if f1 > best_f1: best_f1 = f1 @@ -273,19 +273,19 @@ def create_ensemble_model(model_path: str, device: torch.device) -> EnsembleMode Ensemble model """ logger.info("Creating ensemble model...") - + # For now, create a single model ensemble # TODO: Implement multiple model ensemble model, _ = create_bert_emotion_classifier() - + if Path(model_path).exists(): checkpoint = torch.load(model_path, map_location=device) model.load_state_dict(checkpoint['model_state_dict']) logger.info(f"Loaded model from {model_path}") - + model.to(device) model.eval() - + return EnsembleModel([model]) @@ -304,11 +304,11 @@ def evaluate_ensemble( Evaluation metrics """ logger.info("Evaluating ensemble model...") - + ensemble.eval() predictions = [] labels = [] - + with torch.no_grad(): for batch in test_data: outputs = ensemble( @@ -316,21 +316,21 @@ def evaluate_ensemble( attention_mask=batch["attention_mask"].to(device) ) batch_predictions = (torch.sigmoid(outputs) > OPTIMAL_THRESHOLD).float() - + predictions.append(batch_predictions.cpu()) labels.append(batch["labels"].cpu()) - + # Concatenate results predictions = torch.cat(predictions, dim=0) labels = torch.cat(labels, dim=0) - + # Calculate metrics micro_f1 = f1_score(labels, predictions, average='micro', zero_division=0) macro_f1 = f1_score(labels, predictions, average='macro', zero_division=0) precision, recall, _, _ = precision_recall_fscore_support( labels, predictions, average='micro', zero_division=0 ) - + return { 'micro_f1': micro_f1, 'macro_f1': macro_f1, @@ -350,10 +350,10 @@ def save_ensemble_model( output_path: Path to save the model """ logger.info(f"Saving ensemble model to {output_path}") - + # Create output directory Path(output_path).parent.mkdir(parents=True, exist_ok=True) - + # Save model torch.save({ 'ensemble_state_dict': ensemble.state_dict(), @@ -361,7 +361,7 @@ def save_ensemble_model( 'temperature': ensemble.temperature, 'threshold': ensemble.threshold, }, output_path) - + logger.info(f"Model saved successfully!") logger.info(f"Final metrics: {metrics}") @@ -387,38 +387,38 @@ def main(): default=16, help="Training batch size" ) - + args = parser.parse_args() - + logger.info("๐Ÿš€ Starting emotion detection model finalization...") - + # Train final model training_results = train_final_model( output_model=args.output_model, epochs=args.epochs, batch_size=args.batch_size ) - + logger.info(f"Training completed! Best F1: {training_results['best_f1']:.4f}") - + # Check if target F1 score is achieved if training_results['best_f1'] >= TARGET_F1_SCORE: logger.info(f"๐ŸŽ‰ Target F1 score of {TARGET_F1_SCORE} achieved!") - + # Create and evaluate ensemble device = torch.device("cuda" if torch.cuda.is_available() else "cpu") ensemble = create_ensemble_model(args.output_model, device) - + data_loader = GoEmotionsDataLoader() test_data = data_loader.get_test_data() _, tokenizer = create_bert_emotion_classifier() - + metrics = evaluate_ensemble(ensemble, test_data, tokenizer, device) - + # Save ensemble model ensemble_path = args.output_model.replace('.pt', '_ensemble.pt') save_ensemble_model(ensemble, metrics, ensemble_path) - + else: logger.warning(f"โš ๏ธ Target F1 score of {TARGET_F1_SCORE} not achieved. Best: {training_results['best_f1']:.4f}") diff --git a/scripts/legacy/improve_model_f1.py b/scripts/legacy/improve_model_f1.py index 5e05e5972..f0feed59f 100755 --- a/scripts/legacy/improve_model_f1.py +++ b/scripts/legacy/improve_model_f1.py @@ -139,7 +139,7 @@ def improve_model_f1(): for epoch in range(5): logger.info(f"๐Ÿ“š Epoch {epoch + 1}/5") epoch_loss = 0.0 - + for batch_idx, batch in enumerate(train_dataloader): input_ids, attention_mask, batch_labels = batch input_ids = input_ids.to(device) diff --git a/scripts/legacy/integrate_cmu_mosei.py b/scripts/legacy/integrate_cmu_mosei.py index 686b0c743..0b62535f2 100644 --- a/scripts/legacy/integrate_cmu_mosei.py +++ b/scripts/legacy/integrate_cmu_mosei.py @@ -27,36 +27,36 @@ def download_cmu_mosei(): """Download CMU-MOSEI dataset""" print("๐Ÿ“ฅ Downloading CMU-MOSEI dataset...") - + try: # Initialize MOSEI loader mosei = mmdata.MOSEI() - + # Download text embeddings (transcribed sentences) print("๐Ÿ“ Downloading text embeddings...") mosei_emb = mosei.embeddings() - + # Download words (transcribed text) print("๐Ÿ“ Downloading transcribed words...") mosei_words = mosei.words() - + # Get sentiment labels print("๐Ÿท๏ธ Downloading sentiment labels...") sentiments = mosei.sentiments() - + # Get train/validation/test splits print("๐Ÿ“Š Getting dataset splits...") train_ids = mosei.train() valid_ids = mosei.valid() test_ids = mosei.test() - + print(f"โœ… CMU-MOSEI downloaded successfully!") print(f"๐Ÿ“Š Train videos: {len(train_ids)}") print(f"๐Ÿ“Š Validation videos: {len(valid_ids)}") print(f"๐Ÿ“Š Test videos: {len(test_ids)}") - + return mosei_emb, mosei_words, sentiments, train_ids, valid_ids, test_ids - + except Exception as e: print(f"โŒ Error downloading CMU-MOSEI: {e}") return None, None, None, None, None, None @@ -64,9 +64,9 @@ def download_cmu_mosei(): def extract_text_and_emotions(mosei_words, sentiments, train_ids, valid_ids, test_ids): """Extract text sentences and emotion labels from CMU-MOSEI""" print("๐Ÿ” Extracting text and emotion data...") - + dataset_samples = [] - + # Process each video for video_id in list(train_ids) + list(valid_ids) + list(test_ids): if video_id in mosei_words and video_id in sentiments: @@ -77,10 +77,10 @@ def extract_text_and_emotions(mosei_words, sentiments, train_ids, valid_ids, tes if segment_words: # Convert word timestamps to text text = " ".join([word[2] for word in segment_words if word[2]]) - + # Get sentiment label sentiment = sentiments[video_id][segment_id] - + if text.strip() and sentiment is not None: dataset_samples.append({ 'text': text.strip(), @@ -88,17 +88,17 @@ def extract_text_and_emotions(mosei_words, sentiments, train_ids, valid_ids, tes 'video_id': video_id, 'segment_id': segment_id }) - + print(f"โœ… Extracted {len(dataset_samples)} samples") return dataset_samples def map_sentiment_to_emotions(samples): """Map CMU-MOSEI sentiment scores to our 12 target emotions""" print("๐Ÿ—บ๏ธ Mapping sentiments to emotions...") - + # CMU-MOSEI sentiment range: [-3, 3] # Our target emotions: anxious, calm, content, excited, frustrated, grateful, happy, hopeful, overwhelmed, proud, sad, tired - + emotion_mapping = { # Very negative sentiments (-3, -2.5): 'sad', @@ -106,10 +106,10 @@ def map_sentiment_to_emotions(samples): (-2, -1.5): 'anxious', (-1.5, -1): 'tired', (-1, -0.5): 'overwhelmed', - + # Neutral sentiments (-0.5, 0.5): 'calm', - + # Positive sentiments (0.5, 1): 'content', (1, 1.5): 'hopeful', @@ -117,19 +117,19 @@ def map_sentiment_to_emotions(samples): (2, 2.5): 'happy', (2.5, 3): 'excited', } - + mapped_samples = [] - + for sample in samples: sentiment = sample['sentiment'] - + # Find appropriate emotion mapping mapped_emotion = None for (min_sent, max_sent), emotion in emotion_mapping.items(): if min_sent <= sentiment < max_sent: mapped_emotion = emotion break - + # Default mapping for edge cases if mapped_emotion is None: if sentiment < -2.5: @@ -138,7 +138,7 @@ def map_sentiment_to_emotions(samples): mapped_emotion = 'excited' else: mapped_emotion = 'calm' - + mapped_samples.append({ 'text': sample['text'], 'emotion': mapped_emotion, @@ -146,82 +146,82 @@ def map_sentiment_to_emotions(samples): 'video_id': sample['video_id'], 'segment_id': sample['segment_id'] }) - + print(f"โœ… Mapped {len(mapped_samples)} samples to emotions") - + # Show emotion distribution emotion_counts = defaultdict(int) for sample in mapped_samples: emotion_counts[sample['emotion']] += 1 - + print("๐Ÿ“Š Emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + return mapped_samples def save_cmu_mosei_dataset(samples): """Save processed CMU-MOSEI dataset""" print("๐Ÿ’พ Saving CMU-MOSEI dataset...") - + # Save full dataset output_file = 'data/cmu_mosei_emotion_dataset.json' with open(output_file, 'w') as f: json.dump(samples, f, indent=2) - + print(f"โœ… Saved {len(samples)} samples to {output_file}") - + # Create balanced subset for training (similar to your 12 emotions) print("โš–๏ธ Creating balanced training subset...") - + emotion_samples = defaultdict(list) for sample in samples: emotion_samples[sample['emotion']].append(sample) - + # Find minimum samples per emotion min_samples = min(len(samples) for samples in emotion_samples.values()) print(f"๐Ÿ“Š Minimum samples per emotion: {min_samples}") - + # Create balanced dataset balanced_samples = [] for emotion, samples_list in emotion_samples.items(): # Randomly sample min_samples from each emotion selected_samples = np.random.choice(samples_list, size=min_samples, replace=False) balanced_samples.extend(selected_samples) - + balanced_file = 'data/cmu_mosei_balanced_dataset.json' with open(balanced_file, 'w') as f: json.dump(balanced_samples, f, indent=2) - + print(f"โœ… Saved {len(balanced_samples)} balanced samples to {balanced_file}") - + return output_file, balanced_file def main(): """Main integration process""" print("๐Ÿš€ CMU-MOSEI DATASET INTEGRATION") print("=" * 50) - + # Step 1: Download dataset mosei_emb, mosei_words, sentiments, train_ids, valid_ids, test_ids = download_cmu_mosei() - + if mosei_words is None: print("โŒ Failed to download CMU-MOSEI dataset") return - + # Step 2: Extract text and emotions samples = extract_text_and_emotions(mosei_words, sentiments, train_ids, valid_ids, test_ids) - + if not samples: print("โŒ No samples extracted") return - + # Step 3: Map to target emotions mapped_samples = map_sentiment_to_emotions(samples) - + # Step 4: Save datasets full_file, balanced_file = save_cmu_mosei_dataset(mapped_samples) - + print("\n๐ŸŽ‰ CMU-MOSEI Integration Complete!") print("๐Ÿ“‹ Next steps:") print(" 1. Review the datasets in data/") diff --git a/scripts/legacy/reorganize_model_directory.py b/scripts/legacy/reorganize_model_directory.py index eaf859d7a..cb3c05775 100644 --- a/scripts/legacy/reorganize_model_directory.py +++ b/scripts/legacy/reorganize_model_directory.py @@ -16,33 +16,33 @@ def reorganize_model_directory(): """Reorganize the model directory with versioning.""" - + print("๐Ÿ“ REORGANIZING MODEL DIRECTORY") print("=" * 50) - + # Define paths current_model_path = "deployment/model" models_dir = "deployment/models" model_1_path = os.path.join(models_dir, "model_1_fallback") default_model_path = os.path.join(models_dir, "default") - + # Create models directory if it doesn't exist if not os.path.exists(models_dir): os.makedirs(models_dir) print(f"โœ… Created models directory: {models_dir}") - + # 1. Save current model as model_1 (fallback) print(f"\n๐Ÿ’พ SAVING CURRENT MODEL AS FALLBACK") print("-" * 40) - + if os.path.exists(current_model_path): # Copy current model to model_1_fallback if os.path.exists(model_1_path): shutil.rmtree(model_1_path) - + shutil.copytree(current_model_path, model_1_path) print(f"โœ… Saved current model as: {model_1_path}") - + # Create model metadata model_1_metadata = { "version": "1.0", @@ -65,27 +65,27 @@ def reorganize_model_directory(): "status": "fallback_model", "notes": "Successfully resolved configuration persistence issue. Ready for deployment." } - + # Save metadata metadata_path = os.path.join(model_1_path, "model_metadata.json") with open(metadata_path, 'w') as f: json.dump(model_1_metadata, f, indent=2) print(f"โœ… Created model metadata: {metadata_path}") - + else: print(f"โŒ Current model not found at: {current_model_path}") return - + # 2. Create default model directory structure print(f"\n๐Ÿ“‚ CREATING DEFAULT MODEL STRUCTURE") print("-" * 40) - + if os.path.exists(default_model_path): shutil.rmtree(default_model_path) - + os.makedirs(default_model_path) print(f"โœ… Created default model directory: {default_model_path}") - + # Create placeholder metadata for default model default_metadata = { "version": "2.0", @@ -113,17 +113,17 @@ def reorganize_model_directory(): "status": "pending_training", "notes": "Will be trained using COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb" } - + # Save default metadata default_metadata_path = os.path.join(default_model_path, "model_metadata.json") with open(default_metadata_path, 'w') as f: json.dump(default_metadata, f, indent=2) print(f"โœ… Created default model metadata: {default_metadata_path}") - + # 3. Create models index file print(f"\n๐Ÿ“‹ CREATING MODELS INDEX") print("-" * 40) - + models_index = { "models_directory": models_dir, "current_default": "default", @@ -145,16 +145,16 @@ def reorganize_model_directory(): "last_updated": datetime.now().isoformat(), "notes": "Use default model for production, model_1_fallback as backup" } - + index_path = os.path.join(models_dir, "models_index.json") with open(index_path, 'w') as f: json.dump(models_index, f, indent=2) print(f"โœ… Created models index: {index_path}") - + # 4. Create README for models directory print(f"\n๐Ÿ“– CREATING MODELS README") print("-" * 40) - + readme_content = """# Model Versions This directory contains different versions of the emotion detection model. @@ -222,16 +222,16 @@ def reorganize_model_directory(): - Always test models before deployment - Keep fallback models for safety """ - + readme_path = os.path.join(models_dir, "README.md") with open(readme_path, 'w') as f: f.write(readme_content) print(f"โœ… Created models README: {readme_path}") - + # 5. Create symlink for easy access print(f"\n๐Ÿ”— CREATING SYMLINKS") print("-" * 40) - + # Create symlink from deployment/model to default model symlink_path = "deployment/model" if os.path.exists(symlink_path): @@ -244,7 +244,7 @@ def reorganize_model_directory(): shutil.rmtree(backup_path) shutil.move(symlink_path, backup_path) print(f"โœ… Backed up original model to: {backup_path}") - + # Create symlink to default model try: os.symlink(default_model_path, symlink_path) @@ -252,11 +252,11 @@ def reorganize_model_directory(): except Exception as e: print(f"โš ๏ธ Could not create symlink: {e}") print(f" You can manually link {symlink_path} to {default_model_path}") - + # 6. Summary print(f"\n๐Ÿ“‹ REORGANIZATION SUMMARY") print("=" * 50) - + print("โœ… Model directory reorganized successfully!") print() print("๐Ÿ“ New Structure:") diff --git a/scripts/legacy/retrain_with_expanded_dataset.py b/scripts/legacy/retrain_with_expanded_dataset.py index a2845206f..560d10690 100644 --- a/scripts/legacy/retrain_with_expanded_dataset.py +++ b/scripts/legacy/retrain_with_expanded_dataset.py @@ -15,22 +15,22 @@ def load_expanded_dataset(): """Load the expanded journal dataset.""" print("๐Ÿ“Š Loading expanded dataset...") - + with open('data/expanded_journal_dataset.json', 'r') as f: data = json.load(f) - + print(f"โœ… Loaded {len(data)} samples") - + # Analyze distribution emotion_counts = {} for entry in data: emotion = entry['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - + print("๐Ÿ“ˆ Emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + return data class ExpandedEmotionDataset(Dataset): @@ -39,14 +39,14 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] - + encoding = self.tokenizer( text, truncation=True, @@ -54,7 +54,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -68,7 +68,7 @@ def __init__(self, model_name="bert-base-uncased", num_labels=12): self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(0.3) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + def forward(self, input_ids, attention_mask): outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.pooler_output @@ -78,128 +78,128 @@ def forward(self, input_ids, attention_mask): def prepare_expanded_data(data, test_size=0.2, val_size=0.1): """Prepare data for training with expanded dataset.""" print("๐Ÿ”ง Preparing expanded data...") - + # Extract texts and emotions texts = [entry['content'] for entry in data] emotions = [entry['emotion'] for entry in data] - + # Create label encoder label_encoder = LabelEncoder() labels = label_encoder.fit_transform(emotions) - + print(f"โœ… Label encoder created with {len(label_encoder.classes_)} classes") print(f"๐Ÿ“Š Classes: {list(label_encoder.classes_)}") - + # Split data X_temp, X_test, y_temp, y_test = train_test_split( texts, labels, test_size=test_size, random_state=42, stratify=labels ) - + X_train, X_val, y_train, y_val = train_test_split( X_temp, y_temp, test_size=val_size/(1-test_size), random_state=42, stratify=y_temp ) - + print(f"๐Ÿ“Š Data split:") print(f" Training: {len(X_train)} samples") print(f" Validation: {len(X_val)} samples") print(f" Test: {len(X_test)} samples") - + return (X_train, y_train), (X_val, y_val), (X_test, y_test), label_encoder def train_expanded_model(train_data, val_data, label_encoder, epochs=5, batch_size=16): """Train the model with expanded dataset.""" print("๐Ÿš€ Training with expanded dataset...") - + # Setup device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print(f"โœ… Using device: {device}") - + # Load tokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") - + # Create datasets X_train, y_train = train_data X_val, y_val = val_data - + train_dataset = ExpandedEmotionDataset(X_train, y_train, tokenizer) val_dataset = ExpandedEmotionDataset(X_val, y_val, tokenizer) - + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) - + # Initialize model model = ExpandedEmotionClassifier(num_labels=len(label_encoder.classes_)) model.to(device) - + # Setup training optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5) criterion = nn.CrossEntropyLoss() - + # Training loop best_f1 = 0 training_history = [] - + for epoch in range(epochs): print(f"\n๐Ÿ”„ Epoch {epoch + 1}/{epochs}") - + # Training model.train() total_loss = 0 - + for i, batch in enumerate(train_loader): input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() - + if i % 50 == 0: print(f" Batch {i}/{len(train_loader)}, Loss: {loss.item():.4f}") - + # Validation model.eval() val_loss = 0 all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in val_loader: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) val_loss += loss.item() - + preds = torch.argmax(outputs, dim=1) all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + # Calculate metrics avg_train_loss = total_loss / len(train_loader) avg_val_loss = val_loss / len(val_loader) f1_macro = f1_score(all_labels, all_preds, average='macro') accuracy = accuracy_score(all_labels, all_preds) - + print(f"๐Ÿ“Š Epoch {epoch + 1} Results:") print(f" Train Loss: {avg_train_loss:.4f}") print(f" Val Loss: {avg_val_loss:.4f}") print(f" Val F1 (Macro): {f1_macro:.4f}") print(f" Val Accuracy: {accuracy:.4f}") - + # Save best model if f1_macro > best_f1: best_f1 = f1_macro torch.save(model.state_dict(), 'best_expanded_model.pth') print(f"๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") - + training_history.append({ 'epoch': epoch, 'train_loss': avg_train_loss, @@ -207,46 +207,46 @@ def train_expanded_model(train_data, val_data, label_encoder, epochs=5, batch_si 'val_f1_macro': f1_macro, 'val_accuracy': accuracy }) - + return model, training_history, best_f1 def save_expanded_results(training_history, best_f1, label_encoder, test_data): """Save training results.""" print("๐Ÿ’พ Saving results...") - + # Test final model X_test, y_test = test_data device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - + # Load best model model = ExpandedEmotionClassifier(num_labels=len(label_encoder.classes_)) model.load_state_dict(torch.load('best_expanded_model.pth')) model.to(device) model.eval() - + # Test predictions tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") test_dataset = ExpandedEmotionDataset(X_test, y_test, tokenizer) test_loader = DataLoader(test_dataset, batch_size=16, shuffle=False) - + all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in test_loader: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) preds = torch.argmax(outputs, dim=1) all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + # Calculate final metrics final_f1 = f1_score(all_labels, all_preds, average='macro') final_accuracy = accuracy_score(all_labels, all_preds) - + # Save results results = { 'best_f1': best_f1, @@ -259,10 +259,10 @@ def save_expanded_results(training_history, best_f1, label_encoder, test_data): 'expanded_samples': len(X_test) + len([x for x in train_data[0]]) + len([x for x in val_data[0]]), 'test_samples': len(X_test) } - + with open('expanded_training_results.json', 'w') as f: json.dump(results, f, indent=2) - + print(f"โœ… Results saved!") print(f"๐Ÿ“Š Final F1 Score: {final_f1:.4f}") print(f"๐Ÿ“Š Final Accuracy: {final_accuracy:.4f}") @@ -272,19 +272,19 @@ def main(): """Main training function.""" print("๐Ÿš€ RETRAINING WITH EXPANDED DATASET") print("=" * 60) - + # Load expanded dataset data = load_expanded_dataset() - + # Prepare data train_data, val_data, test_data, label_encoder = prepare_expanded_data(data) - + # Train model model, training_history, best_f1 = train_expanded_model(train_data, val_data, label_encoder) - + # Save results save_expanded_results(training_history, best_f1, label_encoder, test_data) - + print("\n๐ŸŽ‰ Retraining completed!") print("๐Ÿ“‹ Next steps:") print(" 1. Test the new model") diff --git a/scripts/legacy/retrain_with_validation.py b/scripts/legacy/retrain_with_validation.py index 8710f134a..20dcd3c17 100644 --- a/scripts/legacy/retrain_with_validation.py +++ b/scripts/legacy/retrain_with_validation.py @@ -8,19 +8,19 @@ def create_improved_training_plan(): """Create an improved training plan with proper validation""" - + print("๐Ÿ”„ IMPROVED TRAINING PLAN") print("=" * 50) print("๐ŸŽฏ Goal: Retrain model to achieve reliable 75-85% F1 score") print("=" * 50) - + print(f"\nโŒ CURRENT ISSUES IDENTIFIED:") print("-" * 40) print("1. Model bias towards 'grateful' and 'happy' emotions") print("2. Poor generalization (58.3% accuracy on basic tests)") print("3. Overfitting to specific training patterns") print("4. Label mapping inconsistencies") - + print(f"\nโœ… IMPROVED TRAINING STRATEGY:") print("-" * 40) print("1. Use balanced dataset with equal emotion distribution") @@ -28,7 +28,7 @@ def create_improved_training_plan(): print("3. Add regularization to prevent overfitting") print("4. Use early stopping based on validation performance") print("5. Test on diverse, realistic examples") - + print(f"\n๐Ÿ“Š VALIDATION REQUIREMENTS:") print("-" * 40) print("โœ… Basic functionality test: >80% accuracy") @@ -36,7 +36,7 @@ def create_improved_training_plan(): print("โœ… Edge case handling: >70% success rate") print("โœ… No emotion bias: <30% predictions for any single emotion") print("โœ… Consistent predictions: 100% consistency for same input") - + print(f"\n๐Ÿš€ RECOMMENDED ACTIONS:") print("-" * 40) print("1. Create balanced training dataset") @@ -44,15 +44,15 @@ def create_improved_training_plan(): print("3. Use regularization techniques") print("4. Test extensively before deployment") print("5. Monitor for bias and overfitting") - + # Create improved training notebook create_improved_notebook() - + return True def create_improved_notebook(): """Create an improved training notebook""" - + notebook_content = '''{ "cells": [ { @@ -382,12 +382,12 @@ def create_improved_notebook(): "nbformat": 4, "nbformat_minor": 4 }''' - + # Save the notebook notebook_path = Path(__file__).parent.parent / 'notebooks' / 'IMPROVED_TRAINING_WITH_VALIDATION.ipynb' with open(notebook_path, 'w') as f: f.write(notebook_content) - + print(f"โœ… Created improved training notebook: {notebook_path}") print(f"๐Ÿ“‹ Instructions:") print(f" 1. Download the notebook file") diff --git a/scripts/legacy/simple_cmu_mosei_download.py b/scripts/legacy/simple_cmu_mosei_download.py index 1723581c8..71575a53a 100644 --- a/scripts/legacy/simple_cmu_mosei_download.py +++ b/scripts/legacy/simple_cmu_mosei_download.py @@ -13,18 +13,18 @@ def download_cmu_mosei_sample(): """Download a sample of CMU-MOSEI data from Hugging Face""" print("๐Ÿ“ฅ Attempting to download CMU-MOSEI sample...") - + # Try to get CMU-MOSEI from Hugging Face datasets try: from datasets import load_dataset print("โœ… Hugging Face datasets available") - + # Try to load CMU-MOSEI dataset = load_dataset("cmu-mosei") print("โœ… CMU-MOSEI dataset loaded successfully!") - + return dataset - + except ImportError: print("โŒ Hugging Face datasets not available") return None @@ -35,10 +35,10 @@ def download_cmu_mosei_sample(): def create_synthetic_cmu_mosei(): """Create synthetic CMU-MOSEI-like data for testing""" print("๐Ÿ”ง Creating synthetic CMU-MOSEI-like dataset...") - + # Generate realistic text samples with sentiment scores synthetic_data = [] - + # Negative sentiment samples (sad, frustrated, anxious) negative_samples = [ ("I'm really disappointed with how this turned out", -2.5), @@ -52,7 +52,7 @@ def create_synthetic_cmu_mosei(): ("I'm tired of dealing with this", -1.6), ("This situation is really stressful", -2.1), ] - + # Neutral sentiment samples (calm, content) neutral_samples = [ ("I'm feeling okay about this", 0.2), @@ -66,7 +66,7 @@ def create_synthetic_cmu_mosei(): ("I'm feeling calm", 0.4), ("It's manageable", 0.2), ] - + # Positive sentiment samples (happy, excited, grateful, hopeful, proud) positive_samples = [ ("I'm really happy with the results", 2.5), @@ -80,10 +80,10 @@ def create_synthetic_cmu_mosei(): ("I'm optimistic about this", 1.9), ("This is fantastic", 2.9), ] - + # Combine all samples all_samples = negative_samples + neutral_samples + positive_samples - + # Create dataset entries for i, (text, sentiment) in enumerate(all_samples): synthetic_data.append({ @@ -92,14 +92,14 @@ def create_synthetic_cmu_mosei(): 'video_id': f'video_{i//10:03d}', 'segment_id': f'{i%10}' }) - + print(f"โœ… Created {len(synthetic_data)} synthetic samples") return synthetic_data def map_sentiment_to_emotions(samples): """Map sentiment scores to our 12 target emotions""" print("๐Ÿ—บ๏ธ Mapping sentiments to emotions...") - + emotion_mapping = { # Very negative sentiments (-3, -2.5): 'sad', @@ -107,10 +107,10 @@ def map_sentiment_to_emotions(samples): (-2, -1.5): 'anxious', (-1.5, -1): 'tired', (-1, -0.5): 'overwhelmed', - + # Neutral sentiments (-0.5, 0.5): 'calm', - + # Positive sentiments (0.5, 1): 'content', (1, 1.5): 'hopeful', @@ -118,19 +118,19 @@ def map_sentiment_to_emotions(samples): (2, 2.5): 'happy', (2.5, 3): 'excited', } - + mapped_samples = [] - + for sample in samples: sentiment = sample['sentiment'] - + # Find appropriate emotion mapping mapped_emotion = None for (min_sent, max_sent), emotion in emotion_mapping.items(): if min_sent <= sentiment < max_sent: mapped_emotion = emotion break - + # Default mapping for edge cases if mapped_emotion is None: if sentiment < -2.5: @@ -139,7 +139,7 @@ def map_sentiment_to_emotions(samples): mapped_emotion = 'excited' else: mapped_emotion = 'calm' - + mapped_samples.append({ 'text': sample['text'], 'emotion': mapped_emotion, @@ -147,37 +147,37 @@ def map_sentiment_to_emotions(samples): 'video_id': sample['video_id'], 'segment_id': sample['segment_id'] }) - + print(f"โœ… Mapped {len(mapped_samples)} samples to emotions") - + # Show emotion distribution emotion_counts = defaultdict(int) for sample in mapped_samples: emotion_counts[sample['emotion']] += 1 - + print("๐Ÿ“Š Emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + return mapped_samples def save_dataset(samples, filename): """Save dataset to JSON file""" print(f"๐Ÿ’พ Saving dataset to {filename}...") - + with open(filename, 'w') as f: json.dump(samples, f, indent=2) - + print(f"โœ… Saved {len(samples)} samples to {filename}") def main(): """Main function""" print("๐Ÿš€ SIMPLE CMU-MOSEI DOWNLOAD") print("=" * 40) - + # Try to download real CMU-MOSEI dataset = download_cmu_mosei_sample() - + if dataset is None: print("๐Ÿ“ Using synthetic CMU-MOSEI-like data for testing...") samples = create_synthetic_cmu_mosei() @@ -195,29 +195,29 @@ def main(): 'video_id': item.get('video_id', 'unknown'), 'segment_id': item.get('segment_id', '0') }) - + # Map to emotions mapped_samples = map_sentiment_to_emotions(samples) - + # Save datasets save_dataset(mapped_samples, 'data/cmu_mosei_emotion_dataset.json') - + # Create balanced subset print("โš–๏ธ Creating balanced training subset...") emotion_samples = defaultdict(list) for sample in mapped_samples: emotion_samples[sample['emotion']].append(sample) - + min_samples = min(len(samples) for samples in emotion_samples.values()) print(f"๐Ÿ“Š Minimum samples per emotion: {min_samples}") - + balanced_samples = [] for emotion, samples_list in emotion_samples.items(): selected_samples = np.random.choice(samples_list, size=min_samples, replace=False) balanced_samples.extend(selected_samples) - + save_dataset(balanced_samples, 'data/cmu_mosei_balanced_dataset.json') - + print("\n๐ŸŽ‰ CMU-MOSEI Integration Complete!") print("๐Ÿ“‹ Next steps:") print(" 1. Review the datasets in data/") diff --git a/scripts/legacy/simple_f1_evaluation.py b/scripts/legacy/simple_f1_evaluation.py index 66e99ccc4..9503fc123 100644 --- a/scripts/legacy/simple_f1_evaluation.py +++ b/scripts/legacy/simple_f1_evaluation.py @@ -39,14 +39,14 @@ def evaluate_current_f1(): # Load model logger.info("๐Ÿค– Loading emotion detection model...") model, loss_fn = create_bert_emotion_classifier() - + # Check for existing checkpoint checkpoint_paths = [ "models/checkpoints/bert_emotion_classifier_final.pt", "test_checkpoints/best_model.pt", "test_checkpoints_dev/best_model.pt", ] - + checkpoint_loaded = False for checkpoint_path in checkpoint_paths: if Path(checkpoint_path).exists(): @@ -61,7 +61,7 @@ def evaluate_current_f1(): except Exception as e: logger.warning(f"โš ๏ธ Failed to load checkpoint {checkpoint_path}: {e}") continue - + if not checkpoint_loaded: logger.warning("โš ๏ธ No valid checkpoint found, using untrained model") @@ -75,22 +75,22 @@ def evaluate_current_f1(): # Evaluate on test set logger.info("๐Ÿงช Evaluating on test set...") - + test_data = datasets["test_data"] all_predictions = [] all_labels = [] - + batch_size = 16 num_classes = 28 # GoEmotions has 28 emotion classes - + with torch.no_grad(): for i in range(0, len(test_data), batch_size): end_idx = min(i + batch_size, len(test_data)) batch_data = test_data.select(range(i, end_idx)) - + texts = batch_data["text"] labels = batch_data["labels"] - + # Convert labels to one-hot format batch_labels = [] for label_list in labels: @@ -99,7 +99,7 @@ def evaluate_current_f1(): if 0 <= label_idx < num_classes: label_vector[label_idx] = 1 batch_labels.append(label_vector) - + # Tokenize inputs = tokenizer( texts, @@ -108,36 +108,36 @@ def evaluate_current_f1(): max_length=512, return_tensors="pt" ) - + input_ids = inputs["input_ids"].to(device) attention_mask = inputs["attention_mask"].to(device) - + # Get predictions outputs = model(input_ids, attention_mask) predictions = torch.sigmoid(outputs) > 0.5 - + all_predictions.extend(predictions.cpu().numpy()) all_labels.extend(batch_labels) - + if (i // batch_size + 1) % 10 == 0: logger.info(f" Processed {end_idx}/{len(test_data)} samples") # Calculate metrics logger.info("๐Ÿ“ˆ Calculating metrics...") - + # Convert to numpy arrays all_predictions = np.array(all_predictions) all_labels = np.array(all_labels) - + # Calculate F1 scores micro_f1 = f1_score(all_labels, all_predictions, average='micro', zero_division=0) macro_f1 = f1_score(all_labels, all_predictions, average='macro', zero_division=0) weighted_f1 = f1_score(all_labels, all_predictions, average='weighted', zero_division=0) - + # Calculate precision and recall micro_precision = precision_score(all_labels, all_predictions, average='micro', zero_division=0) micro_recall = recall_score(all_labels, all_predictions, average='micro', zero_division=0) - + # Display results logger.info("๐Ÿ“Š EVALUATION RESULTS:") logger.info("=" * 50) @@ -147,21 +147,21 @@ def evaluate_current_f1(): logger.info(f"Micro Precision: {micro_precision:.4f} ({micro_precision*100:.2f}%)") logger.info(f"Micro Recall: {micro_recall:.4f} ({micro_recall*100:.2f}%)") logger.info("=" * 50) - + # Assessment target_f1 = 0.80 # 80% target progress = (micro_f1 / target_f1) * 100 - + logger.info(f"๐ŸŽฏ TARGET F1: {target_f1*100:.0f}%") logger.info(f"๐Ÿ“Š CURRENT F1: {micro_f1*100:.2f}%") logger.info(f"๐Ÿ“ˆ PROGRESS: {progress:.1f}% of target") - + if micro_f1 >= target_f1: logger.info("๐ŸŽ‰ TARGET ACHIEVED!") else: gap = target_f1 - micro_f1 logger.info(f"๐Ÿ“‰ GAP: {gap*100:.2f} percentage points needed") - + return { "micro_f1": micro_f1, "macro_f1": macro_f1, diff --git a/scripts/legacy/validate_model_performance.py b/scripts/legacy/validate_model_performance.py index 1a0d10045..93fd02c7e 100644 --- a/scripts/legacy/validate_model_performance.py +++ b/scripts/legacy/validate_model_performance.py @@ -28,11 +28,11 @@ def check_model_configuration(model_path): """Check if the model configuration is correct.""" print("๐Ÿ” CHECKING MODEL CONFIGURATION") print("=" * 50) - + try: with open(os.path.join(model_path, 'config.json'), 'r') as f: config = json.load(f) - + print(f"Model type: {config.get('model_type', 'NOT FOUND')}") print(f"Architecture: {config.get('architectures', ['NOT FOUND'])[0]}") print(f"Hidden layers: {config.get('num_hidden_layers', 'NOT FOUND')}") @@ -40,13 +40,13 @@ def check_model_configuration(model_path): print(f"Number of labels: {config.get('num_labels', 'NOT FOUND')}") print(f"ID to label mapping: {config.get('id2label', 'NOT FOUND')}") print(f"Label to ID mapping: {config.get('label2id', 'NOT FOUND')}") - + # Check if emotion labels are properly set id2label = config.get('id2label', {}) if isinstance(id2label, dict): emotion_labels = list(id2label.values()) print(f"Emotion labels: {emotion_labels}") - + # Check if labels are emotion names or generic if all(label.startswith('LABEL_') for label in emotion_labels): print("โŒ WARNING: Model uses generic LABEL_X format instead of emotion names") @@ -57,7 +57,7 @@ def check_model_configuration(model_path): else: print("โŒ ERROR: Invalid id2label configuration") return False - + except Exception as e: print(f"โŒ Error reading configuration: {str(e)}") return False @@ -66,70 +66,70 @@ def create_test_dataset(): """Create a proper test dataset with unseen examples.""" print("\n๐Ÿ“Š CREATING PROPER TEST DATASET") print("=" * 50) - + # Test examples that are DIFFERENT from training data test_examples = [ # anxious - different phrasing {'text': 'The upcoming deadline is causing me stress and worry.', 'expected': 'anxious'}, {'text': 'I have butterflies in my stomach about tomorrow.', 'expected': 'anxious'}, {'text': 'The uncertainty of the situation is making me nervous.', 'expected': 'anxious'}, - + # calm - different phrasing {'text': 'I feel at peace with the world around me.', 'expected': 'calm'}, {'text': 'There is a sense of tranquility in my mind.', 'expected': 'calm'}, {'text': 'I am in a state of serenity right now.', 'expected': 'calm'}, - + # content - different phrasing {'text': 'I am satisfied with how things are going.', 'expected': 'content'}, {'text': 'Life feels complete and fulfilling at the moment.', 'expected': 'content'}, {'text': 'I have a sense of inner satisfaction.', 'expected': 'content'}, - + # excited - different phrasing {'text': 'I am thrilled about the upcoming adventure.', 'expected': 'excited'}, {'text': 'My heart is racing with anticipation.', 'expected': 'excited'}, {'text': 'I can barely contain my enthusiasm.', 'expected': 'excited'}, - + # frustrated - different phrasing {'text': 'This situation is driving me up the wall.', 'expected': 'frustrated'}, {'text': 'I am at my wit\'s end with this problem.', 'expected': 'frustrated'}, {'text': 'This is really getting on my nerves.', 'expected': 'frustrated'}, - + # grateful - different phrasing {'text': 'I appreciate all the kindness shown to me.', 'expected': 'grateful'}, {'text': 'My heart is full of thankfulness.', 'expected': 'grateful'}, {'text': 'I am blessed with wonderful people in my life.', 'expected': 'grateful'}, - + # happy - different phrasing {'text': 'Joy fills my heart today.', 'expected': 'happy'}, {'text': 'I am in a wonderful mood.', 'expected': 'happy'}, {'text': 'My spirits are lifted and bright.', 'expected': 'happy'}, - + # hopeful - different phrasing {'text': 'I see a bright future ahead.', 'expected': 'hopeful'}, {'text': 'There is light at the end of the tunnel.', 'expected': 'hopeful'}, {'text': 'I believe better days are coming.', 'expected': 'hopeful'}, - + # overwhelmed - different phrasing {'text': 'I feel like I am drowning in responsibilities.', 'expected': 'overwhelmed'}, {'text': 'Everything is too much to handle right now.', 'expected': 'overwhelmed'}, {'text': 'I am buried under a mountain of tasks.', 'expected': 'overwhelmed'}, - + # proud - different phrasing {'text': 'I have accomplished something meaningful.', 'expected': 'proud'}, {'text': 'My achievements make me stand tall.', 'expected': 'proud'}, {'text': 'I feel a sense of accomplishment.', 'expected': 'proud'}, - + # sad - different phrasing {'text': 'My heart feels heavy with sorrow.', 'expected': 'sad'}, {'text': 'There is a cloud of melancholy over me.', 'expected': 'sad'}, {'text': 'I am feeling down and blue.', 'expected': 'sad'}, - + # tired - different phrasing {'text': 'I am completely exhausted from the day.', 'expected': 'tired'}, {'text': 'My energy is completely drained.', 'expected': 'tired'}, {'text': 'I feel like I could sleep for days.', 'expected': 'tired'} ] - + print(f"โœ… Created test dataset with {len(test_examples)} unseen examples") return test_examples @@ -137,43 +137,43 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): """Evaluate model performance on unseen examples.""" print("\n๐Ÿงช EVALUATING MODEL PERFORMANCE") print("=" * 50) - + model.eval() device = next(model.parameters()).device - + results = [] predictions_by_emotion = {emotion: 0 for emotion in emotions} - + print("Testing on unseen examples...") print("-" * 50) - + for i, example in enumerate(test_examples): text = example['text'] expected = example['expected'] - + # Tokenize inputs = tokenizer(text, return_tensors='pt', truncation=True, max_length=128) inputs = {k: v.to(device) for k, v in inputs.items()} - + # Predict with torch.no_grad(): outputs = model(**inputs) predictions = torch.softmax(outputs.logits, dim=1) predicted_class = torch.argmax(predictions, dim=1).item() confidence = predictions[0][predicted_class].item() - + # Get predicted emotion if predicted_class < len(emotions): predicted_emotion = emotions[predicted_class] else: predicted_emotion = f"UNKNOWN_{predicted_class}" - + predictions_by_emotion[predicted_emotion] += 1 - + # Check if correct is_correct = predicted_emotion == expected status = "โœ…" if is_correct else "โŒ" - + results.append({ 'text': text, 'expected': expected, @@ -181,29 +181,29 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): 'confidence': confidence, 'correct': is_correct }) - + print(f"{status} {text[:50]}... โ†’ {predicted_emotion} (expected: {expected}, confidence: {confidence:.3f})") - + # Calculate metrics correct = sum(1 for r in results if r['correct']) accuracy = correct / len(results) - + print(f"\n๐Ÿ“Š PERFORMANCE SUMMARY") print("=" * 30) print(f"Total examples: {len(results)}") print(f"Correct predictions: {correct}") print(f"Accuracy: {accuracy:.1%}") - + # Bias analysis print(f"\n๐ŸŽฏ BIAS ANALYSIS") print("=" * 20) for emotion, count in predictions_by_emotion.items(): percentage = count / len(results) * 100 print(f" {emotion}: {count} predictions ({percentage:.1f}%)") - + # Determine if model is reliable max_bias = max(predictions_by_emotion.values()) / len(results) - + print(f"\n๐Ÿ” RELIABILITY ASSESSMENT") print("=" * 30) if accuracy >= 0.8 and max_bias <= 0.3: @@ -215,35 +215,35 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): print(f"โŒ Accuracy too low: {accuracy:.1%} (need >80%)") if max_bias > 0.3: print(f"โŒ Too much bias: {max_bias:.1%} (need <30%)") - + return results, accuracy, max_bias def check_for_data_leakage(training_data, test_examples): """Check if there's data leakage between training and test sets.""" print("\n๐Ÿ” CHECKING FOR DATA LEAKAGE") print("=" * 40) - + training_texts = [item['text'].lower() for item in training_data] test_texts = [item['text'].lower() for item in test_examples] - + exact_matches = 0 similar_matches = 0 - + for test_text in test_texts: # Check for exact matches if test_text in training_texts: exact_matches += 1 print(f"โŒ EXACT MATCH FOUND: {test_text[:50]}...") - + # Check for similar matches (same emotion words) for train_text in training_texts: if any(word in test_text for word in train_text.split() if len(word) > 4): similar_matches += 1 break - + print(f"Exact matches: {exact_matches}/{len(test_texts)}") print(f"Similar matches: {similar_matches}/{len(test_texts)}") - + if exact_matches > 0: print("โŒ CRITICAL: Data leakage detected! Test examples are in training data.") return True @@ -258,33 +258,33 @@ def main(): """Main validation function.""" print("๐Ÿ”ฌ COMPREHENSIVE MODEL VALIDATION") print("=" * 60) - + # Model path model_path = "./deployment/model" - + # Check if model exists if not os.path.exists(model_path): print(f"โŒ Model not found at: {model_path}") print("Please ensure the model is saved in the deployment/model directory.") return - + # Load model and tokenizer tokenizer, model = load_model_and_tokenizer(model_path) if tokenizer is None or model is None: return - + # Check model configuration config_ok = check_model_configuration(model_path) - + # Define emotions emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - + # Create test dataset test_examples = create_test_dataset() - + # Evaluate performance results, accuracy, max_bias = evaluate_model_performance(model, tokenizer, test_examples, emotions) - + # Check for data leakage (if training data is available) training_data_path = "./data/balanced_training_data.json" if os.path.exists(training_data_path): @@ -296,7 +296,7 @@ def main(): print("โš ๏ธ Could not check for data leakage (training data not accessible)") else: print("โš ๏ธ Training data not found, skipping data leakage check") - + # Summary print(f"\n๐Ÿ“‹ VALIDATION SUMMARY") print("=" * 30) @@ -304,7 +304,7 @@ def main(): print(f"Accuracy on unseen data: {accuracy:.1%}") print(f"Maximum bias: {max_bias:.1%}") print(f"Model reliable: {'โœ…' if accuracy >= 0.8 and max_bias <= 0.3 else 'โŒ'}") - + if accuracy < 0.8: print(f"\n๐Ÿ’ก RECOMMENDATIONS:") print("1. Increase training dataset size") diff --git a/scripts/maintenance/emergency_f1_fix.py b/scripts/maintenance/emergency_f1_fix.py index 947b378ac..2809cc7a2 100644 --- a/scripts/maintenance/emergency_f1_fix.py +++ b/scripts/maintenance/emergency_f1_fix.py @@ -38,28 +38,28 @@ class FocalLoss(nn.Module): """Focal Loss for handling class imbalance.""" - + def __init__(self, alpha=0.25, gamma=2.0, class_weights=None): super().__init__() self.alpha = alpha self.gamma = gamma self.class_weights = class_weights - + def forward(self, inputs, targets): bce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction='none') pt = torch.exp(-bce_loss) focal_loss = self.alpha * (1 - pt) ** self.gamma * bce_loss - + if self.class_weights is not None: focal_loss = focal_loss * self.class_weights.unsqueeze(0) - + return focal_loss.mean() def create_optimized_model(class_weights): """Create model with optimal settings for F1 improvement.""" logger.info("๐Ÿค– Creating optimized BERT model...") - + model = BERTEmotionClassifier( model_name="bert-base-uncased", num_emotions=28, @@ -69,21 +69,21 @@ def create_optimized_model(class_weights): temperature=1.0, class_weights=torch.tensor(class_weights, dtype=torch.float32) if class_weights is not None else None ) - + return model def prepare_training_data(datasets, tokenizer, batch_size=16): """Prepare training data with proper tokenization.""" logger.info("๐Ÿ“Š Preparing training data...") - + train_data = datasets["train_data"] val_data = datasets["val_data"] - + def tokenize_dataset(dataset): texts = dataset["text"] labels = dataset["labels"] - + # Tokenize inputs = tokenizer( texts, @@ -92,7 +92,7 @@ def tokenize_dataset(dataset): max_length=256, # Reduced for faster training return_tensors="pt" ) - + # Convert labels to one-hot num_classes = 28 label_vectors = [] @@ -102,19 +102,19 @@ def tokenize_dataset(dataset): if 0 <= label_idx < num_classes: label_vector[label_idx] = 1 label_vectors.append(label_vector) - + return TensorDataset( inputs["input_ids"], inputs["attention_mask"], torch.tensor(label_vectors, dtype=torch.float32) ) - + train_dataset = tokenize_dataset(train_data) val_dataset = tokenize_dataset(val_data) - + train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False) - + return train_loader, val_loader @@ -123,27 +123,27 @@ def evaluate_model(model, dataloader, device, threshold=0.3): model.eval() all_predictions = [] all_labels = [] - + with torch.no_grad(): for batch in dataloader: input_ids, attention_mask, labels = batch input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device) - + outputs = model(input_ids, attention_mask) predictions = torch.sigmoid(outputs) > threshold - + all_predictions.extend(predictions.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + # Calculate metrics all_predictions = np.array(all_predictions) all_labels = np.array(all_labels) - + micro_f1 = f1_score(all_labels, all_predictions, average='micro', zero_division=0) macro_f1 = f1_score(all_labels, all_predictions, average='macro', zero_division=0) - + return { 'micro_f1': micro_f1, 'macro_f1': macro_f1, @@ -155,10 +155,10 @@ def evaluate_model(model, dataloader, device, threshold=0.3): def train_with_focal_loss(model, train_loader, val_loader, device, epochs=5): """Train model with focal loss and optimization.""" logger.info("๐Ÿš€ Starting Focal Loss training...") - + # Optimizer with lower learning rate optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5, weight_decay=0.01) - + # Learning rate scheduler total_steps = len(train_loader) * epochs scheduler = get_linear_schedule_with_warmup( @@ -166,18 +166,18 @@ def train_with_focal_loss(model, train_loader, val_loader, device, epochs=5): num_warmup_steps=total_steps // 10, num_training_steps=total_steps ) - + # Focal loss class_weights = model.class_weights.to(device) if model.class_weights is not None else None focal_loss = FocalLoss(alpha=0.25, gamma=2.0, class_weights=class_weights) - + best_f1 = 0.0 patience = 3 patience_counter = 0 - + for epoch in range(epochs): logger.info(f"๐Ÿ“ˆ Epoch {epoch + 1}/{epochs}") - + # Training model.train() total_loss = 0 @@ -186,94 +186,94 @@ def train_with_focal_loss(model, train_loader, val_loader, device, epochs=5): input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device) - + optimizer.zero_grad() - + outputs = model(input_ids, attention_mask) loss = focal_loss(outputs, labels) - + loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() scheduler.step() - + total_loss += loss.item() - + if batch_idx % 50 == 0: logger.info(f" Batch {batch_idx}: Loss = {loss.item():.4f}") - + avg_loss = total_loss / len(train_loader) logger.info(f" Average Loss: {avg_loss:.4f}") - + # Validation val_results = evaluate_model(model, val_loader, device, threshold=0.3) val_f1 = val_results['micro_f1'] - + logger.info(f" Validation F1: {val_f1:.4f} ({val_f1*100:.2f}%)") - + # Save best model if val_f1 > best_f1: best_f1 = val_f1 patience_counter = 0 - + # Save checkpoint checkpoint_path = Path("models/checkpoints/emergency_f1_fix.pt") checkpoint_path.parent.mkdir(parents=True, exist_ok=True) - + torch.save({ 'model_state_dict': model.state_dict(), 'epoch': epoch, 'val_f1': val_f1, 'optimizer_state_dict': optimizer.state_dict(), }, checkpoint_path) - + logger.info(f" โœ… New best model saved! F1: {val_f1:.4f}") else: patience_counter += 1 if patience_counter >= patience: logger.info(f" โน๏ธ Early stopping at epoch {epoch + 1}") break - + return best_f1 def optimize_threshold(model, val_loader, device): """Optimize prediction threshold for maximum F1.""" logger.info("๐ŸŽฏ Optimizing prediction threshold...") - + model.eval() all_outputs = [] all_labels = [] - + with torch.no_grad(): for batch in val_loader: input_ids, attention_mask, labels = batch input_ids = input_ids.to(device) attention_mask = attention_mask.to(device) labels = labels.to(device) - + outputs = model(input_ids, attention_mask) probabilities = torch.sigmoid(outputs) - + all_outputs.extend(probabilities.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + all_outputs = np.array(all_outputs) all_labels = np.array(all_labels) - + # Test different thresholds thresholds = np.arange(0.1, 0.6, 0.05) best_threshold = 0.3 best_f1 = 0.0 - + for threshold in thresholds: predictions = all_outputs > threshold f1 = f1_score(all_labels, predictions, average='micro', zero_division=0) - + if f1 > best_f1: best_f1 = f1 best_threshold = threshold - + logger.info(f" Best threshold: {best_threshold:.2f} (F1: {best_f1:.4f})") return best_threshold @@ -282,47 +282,47 @@ def emergency_f1_fix(): """Main function to fix F1 score emergency.""" logger.info("๐Ÿšจ EMERGENCY F1 FIX - SENIOR ENGINEER APPROACH") logger.info("=" * 60) - + start_time = time.time() - + try: # Load dataset logger.info("๐Ÿ“Š Loading GoEmotions dataset...") data_loader = GoEmotionsDataLoader() data_loader.download_dataset() datasets = data_loader.prepare_datasets() - + # Get class weights class_weights = datasets["class_weights"] logger.info(f"๐Ÿ“Š Class weights computed: min={class_weights.min():.3f}, max={class_weights.max():.3f}") - + # Create model model = create_optimized_model(class_weights) - + # Create tokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") - + # Prepare data train_loader, val_loader = prepare_training_data(datasets, tokenizer, batch_size=16) - + # Set device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) - + # Train with focal loss best_val_f1 = train_with_focal_loss(model, train_loader, val_loader, device, epochs=5) - + # Optimize threshold best_threshold = optimize_threshold(model, val_loader, device) - + # Final evaluation on test set logger.info("๐Ÿงช Final evaluation on test set...") test_data = datasets["test_data"] - + # Create test loader test_texts = test_data["text"] test_labels = test_data["labels"] - + inputs = tokenizer( test_texts, padding=True, @@ -330,7 +330,7 @@ def emergency_f1_fix(): max_length=256, return_tensors="pt" ) - + # Convert labels to one-hot num_classes = 28 test_label_vectors = [] @@ -340,17 +340,17 @@ def emergency_f1_fix(): if 0 <= label_idx < num_classes: label_vector[label_idx] = 1 test_label_vectors.append(label_vector) - + test_dataset = TensorDataset( inputs["input_ids"], inputs["attention_mask"], torch.tensor(test_label_vectors, dtype=torch.float32) ) test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False) - + # Evaluate with optimized threshold test_results = evaluate_model(model, test_loader, device, threshold=best_threshold) - + # Display results logger.info("๐Ÿ“Š FINAL RESULTS:") logger.info("=" * 60) @@ -359,23 +359,23 @@ def emergency_f1_fix(): logger.info(f"Best Threshold: {best_threshold:.2f}") logger.info(f"Training Time: {time.time() - start_time:.1f}s") logger.info("=" * 60) - + # Assessment target_f1 = 0.60 # 60% target for emergency fix progress = (test_results['micro_f1'] / target_f1) * 100 - + logger.info(f"๐ŸŽฏ TARGET F1: {target_f1*100:.0f}%") logger.info(f"๐Ÿ“Š ACHIEVED F1: {test_results['micro_f1']*100:.2f}%") logger.info(f"๐Ÿ“ˆ PROGRESS: {progress:.1f}% of target") - + if test_results['micro_f1'] >= target_f1: logger.info("๐ŸŽ‰ EMERGENCY TARGET ACHIEVED!") else: gap = target_f1 - test_results['micro_f1'] logger.info(f"๐Ÿ“‰ GAP: {gap*100:.2f} percentage points needed") - + return test_results['micro_f1'] - + except Exception as e: logger.error(f"โŒ Emergency F1 fix failed: {e}") import traceback diff --git a/scripts/maintenance/fix_code_quality.py b/scripts/maintenance/fix_code_quality.py index 0ff64ea22..9223d43b1 100644 --- a/scripts/maintenance/fix_code_quality.py +++ b/scripts/maintenance/fix_code_quality.py @@ -59,10 +59,10 @@ def fix_f_strings(self, content: str) -> str: # Fix f-strings without placeholders content = re.sub(r'f"([^"]*)"', r'"\1"', content) content = re.sub(r"f'([^']*)'", r"'\1'", content) - + # Fix f-strings with invalid syntax content = re.sub(r'f"([^"]*)\{([^}]*)\}([^"]*)"', r'f"\1{\2}\3"', content) - + return content def fix_import_order(self, content: str) -> str: @@ -70,16 +70,16 @@ def fix_import_order(self, content: str) -> str: lines = content.split("\n") import_lines = [] other_lines = [] - + for line in lines: if line.strip().startswith(("import ", "from ")): import_lines.append(line) else: other_lines.append(line) - + # Sort import lines import_lines.sort() - + # Reconstruct content return "\n".join(import_lines + [""] + other_lines) @@ -87,14 +87,14 @@ def fix_unused_imports(self, content: str) -> str: """Remove unused imports.""" lines = content.split("\n") filtered_lines = [] - + for line in lines: if line.strip().startswith(("import ", "from ")): # Keep all imports for now - let Ruff handle specific removals filtered_lines.append(line) else: filtered_lines.append(line) - + return "\n".join(filtered_lines) def fix_trailing_whitespace(self, content: str) -> str: diff --git a/scripts/maintenance/fix_import_paths.py b/scripts/maintenance/fix_import_paths.py index 7743b243a..e4bc015a5 100644 --- a/scripts/maintenance/fix_import_paths.py +++ b/scripts/maintenance/fix_import_paths.py @@ -11,34 +11,34 @@ def fix_import_paths_in_file(file_path): try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() - + original_content = content - + # Fix common import path issues replacements = [ # Fix models imports (r'from models\.', 'from src.models.'), (r'import models\.', 'import src.models.'), - + # Fix src imports (r'from src\.src\.', 'from src.'), (r'import src\.src\.', 'import src.'), - + # Fix relative imports for moved scripts (r'from \.\.models\.', 'from src.models.'), (r'from \.\.src\.', 'from src.'), (r'from \.\.data\.', 'from data.'), - + # Fix sys.path insertions (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent / "src"\)\)', 'sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))'), (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent\.parent / "src"\)\)', 'sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))'), ] - + for pattern, replacement in replacements: content = re.sub(pattern, replacement, content) - + # Only write if content changed if content != original_content: with open(file_path, 'w', encoding='utf-8') as f: @@ -48,7 +48,7 @@ def fix_import_paths_in_file(file_path): else: print(f"No changes needed in: {file_path}") return False - + except Exception as e: print(f"Error processing {file_path}: {e}") return False @@ -56,19 +56,19 @@ def fix_import_paths_in_file(file_path): def main(): """Fix import paths in all Python files.""" print("Fixing import paths after reorganization...") - + # Get all Python files in scripts directory script_files = [] for pattern in ['scripts/**/*.py', 'src/**/*.py']: script_files.extend(glob.glob(pattern, recursive=True)) - + print(f"Found {len(script_files)} Python files to check") - + fixed_count = 0 for file_path in script_files: if fix_import_paths_in_file(file_path): fixed_count += 1 - + print(f"\nFixed import paths in {fixed_count} files") print("Import path fixes completed!") diff --git a/scripts/maintenance/fix_label_mapping.py b/scripts/maintenance/fix_label_mapping.py index a7f8fcca8..fbd9e7b98 100644 --- a/scripts/maintenance/fix_label_mapping.py +++ b/scripts/maintenance/fix_label_mapping.py @@ -29,13 +29,13 @@ def install_dependencies(): def analyze_label_mapping(): """Analyze the label mapping issue.""" print("๐Ÿ” Analyzing label mapping issue...") - + # Load datasets go_emotions = load_dataset("go_emotions", "simplified") with open('data/journal_test_dataset.json', 'r') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) - + # Analyze GoEmotions labels print("\n๐Ÿ“Š GoEmotions Analysis:") go_label_counts = {} @@ -43,36 +43,36 @@ def analyze_label_mapping(): if example['labels']: for label in example['labels']: go_label_counts[label] = go_label_counts.get(label, 0) + 1 - + print(f"GoEmotions unique labels: {len(go_label_counts)}") print(f"GoEmotions labels: {sorted(list(go_label_counts.keys()))}") print(f"Top 10 GoEmotions labels: {dict(sorted(go_label_counts.items(), key=lambda x: x[1], reverse=True)[:10])}") - + # Analyze Journal labels print("\n๐Ÿ“Š Journal Analysis:") journal_label_counts = journal_df['emotion'].value_counts().to_dict() print(f"Journal unique labels: {len(journal_label_counts)}") print(f"Journal labels: {sorted(list(journal_label_counts.keys()))}") print(f"Journal label counts: {journal_label_counts}") - + # Check for any common labels go_labels_set = set(go_label_counts.keys()) journal_labels_set = set(journal_label_counts.keys()) common_labels = go_labels_set.intersection(journal_labels_set) - + print(f"\n๐Ÿ” Common labels: {len(common_labels)}") if common_labels: print(f"Common labels: {sorted(list(common_labels))}") else: print("โŒ NO COMMON LABELS FOUND!") print("This is why we get 0 GoEmotions samples!") - + return go_label_counts, journal_label_counts def create_emotion_mapping(): """Create a mapping between GoEmotions and Journal emotions.""" print("\n๐Ÿ”ง Creating emotion mapping...") - + # GoEmotions emotion labels (from their documentation) go_emotions_mapping = { 'admiration': 'admiration', @@ -104,13 +104,13 @@ def create_emotion_mapping(): 'surprise': 'excited', 'neutral': 'calm' } - + print(f"Created mapping with {len(go_emotions_mapping)} emotions") return go_emotions_mapping def create_fixed_bulletproof_cell(): """Create a fixed bulletproof cell with proper emotion mapping.""" - + cell_code = '''# ๐Ÿš€ BULLETPROOF TRAINING CELL - FIXED LABEL MAPPING # Runtime โ†’ Change runtime type โ†’ GPU (T4 or V100) # Kernel โ†’ Restart and run all @@ -251,30 +251,30 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + # Validate data if len(texts) != len(labels): raise ValueError(f"Texts and labels have different lengths: {len(texts)} vs {len(labels)}") - + # Validate labels for i, label in enumerate(labels): if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {i}: {label}") - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] - + # Validate inputs if not isinstance(text, str) or not text.strip(): raise ValueError(f"Invalid text at index {idx}") - + if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {idx}: {label}") - + encoding = self.tokenizer( text, truncation=True, @@ -282,7 +282,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -293,33 +293,33 @@ def __getitem__(self, idx): class SimpleEmotionClassifier(nn.Module): def __init__(self, model_name="bert-base-uncased", num_labels=None): super().__init__() - + if num_labels is None or num_labels <= 0: raise ValueError(f"Invalid num_labels: {num_labels}") - + self.num_labels = num_labels self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(0.3) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + print(f"โœ… Model initialized with {num_labels} labels") - + def forward(self, input_ids, attention_mask): # Validate inputs if input_ids.dim() != 2: raise ValueError(f"Expected input_ids to be 2D, got {input_ids.dim()}D") - + if attention_mask.dim() != 2: raise ValueError(f"Expected attention_mask to be 2D, got {attention_mask.dim()}D") - + outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.pooler_output logits = self.classifier(self.dropout(pooled_output)) - + # Validate outputs if logits.shape[-1] != self.num_labels: raise ValueError(f"Expected {self.num_labels} output classes, got {logits.shape[-1]}") - + return logits # Step 7: Setup training @@ -365,12 +365,12 @@ def forward(self, input_ids, attention_mask): for epoch in range(num_epochs): print(f"\\n๐Ÿ”„ Epoch {epoch + 1}/{num_epochs}") - + # Training model.train() total_loss = 0 num_batches = 0 - + # Train on GoEmotions print(" ๐Ÿ“š Training on GoEmotions...") for i, batch in enumerate(go_loader): @@ -379,34 +379,34 @@ def forward(self, input_ids, attention_mask): if 'input_ids' not in batch or 'attention_mask' not in batch or 'labels' not in batch: print(f"โš ๏ธ Invalid batch structure at batch {i}") continue - + # Move to device with validation input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + # Validate labels if torch.any(labels >= num_labels) or torch.any(labels < 0): print(f"โš ๏ธ Invalid labels in batch {i}: {labels}") continue - + # Forward pass optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 50 == 0: print(f" Batch {i}/{len(go_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in batch {i}: {e}") continue - + # Train on journal data print(" ๐Ÿ“ Training on journal data...") for i, batch in enumerate(journal_train_loader): @@ -414,67 +414,67 @@ def forward(self, input_ids, attention_mask): input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + if torch.any(labels >= num_labels) or torch.any(labels < 0): continue - + optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 10 == 0: print(f" Batch {i}/{len(journal_train_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: print(f"โŒ Error in journal batch {i}: {e}") continue - + # Validation print(" ๐ŸŽฏ Validating...") model.eval() all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in journal_val_loader: try: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) preds = torch.argmax(outputs, dim=1) - + all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + except Exception as e: print(f"โŒ Error in validation batch: {e}") continue - + # Calculate metrics if all_preds and all_labels: f1_macro = f1_score(all_labels, all_preds, average='macro') accuracy = accuracy_score(all_labels, all_preds) - + avg_loss = total_loss / num_batches if num_batches > 0 else 0 - + print(f" ๐Ÿ“Š Epoch {epoch + 1} Results:") print(f" Average Loss: {avg_loss:.4f}") print(f" Validation F1 (Macro): {f1_macro:.4f}") print(f" Validation Accuracy: {accuracy:.4f}") - + # Save best model if f1_macro > best_f1: best_f1 = f1_macro torch.save(model.state_dict(), 'best_simple_model.pth') print(f" ๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") - + # Clear GPU cache if torch.cuda.is_available(): torch.cuda.empty_cache() @@ -505,24 +505,24 @@ def forward(self, input_ids, attention_mask): print("\\n๐ŸŽ‰ BULLETPROOF TRAINING COMPLETED!") print("๐Ÿ“ Files downloaded: best_simple_model.pth, simple_training_results.json")''' - + # Write to file with open('bulletproof_training_cell_fixed.py', 'w') as f: f.write(cell_code) - + print("โœ… Created fixed bulletproof training cell: bulletproof_training_cell_fixed.py") print("๐Ÿ“‹ This version has proper emotion mapping!") if __name__ == "__main__": # Analyze the issue go_label_counts, journal_label_counts = analyze_label_mapping() - + # Create emotion mapping emotion_mapping = create_emotion_mapping() - + # Create fixed bulletproof cell create_fixed_bulletproof_cell() - + print("\n๐ŸŽฏ SUMMARY:") print("The issue was that GoEmotions uses emotion names (like 'admiration')") print("while Journal uses different emotion names (like 'proud').") diff --git a/scripts/maintenance/fix_linting_issues_conservative.py b/scripts/maintenance/fix_linting_issues_conservative.py index a57f3b45c..20215edae 100644 --- a/scripts/maintenance/fix_linting_issues_conservative.py +++ b/scripts/maintenance/fix_linting_issues_conservative.py @@ -97,7 +97,7 @@ def fix_e402_import_order(self, content: str) -> str: result.extend(import_lines) result.append('') # Add blank line after imports result.extend(other_lines) - + return '\n'.join(result) def fix_ruf022_all_sorting(self, content: str) -> str: diff --git a/scripts/maintenance/fix_model_architecture_mismatch.py b/scripts/maintenance/fix_model_architecture_mismatch.py index bbfb75756..0df7489f2 100644 --- a/scripts/maintenance/fix_model_architecture_mismatch.py +++ b/scripts/maintenance/fix_model_architecture_mismatch.py @@ -11,11 +11,11 @@ def fix_model_architecture(): """Fix the model architecture mismatch in the minimal notebook.""" - + # Read the existing notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Find and replace the model setup cell for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'model_name =' in ''.join(cell['source']): @@ -65,11 +65,11 @@ def fix_model_architecture(): " print('โš ๏ธ CUDA not available, model will run on CPU')" ] break - + # Save the updated notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Fixed model architecture mismatch!') print('๐Ÿ“‹ Changes made:') print(' โœ… Properly reconfigured classifier layer for 12 emotions') diff --git a/scripts/maintenance/fix_model_reconfiguration.py b/scripts/maintenance/fix_model_reconfiguration.py index a3dc88310..92e90af3f 100644 --- a/scripts/maintenance/fix_model_reconfiguration.py +++ b/scripts/maintenance/fix_model_reconfiguration.py @@ -12,11 +12,11 @@ def fix_model_reconfiguration(): """Fix the model reconfiguration in the minimal notebook.""" - + # Read the existing notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Find and replace the model setup cell for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'model_name =' in ''.join(cell['source']): @@ -75,11 +75,11 @@ def fix_model_reconfiguration(): " print('โš ๏ธ CUDA not available, model will run on CPU')" ] break - + # Save the updated notebook with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Fixed model reconfiguration!') print('๐Ÿ“‹ Changes made:') print(' โœ… Created new model with correct architecture from scratch') diff --git a/scripts/maintenance/fix_remaining_linting.py b/scripts/maintenance/fix_remaining_linting.py index a877fe1a8..c191fc161 100644 --- a/scripts/maintenance/fix_remaining_linting.py +++ b/scripts/maintenance/fix_remaining_linting.py @@ -71,7 +71,7 @@ def fix_file(self, file_path: str) -> bool: if content != original_content: with open(file_path, 'w', encoding='utf-8') as f: f.write(content) - + self.fixed_files.append(file_path) self.total_fixes += fixes_applied print(f" โœ… Fixed {fixes_applied} issues") @@ -86,13 +86,13 @@ def fix_file(self, file_path: str) -> bool: def fix_undefined_names(self, content: str) -> tuple[str, int]: """Fix F821: Undefined name errors.""" fixes = 0 - + patterns = [ (r'for ___(\w+) in (\w+):', r'for \1 in \2:'), (r'except Exception as e:', r'except Exception as e:'), (r'f"([^"]*)\{(\w+)\}([^"]*)"', r'f"\1{\2}\3"'), ] - + for pattern, replacement in patterns: new_content = re.sub(pattern, replacement, content) if new_content != content: @@ -104,11 +104,11 @@ def fix_undefined_names(self, content: str) -> tuple[str, int]: def fix_import_sorting(self, content: str) -> tuple[str, int]: """Fix S-series: Import sorting issues.""" fixes = 0 - + lines = content.split('\n') import_lines = [] non_import_lines = [] - + for line in lines: stripped = line.strip() if (stripped.startswith('import ') or @@ -117,25 +117,25 @@ def fix_import_sorting(self, content: str) -> tuple[str, int]: import_lines.append(line) else: non_import_lines.append(line) - + import_lines.sort() - + new_content = '\n'.join(import_lines + non_import_lines) if new_content != content: fixes += 1 - + return new_content, fixes def fix_path_issues(self, content: str) -> tuple[str, int]: """Fix P-series: Path issues.""" fixes = 0 - + patterns = [ (r'os\.path\.abspath\(', r'Path('), (r'os\.path\.join\(', r'Path('), (r'os\.path\.exists\(', r'Path('), ] - + for pattern, replacement in patterns: new_content = re.sub(pattern, replacement, content) if new_content != content: @@ -147,10 +147,10 @@ def fix_path_issues(self, content: str) -> tuple[str, int]: def fix_logging_issues(self, content: str) -> tuple[str, int]: """Fix G003: Logging issues.""" fixes = 0 - + pattern = r'logging\.(info|debug|warning|error)\("([^"]*)" \+ "([^"]*)"' replacement = r'logging.\1(f"\2\3"' - + new_content = re.sub(pattern, replacement, content) if new_content != content: content = new_content @@ -161,10 +161,10 @@ def fix_logging_issues(self, content: str) -> tuple[str, int]: def fix_loop_variables(self, content: str) -> tuple[str, int]: """Fix B007: Loop control variable issues.""" fixes = 0 - + pattern = r'for (\w+), (\w+) in enumerate\((\w+)\):' replacement = r'for _\1, \2 in enumerate(\3):' - + new_content = re.sub(pattern, replacement, content) if new_content != content: content = new_content @@ -175,10 +175,10 @@ def fix_loop_variables(self, content: str) -> tuple[str, int]: def fix_minor_issues(self, content: str) -> tuple[str, int]: """Fix other minor issues.""" fixes = 0 - + pattern = r'TEST_USER_PASSWORD_HASH = "test_hashed_password_123" # noqa: S105]*)"' replacement = r'TEST_USER_PASSWORD_HASH = "test_hashed_password_123" # noqa: S105 # noqa: S105' - + new_content = re.sub(pattern, replacement, content) if new_content != content: content = new_content @@ -189,7 +189,7 @@ def fix_minor_issues(self, content: str) -> tuple[str, int]: def process_directory(self, directory: str) -> None: """Process all Python files in a directory.""" print(f"\n๐Ÿ”ง Processing directory: {directory}") - + for file_path in Path(directory).rglob("*.py"): if file_path.is_file(): print(f" ๐Ÿ“ {file_path}") @@ -199,18 +199,18 @@ def run(self) -> None: """Run the comprehensive linting fix.""" print("๐Ÿš€ Starting Comprehensive Linting Fix...") print("=" * 60) - + directories = ["src", "tests", "scripts"] - + for directory in directories: if Path(directory): self.process_directory(directory) - + print("\n" + "=" * 60) print("๐ŸŽ‰ COMPREHENSIVE LINTING FIX COMPLETE!") print(f"๐Ÿ“Š Files fixed: {len(self.fixed_files)}") print(f"๐Ÿ”ง Total fixes applied: {self.total_fixes}") - + if self.fixed_files: print("\nโœ… Fixed files:") for file_path in self.fixed_files: diff --git a/scripts/maintenance/quick_label_fix.py b/scripts/maintenance/quick_label_fix.py index 8fab9044a..6b875e90b 100644 --- a/scripts/maintenance/quick_label_fix.py +++ b/scripts/maintenance/quick_label_fix.py @@ -13,43 +13,43 @@ def quick_label_fix(): """Quick fix for label mismatch issues.""" print("๐Ÿ”ง Applying quick label fix...") - + # Load datasets go_emotions = load_dataset("go_emotions", "simplified") - + with open('data/journal_test_dataset.json', 'r') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) - + # Get all unique labels go_labels = set() for example in go_emotions['train']: if example['labels']: go_labels.update(example['labels']) - + journal_labels = set(journal_df['emotion'].unique()) - + # Use only common labels to avoid mismatches common_labels = sorted(list(go_labels.intersection(journal_labels))) - + if not common_labels: print("โš ๏ธ No common labels found! Using all labels...") common_labels = sorted(list(go_labels.union(journal_labels))) - + print(f"๐Ÿ“Š Using {len(common_labels)} labels: {common_labels}") - + # Create label encoder label_encoder = LabelEncoder() label_encoder.fit(common_labels) - + # Create mappings label_to_id = {label: idx for idx, label in enumerate(label_encoder.classes_)} id_to_label = {idx: label for label, idx in label_to_id.items()} - + # Save fixed encoder with open('fixed_label_encoder.pkl', 'wb') as f: pickle.dump(label_encoder, f) - + # Save mappings with open('label_mappings.json', 'w') as f: json.dump({ @@ -58,12 +58,12 @@ def quick_label_fix(): 'num_labels': len(label_encoder.classes_), 'classes': label_encoder.classes_.tolist() }, f, indent=2) - + print(f"โœ… Fixed label encoder saved!") print(f"๐Ÿ“Š Use num_labels={len(label_encoder.classes_)} in your model") print(f"๐Ÿ“Š Label encoder: fixed_label_encoder.pkl") print(f"๐Ÿ“Š Mappings: label_mappings.json") - + return len(label_encoder.classes_) if __name__ == "__main__": diff --git a/scripts/testing/check_model_health.py b/scripts/testing/check_model_health.py index a598c194c..163427694 100755 --- a/scripts/testing/check_model_health.py +++ b/scripts/testing/check_model_health.py @@ -15,11 +15,11 @@ def check_model_health(base_url=None): if base_url: config.base_url = base_url.rstrip('/') client = create_api_client() - + print("๐Ÿ” Model Health Check") print("=" * 30) print(f"Testing URL: {config.base_url}") - + # Test health endpoint try: data = client.get("/") @@ -41,7 +41,7 @@ def check_model_health(base_url=None): try: payload = {"text": "I am happy"} data = client.post("/predict", payload) - + # Handle confidence formatting with null checks primary_emotion = data.get('primary_emotion', {}) emotion = primary_emotion.get('emotion', 'Unknown') @@ -50,10 +50,10 @@ def check_model_health(base_url=None): confidence_str = f"{confidence:.3f}" else: confidence_str = "N/A" - + print(f"โœ… Prediction: {emotion} (confidence: {confidence_str})") return True - + except requests.exceptions.RequestException as e: print(f"โŒ Prediction check error: {e}") return False @@ -64,10 +64,10 @@ def check_model_health(base_url=None): if __name__ == "__main__": import argparse - + parser = argparse.ArgumentParser(description="Check Model Health") parser.add_argument("--base-url", help="API base URL") args = parser.parse_args() - + success = check_model_health(args.base_url) exit(0 if success else 1) diff --git a/scripts/testing/create_journal_test_dataset.py b/scripts/testing/create_journal_test_dataset.py index 7c31216a6..4e90c8aa5 100644 --- a/scripts/testing/create_journal_test_dataset.py +++ b/scripts/testing/create_journal_test_dataset.py @@ -169,30 +169,30 @@ def generate_journal_content(topic: str, emotion: str) -> str: template = random.choice(JOURNAL_TEMPLATES) emotion_context = random.choice(EMOTION_CONTEXTS.get(emotion, ["I'm feeling this way."])) reflection = random.choice(REFLECTIVE_STATEMENTS) - + content = template.format( topic=topic, emotion=emotion, emotion_context=emotion_context, reflection=reflection ) - + # Add more depth with additional sentences if random.random() > 0.3: # 70% chance of adding more detail additional_context = random.choice(EMOTION_CONTEXTS.get(emotion, ["I'm processing this."])) content += f" {additional_context}" - + if random.random() > 0.5: # 50% chance of adding another reflection second_reflection = random.choice(REFLECTIVE_STATEMENTS) content += f" {second_reflection}" - + return content def generate_journal_entry(entry_id: int, user_id: int, created_at: datetime) -> Dict[str, Any]: """Generate a single realistic journal entry.""" topic = random.choice(JOURNAL_TOPICS) emotion = random.choice(list(EMOTION_CONTEXTS.keys())) - + return { "id": entry_id, "user_id": user_id, @@ -215,40 +215,40 @@ def create_journal_test_dataset( """Create a comprehensive journal test dataset.""" start_date = datetime.now(timezone.utc) - timedelta(days=days_back) end_date = datetime.now(timezone.utc) - + entries = [] for i in range(num_entries): user_id = random.randint(1, num_users) - + # Random date within the range days_offset = random.randint(0, days_back) entry_date = start_date + timedelta(days=days_offset) - + # Random time during the day (more realistic for journaling) entry_date = entry_date.replace( hour=random.randint(6, 23), # Early morning to late night minute=random.randint(0, 59), second=random.randint(0, 59), ) - + entry = generate_journal_entry(i + 1, user_id, entry_date) entries.append(entry) - + return entries def save_test_dataset(entries: List[Dict[str, Any]], output_path: str) -> None: """Save the test dataset to JSON.""" Path(output_path).parent.mkdir(parents=True, exist_ok=True) - + with open(output_path, 'w') as f: json.dump(entries, f, indent=2) - + print(f"โœ… Saved {len(entries)} journal entries to {output_path}") def create_dataset_summary(entries: List[Dict[str, Any]]) -> Dict[str, Any]: """Create a summary of the dataset for validation.""" df = pd.DataFrame(entries) - + summary = { "total_entries": len(entries), "unique_users": df["user_id"].nunique(), @@ -261,46 +261,46 @@ def create_dataset_summary(entries: List[Dict[str, Any]]) -> Dict[str, Any]: }, "sample_entries": entries[:3] # First 3 entries as examples } - + return summary def main(): """Main function to create the journal test dataset.""" print("๐Ÿš€ Creating Journal Entry Test Dataset for Domain Adaptation") print("=" * 60) - + # Create the dataset entries = create_journal_test_dataset( num_entries=150, # Exceeds the 100+ requirement num_users=10, days_back=90 ) - + # Save to data directory output_path = "data/journal_test_dataset.json" save_test_dataset(entries, output_path) - + # Create and save summary summary = create_dataset_summary(entries) summary_path = "data/journal_test_dataset_summary.json" - + with open(summary_path, 'w') as f: json.dump(summary, f, indent=2) - + print(f"โœ… Saved dataset summary to {summary_path}") - + # Print key statistics print("\n๐Ÿ“Š Dataset Statistics:") print(f" Total Entries: {summary['total_entries']}") print(f" Unique Users: {summary['unique_users']}") print(f" Average Word Count: {summary['avg_word_count']:.1f}") print(f" Date Range: {summary['date_range']['start'][:10]} to {summary['date_range']['end'][:10]}") - + print("\n๐ŸŽฏ Emotion Distribution:") for emotion, count in summary['emotion_distribution'].items(): percentage = (count / summary['total_entries']) * 100 print(f" {emotion}: {count} ({percentage:.1f}%)") - + print("\nโœ… Journal Test Dataset Created Successfully!") print(" This dataset will be used for REQ-DL-012 domain adaptation testing") print(" Target: 70% F1 score on journal-style text vs Reddit comments") diff --git a/scripts/testing/debug_dataset_structure.py b/scripts/testing/debug_dataset_structure.py index 8aad21f73..86dc0fcd3 100644 --- a/scripts/testing/debug_dataset_structure.py +++ b/scripts/testing/debug_dataset_structure.py @@ -63,7 +63,7 @@ def debug_dataset_structure(): # Check if it's a HuggingFace dataset if hasattr(test_data, 'features'): logger.info(f"๐Ÿ“Š Dataset features: {test_data.features}") - + if hasattr(test_data, 'column_names'): logger.info(f"๐Ÿ“Š Dataset columns: {test_data.column_names}") diff --git a/scripts/testing/debug_go_emotions_labels.py b/scripts/testing/debug_go_emotions_labels.py index c07515eb5..3028966c1 100644 --- a/scripts/testing/debug_go_emotions_labels.py +++ b/scripts/testing/debug_go_emotions_labels.py @@ -27,16 +27,16 @@ def install_dependencies(): def debug_go_emotions(): """Debug the actual GoEmotions dataset structure.""" print("๐Ÿ” Debugging GoEmotions dataset structure...") - + # Load the dataset go_emotions = load_dataset("go_emotions", "simplified") - + print(f"\n๐Ÿ“Š Dataset structure:") print(f"Keys: {list(go_emotions.keys())}") print(f"Train size: {len(go_emotions['train'])}") print(f"Validation size: {len(go_emotions['validation'])}") print(f"Test size: {len(go_emotions['test'])}") - + # Check first few examples print(f"\n๐Ÿ“Š First 5 examples:") for i in range(min(5, len(go_emotions['train']))): @@ -46,58 +46,58 @@ def debug_go_emotions(): print(f" Labels: {example['labels']}") print(f" Label types: {[type(label) for label in example['labels']]}") print() - + # Check if there's a label mapping print(f"\n๐Ÿ” Checking for label mapping...") - + # Try to get the dataset info try: dataset_info = go_emotions['train'].info print(f"Dataset info: {dataset_info}") except: print("No dataset info available") - + # Check if there are features try: features = go_emotions['train'].features print(f"Features: {features}") except: print("No features available") - + # Look for label names in the dataset print(f"\n๐Ÿ” Looking for label names...") - + # Check if there's a label_names field if hasattr(go_emotions, 'label_names'): print(f"Label names: {go_emotions.label_names}") else: print("No label_names attribute") - + # Check if there's a features attribute with label names if hasattr(go_emotions['train'], 'features'): features = go_emotions['train'].features print(f"Features: {features}") if 'labels' in features: print(f"Labels feature: {features['labels']}") - + # Try to get the original dataset print(f"\n๐Ÿ” Trying original dataset...") try: original_go_emotions = load_dataset("go_emotions") print(f"Original dataset keys: {list(original_go_emotions.keys())}") - + if 'train' in original_go_emotions: print(f"Original train size: {len(original_go_emotions['train'])}") example = original_go_emotions['train'][0] print(f"Original example: {example}") except Exception as e: print(f"Could not load original dataset: {e}") - + # Check the dataset card print(f"\n๐Ÿ” Checking dataset documentation...") print("GoEmotions dataset should have emotion names like:") print("['admiration', 'amusement', 'anger', 'annoyance', 'approval', 'caring', 'confusion', 'curiosity', 'desire', 'disappointment', 'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear', 'gratitude', 'grief', 'joy', 'love', 'nervousness', 'optimism', 'pride', 'realization', 'relief', 'remorse', 'sadness', 'surprise', 'neutral']") - + return go_emotions if __name__ == "__main__": diff --git a/scripts/testing/debug_label_mismatch.py b/scripts/testing/debug_label_mismatch.py index 23ddc4daa..9330808da 100644 --- a/scripts/testing/debug_label_mismatch.py +++ b/scripts/testing/debug_label_mismatch.py @@ -16,65 +16,65 @@ def debug_label_mismatch(): """Debug the label mismatch causing CUDA errors.""" logger.info("๐Ÿ” Debugging label mismatch issue...") - + try: # Step 1: Load datasets logger.info("๐Ÿ“Š Loading datasets...") - + # Load GoEmotions dataset go_emotions = load_dataset("go_emotions", "simplified") logger.info(f"โœ… GoEmotions loaded: {len(go_emotions['train'])} training examples") - + # Load journal dataset with open('data/journal_test_dataset.json', 'r') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) logger.info(f"โœ… Journal dataset loaded: {len(journal_df)} entries") - + # Step 2: Analyze GoEmotions labels logger.info("๐Ÿ” Analyzing GoEmotions labels...") go_labels = set() go_label_counts = {} - + for example in go_emotions['train']: if example['labels']: for label in example['labels']: go_labels.add(label) go_label_counts[label] = go_label_counts.get(label, 0) + 1 - + logger.info(f"๐Ÿ“Š GoEmotions unique labels: {len(go_labels)}") logger.info(f"๐Ÿ“Š GoEmotions labels: {sorted(list(go_labels))}") logger.info(f"๐Ÿ“Š GoEmotions label counts: {dict(sorted(go_label_counts.items(), key=lambda x: x[1], reverse=True)[:10])}") - + # Step 3: Analyze journal labels logger.info("๐Ÿ” Analyzing journal labels...") journal_labels = set(journal_df['emotion'].unique()) journal_label_counts = journal_df['emotion'].value_counts().to_dict() - + logger.info(f"๐Ÿ“Š Journal unique labels: {len(journal_labels)}") logger.info(f"๐Ÿ“Š Journal labels: {sorted(list(journal_labels))}") logger.info(f"๐Ÿ“Š Journal label counts: {journal_label_counts}") - + # Step 4: Check for label mismatches logger.info("๐Ÿ” Checking for label mismatches...") - + # Find labels that exist in one dataset but not the other go_only = go_labels - journal_labels journal_only = journal_labels - go_labels common_labels = go_labels.intersection(journal_labels) - + logger.info(f"๐Ÿ“Š Labels only in GoEmotions: {sorted(list(go_only))}") logger.info(f"๐Ÿ“Š Labels only in Journal: {sorted(list(journal_only))}") logger.info(f"๐Ÿ“Š Common labels: {sorted(list(common_labels))}") - + if go_only: logger.warning(f"โš ๏ธ {len(go_only)} labels only in GoEmotions - may cause issues") if journal_only: logger.warning(f"โš ๏ธ {len(journal_only)} labels only in Journal - may cause issues") - + # Step 5: Create unified label encoder logger.info("๐Ÿงฌ Creating unified label encoder...") - + # Option 1: Use only common labels (safer) if len(common_labels) > 0: all_labels = sorted(list(common_labels)) @@ -83,21 +83,21 @@ def debug_label_mismatch(): # Option 2: Use all labels (may cause issues) all_labels = sorted(list(go_labels.union(journal_labels))) logger.warning(f"โš ๏ธ No common labels found! Using all labels: {len(all_labels)}") - + label_encoder = LabelEncoder() label_encoder.fit(all_labels) num_labels = len(label_encoder.classes_) - + logger.info(f"๐Ÿ“Š Final num_labels: {num_labels}") logger.info(f"๐Ÿ“Š Encoded classes: {label_encoder.classes_}") - + # Step 6: Test label encoding logger.info("๐Ÿงช Testing label encoding...") - + # Test GoEmotions encoding go_encoded = [] go_encoding_errors = [] - + for i, example in enumerate(go_emotions['train'][:100]): # Test first 100 if example['labels']: try: @@ -110,11 +110,11 @@ def debug_label_mismatch(): go_encoding_errors.append(f"Label '{label}' not in encoder classes") except Exception as e: go_encoding_errors.append(f"Error encoding label '{label}': {e}") - + # Test journal encoding journal_encoded = [] journal_encoding_errors = [] - + for i, emotion in enumerate(journal_df['emotion'][:100]): # Test first 100 try: if emotion in label_encoder.classes_: @@ -124,7 +124,7 @@ def debug_label_mismatch(): journal_encoding_errors.append(f"Label '{emotion}' not in encoder classes") except Exception as e: journal_encoding_errors.append(f"Error encoding label '{emotion}': {e}") - + # Report encoding results if go_encoded: logger.info(f"โœ… GoEmotions encoding successful: {len(go_encoded)} samples") @@ -133,7 +133,7 @@ def debug_label_mismatch(): logger.error(f"โŒ GoEmotions encoding errors: {len(go_encoding_errors)}") for error in go_encoding_errors[:5]: # Show first 5 errors logger.error(f" - {error}") - + if journal_encoded: logger.info(f"โœ… Journal encoding successful: {len(journal_encoded)} samples") logger.info(f"๐Ÿ“Š Journal label range: {min(journal_encoded)} to {max(journal_encoded)}") @@ -141,30 +141,30 @@ def debug_label_mismatch(): logger.error(f"โŒ Journal encoding errors: {len(journal_encoding_errors)}") for error in journal_encoding_errors[:5]: # Show first 5 errors logger.error(f" - {error}") - + # Step 7: Validate label ranges logger.info("๐Ÿ” Validating label ranges...") - + expected_range = list(range(num_labels)) go_range = list(range(min(go_encoded), max(go_encoded) + 1)) if go_encoded else [] journal_range = list(range(min(journal_encoded), max(journal_encoded) + 1)) if journal_encoded else [] - + logger.info(f"๐Ÿ“Š Expected range: {expected_range}") logger.info(f"๐Ÿ“Š GoEmotions range: {go_range}") logger.info(f"๐Ÿ“Š Journal range: {journal_range}") - + # Check for out-of-bounds labels go_out_of_bounds = [label for label in go_encoded if label < 0 or label >= num_labels] journal_out_of_bounds = [label for label in journal_encoded if label < 0 or label >= num_labels] - + if go_out_of_bounds: logger.error(f"โŒ GoEmotions has {len(go_out_of_bounds)} out-of-bounds labels") if journal_out_of_bounds: logger.error(f"โŒ Journal has {len(journal_out_of_bounds)} out-of-bounds labels") - + # Step 8: Provide recommendations logger.info("๐Ÿ’ก Recommendations:") - + if go_encoding_errors or journal_encoding_errors: logger.info("1. ๐Ÿ”ง Use only common labels between datasets") logger.info("2. ๐Ÿ”ง Filter out samples with non-common labels") @@ -172,19 +172,19 @@ def debug_label_mismatch(): else: logger.info("1. โœ… Label encoding looks good!") logger.info("2. โœ… Proceed with training using the unified label encoder") - + # Step 9: Create fixed label encoder logger.info("๐Ÿ”ง Creating fixed label encoder...") - + # Save the working label encoder import pickle with open('fixed_label_encoder.pkl', 'wb') as f: pickle.dump(label_encoder, f) - + # Create label mappings label_to_id = {label: idx for idx, label in enumerate(label_encoder.classes_)} id_to_label = {idx: label for label, idx in label_to_id.items()} - + # Save mappings with open('label_mappings.json', 'w') as f: json.dump({ @@ -193,11 +193,11 @@ def debug_label_mismatch(): 'num_labels': num_labels, 'classes': label_encoder.classes_.tolist() }, f, indent=2) - + logger.info("โœ… Fixed label encoder saved:") logger.info(" - fixed_label_encoder.pkl") logger.info(" - label_mappings.json") - + return { 'num_labels': num_labels, 'label_encoder': label_encoder, @@ -206,7 +206,7 @@ def debug_label_mismatch(): 'go_encoding_errors': len(go_encoding_errors), 'journal_encoding_errors': len(journal_encoding_errors) } - + except Exception as e: logger.error(f"โŒ Debugging failed: {e}") return None diff --git a/scripts/testing/debug_model_loading.py b/scripts/testing/debug_model_loading.py index b44fa92ee..8b35af688 100644 --- a/scripts/testing/debug_model_loading.py +++ b/scripts/testing/debug_model_loading.py @@ -15,12 +15,12 @@ def debug_model_loading(): """Debug the model loading issues""" config = create_test_config() client = create_api_client() - + print("๐Ÿ” Debugging Model Loading Issues") print("=" * 50) print(f"Testing URL: {config.base_url}") print(f"API Key: {config.api_key[:20]}...") - + # Test model status with API key print("\n1. Testing model status with API key...") try: @@ -31,7 +31,7 @@ def debug_model_loading(): print(" ๐Ÿ” Unauthorized - API key mismatch") else: print(f" โŒ Model status error: {e}") - + # Test security status print("\n2. Testing security status...") try: @@ -50,7 +50,7 @@ def debug_model_loading(): print(f" โŒ Prediction error: {e}") except ValueError as e: print(f" โŒ Invalid response format: {e}") - + # Test batch prediction print("\n4. Testing batch prediction...") try: @@ -61,7 +61,7 @@ def debug_model_loading(): print(f" โŒ Batch prediction error: {e}") except ValueError as e: print(f" โŒ Invalid response format: {e}") - + # Test with different input formats print("\n5. Testing different input formats...") test_cases = [ diff --git a/scripts/testing/debug_rate_limiter_test.py b/scripts/testing/debug_rate_limiter_test.py index 0519ecba6..e69de29bb 100644 --- a/scripts/testing/debug_rate_limiter_test.py +++ b/scripts/testing/debug_rate_limiter_test.py @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/scripts/testing/final_temperature_test.py b/scripts/testing/final_temperature_test.py index e1ae8d786..b0f2047bc 100644 --- a/scripts/testing/final_temperature_test.py +++ b/scripts/testing/final_temperature_test.py @@ -77,10 +77,10 @@ def final_temperature_test(): # Create simple test data logging.info("๐Ÿ“ Creating test data...") - + # Create emotion labels (simplified for testing) emotion_labels = ["joy", "sadness", "anger", "fear"] - + # Create simple test data test_texts = [ "I am so happy today!", @@ -92,7 +92,7 @@ def final_temperature_test(): "I'm furious with you!", "I'm terrified of the dark." ] - + test_labels = [ [1, 0, 0, 0], # joy [0, 1, 0, 0], # sadness @@ -106,53 +106,53 @@ def final_temperature_test(): # Create tokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") - + # Create dataset dataset = EmotionDataset(test_texts, test_labels, tokenizer, max_length=128) dataloader = DataLoader(dataset, batch_size=4, shuffle=False) # Test different temperatures temperatures = [0.5, 1.0, 1.5, 2.0] - + logging.info("๐Ÿงช Testing temperature scaling...") - + for temp in temperatures: logging.info(f"\n๐ŸŒก๏ธ Temperature: {temp}") - + # Set temperature model.temperature = temp - + all_predictions = [] all_labels = [] - + with torch.no_grad(): for batch in dataloader: input_ids = batch["input_ids"].to(device) attention_mask = batch["attention_mask"].to(device) labels = batch["labels"].to(device) - + # Run evaluation outputs = model(input_ids, attention_mask) probabilities = torch.sigmoid(outputs / temp) - + # Apply threshold predictions = (probabilities > 0.5).float() - + # Convert to numpy for sklearn all_predictions.append(predictions.cpu().numpy()) all_labels.append(labels.cpu().numpy()) - + # Concatenate results all_predictions = np.concatenate(all_predictions, axis=0) all_labels = np.concatenate(all_labels, axis=0) - + # Calculate metrics micro_f1 = f1_score(all_labels, all_predictions, average='micro', zero_division=0) macro_f1 = f1_score(all_labels, all_predictions, average='macro', zero_division=0) - + logging.info(f" Micro F1: {micro_f1:.4f}") logging.info(f" Macro F1: {macro_f1:.4f}") - + # Show some predictions logging.info(" Sample predictions:") for i in range(min(3, len(test_texts))): @@ -162,7 +162,7 @@ def final_temperature_test(): logging.info(f" Predicted: {pred_emotions}") logging.info(f" True: {true_emotions}") logging.info(f" Raw probs: {probabilities[i].cpu().numpy()}") - + logging.info("โœ… Temperature scaling test completed!") diff --git a/scripts/testing/mega_comprehensive_model_test.py b/scripts/testing/mega_comprehensive_model_test.py index 7ae040331..01691edec 100644 --- a/scripts/testing/mega_comprehensive_model_test.py +++ b/scripts/testing/mega_comprehensive_model_test.py @@ -20,13 +20,13 @@ class MegaComprehensiveModelTester: """Mega comprehensive model testing framework.""" - + def __init__(self, model_path="deployment/models/default"): self.model_path = model_path self.tokenizer = None self.model = None self.emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - + # Test results storage self.test_results = { 'basic_tests': {}, @@ -39,44 +39,44 @@ def __init__(self, model_path="deployment/models/default"): 'confidence_analysis': {}, 'error_analysis': {} } - + def load_model(self): """Load the model and tokenizer.""" print("๐Ÿ”ง LOADING MODEL FOR MEGA TESTING") print("=" * 60) - + try: self.tokenizer = AutoTokenizer.from_pretrained(self.model_path) self.model = AutoModelForSequenceClassification.from_pretrained(self.model_path) - + if torch.cuda.is_available(): self.model = self.model.to('cuda') print("โœ… Model moved to GPU") else: print("โš ๏ธ CUDA not available, using CPU") - + print("โœ… Model loaded successfully for mega testing") return True - + except Exception as e: print(f"โŒ Failed to load model: {e}") return False - + def predict_emotion(self, text): """Make a prediction with confidence.""" inputs = self.tokenizer(text, return_tensors='pt', truncation=True, padding=True) if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + # Get all probabilities for analysis all_probs = probabilities[0].cpu().numpy() - + # Get predicted emotion name if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] @@ -84,14 +84,14 @@ def predict_emotion(self, text): predicted_emotion = self.model.config.id2label[str(predicted_label)] else: predicted_emotion = f"unknown_{predicted_label}" - + return predicted_emotion, confidence, all_probs - + def test_basic_functionality(self): """Test basic model functionality.""" print("\n๐Ÿงช BASIC FUNCTIONALITY TESTS") print("=" * 60) - + basic_test_cases = [ # Direct emotion statements ("I am happy", "happy"), @@ -106,7 +106,7 @@ def test_basic_functionality(self): ("I feel overwhelmed", "overwhelmed"), ("I am proud", "proud"), ("I feel tired", "tired"), - + # With context ("I am happy today", "happy"), ("I feel sad about the news", "sad"), @@ -121,101 +121,101 @@ def test_basic_functionality(self): ("I am proud of my work", "proud"), ("I feel tired after exercise", "tired") ] - + correct = 0 confidences = [] - + for i, (text, expected) in enumerate(basic_test_cases, 1): predicted, confidence, _ = self.predict_emotion(text) is_correct = predicted == expected if is_correct: correct += 1 confidences.append(confidence) - + status = "โœ…" if is_correct else "โŒ" print(f"{status} {i:2d}. \"{text}\" โ†’ {predicted} (expected: {expected}) [conf: {confidence:.3f}]") - + accuracy = correct / len(basic_test_cases) * 100 avg_confidence = np.mean(confidences) - + self.test_results['basic_tests'] = { 'accuracy': accuracy, 'avg_confidence': avg_confidence, 'total_tests': len(basic_test_cases), 'correct': correct } - + print(f"\n๐Ÿ“Š Basic Test Results: {accuracy:.2f}% accuracy, {avg_confidence:.3f} avg confidence") - + def test_edge_cases(self): """Test edge cases and unusual inputs.""" print("\n๐Ÿ” EDGE CASES AND UNUSUAL INPUTS") print("=" * 60) - + edge_cases = [ # Very short inputs ("Happy", "happy"), ("Sad", "sad"), ("Excited!", "excited"), ("Anxious?", "anxious"), - + # Very long inputs ("I am feeling incredibly happy and joyful and ecstatic and delighted and pleased and satisfied and content and cheerful and glad and thrilled and overjoyed and elated and jubilant and euphoric and blissful and radiant and beaming and glowing and sparkling and wonderful", "happy"), - + # Mixed emotions ("I am happy but also a bit sad", "happy"), # Should pick dominant emotion ("I feel excited yet anxious", "excited"), ("I am grateful but tired", "grateful"), - + # Ambiguous cases ("I feel okay", "content"), # Neutral should map to content ("I am fine", "content"), ("Not bad", "content"), - + # Intensifiers ("I am EXTREMELY happy", "happy"), ("I feel SO sad", "sad"), ("I am REALLY excited", "excited"), ("I feel VERY anxious", "anxious"), - + # Negations ("I am not happy", "sad"), # Should detect negative emotion ("I don't feel excited", "content"), ("I am not calm", "anxious"), - + # Questions ("Am I happy?", "happy"), ("Why am I sad?", "sad"), ("Should I be excited?", "excited"), - + # Emojis and symbols ("I am happy ๐Ÿ˜Š", "happy"), ("I feel sad :(", "sad"), ("I am excited!!!", "excited"), ("I feel anxious...", "anxious"), - + # Capitalization variations ("I AM HAPPY", "happy"), ("i am sad", "sad"), ("I Am Excited", "excited"), ("i FEEL anxious", "anxious"), - + # Repetition ("Happy happy happy", "happy"), ("Sad sad sad sad", "sad"), ("Excited excited", "excited"), - + # Numbers and special characters ("I am happy 123", "happy"), ("I feel sad @#$%", "sad"), ("I am excited (really!)", "excited"), - + # Empty or minimal ("", "content"), # Should default to something (" ", "content"), ("...", "content") ] - + results = [] for text, expected in edge_cases: predicted, confidence, _ = self.predict_emotion(text) @@ -227,11 +227,11 @@ def test_edge_cases(self): 'confidence': confidence, 'correct': is_correct }) - + correct = sum(1 for r in results if r['correct']) accuracy = correct / len(results) * 100 avg_confidence = np.mean([r['confidence'] for r in results]) - + self.test_results['edge_cases'] = { 'accuracy': accuracy, 'avg_confidence': avg_confidence, @@ -239,28 +239,28 @@ def test_edge_cases(self): 'correct': correct, 'details': results } - + print(f"๐Ÿ“Š Edge Case Results: {accuracy:.2f}% accuracy, {avg_confidence:.3f} avg confidence") print(f" Correct: {correct}/{len(results)}") - + def test_stress_conditions(self): """Test model under stress conditions.""" print("\n๐Ÿ’ช STRESS TESTS") print("=" * 60) - + # Generate random noise text random_texts = [] for _ in range(20): words = ['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'] random_text = ' '.join(random.choices(words, k=random.randint(5, 15))) random_texts.append(random_text) - + # Generate very long texts long_texts = [] for _ in range(10): long_text = "I am feeling " + "very " * random.randint(10, 30) + "happy today because " + "of many reasons " * random.randint(5, 15) long_texts.append(long_text) - + # Generate texts with special characters special_char_texts = [ "I am happy @#$%^&*()", @@ -272,9 +272,9 @@ def test_stress_conditions(self): "I am proud ๐ŸŽ‰๐ŸŽŠ๐ŸŽˆ๐ŸŽ‚๐ŸŽ", "I feel tired ๐Ÿ’ค๐Ÿ˜ด๐Ÿ›๏ธ" ] - + all_stress_tests = random_texts + long_texts + special_char_texts - + results = [] for text in all_stress_tests: try: @@ -293,10 +293,10 @@ def test_stress_conditions(self): 'success': False, 'error': str(e) }) - + successful = sum(1 for r in results if r['success']) avg_confidence = np.mean([r['confidence'] for r in results if r['success']]) - + self.test_results['stress_tests'] = { 'success_rate': successful / len(results) * 100, 'avg_confidence': avg_confidence, @@ -304,15 +304,15 @@ def test_stress_conditions(self): 'successful': successful, 'details': results } - + print(f"๐Ÿ“Š Stress Test Results: {successful/len(results)*100:.2f}% success rate, {avg_confidence:.3f} avg confidence") print(f" Successful: {successful}/{len(results)}") - + def test_bias_analysis(self): """Analyze model for bias across different inputs.""" print("\nโš–๏ธ BIAS ANALYSIS") print("=" * 60) - + # Test with different sentence structures structures = [ "I am {emotion}", @@ -326,9 +326,9 @@ def test_bias_analysis(self): "I am so {emotion}", "I am really {emotion}" ] - + bias_results = defaultdict(list) - + for structure in structures: for emotion in self.emotions: text = structure.format(emotion=emotion) @@ -340,32 +340,32 @@ def test_bias_analysis(self): 'confidence': confidence, 'correct': predicted == emotion }) - + # Analyze bias emotion_accuracies = {} emotion_confidences = {} emotion_predictions = defaultdict(Counter) - + for emotion, results in bias_results.items(): correct = sum(1 for r in results if r['correct']) accuracy = correct / len(results) * 100 avg_confidence = np.mean([r['confidence'] for r in results]) - + emotion_accuracies[emotion] = accuracy emotion_confidences[emotion] = avg_confidence - + # Count what this emotion was predicted as for r in results: emotion_predictions[emotion][r['predicted']] += 1 - + # Find most/least accurate emotions most_accurate = max(emotion_accuracies.items(), key=lambda x: x[1]) least_accurate = min(emotion_accuracies.items(), key=lambda x: x[1]) - + # Find most/least confident emotions most_confident = max(emotion_confidences.items(), key=lambda x: x[1]) least_confident = min(emotion_confidences.items(), key=lambda x: x[1]) - + self.test_results['bias_analysis'] = { 'emotion_accuracies': emotion_accuracies, 'emotion_confidences': emotion_confidences, @@ -377,7 +377,7 @@ def test_bias_analysis(self): 'overall_accuracy': np.mean(list(emotion_accuracies.values())), 'overall_confidence': np.mean(list(emotion_confidences.values())) } - + print(f"๐Ÿ“Š Bias Analysis Results:") print(f" Overall accuracy: {np.mean(list(emotion_accuracies.values())):.2f}%") print(f" Overall confidence: {np.mean(list(emotion_confidences.values())):.3f}") @@ -385,12 +385,12 @@ def test_bias_analysis(self): print(f" Least accurate: {least_accurate[0]} ({least_accurate[1]:.2f}%)") print(f" Most confident: {most_confident[0]} ({most_confident[1]:.3f})") print(f" Least confident: {least_confident[0]} ({least_confident[1]:.3f})") - + def test_robustness(self): """Test model robustness to variations.""" print("\n๐Ÿ›ก๏ธ ROBUSTNESS TESTS") print("=" * 60) - + base_texts = [ "I am happy today", "I feel sad about the news", @@ -405,10 +405,10 @@ def test_robustness(self): "I am proud of my work", "I feel tired after exercise" ] - + # Test with different tokenization lengths robustness_results = [] - + for base_text in base_texts: # Test with truncation for max_length in [10, 20, 50, 100, 200]: @@ -416,18 +416,18 @@ def test_robustness(self): inputs = self.tokenizer(base_text, return_tensors='pt', truncation=True, max_length=max_length, padding=True) if torch.cuda.is_available(): inputs = {k: v.to('cuda') for k, v in inputs.items()} - + with torch.no_grad(): outputs = self.model(**inputs) probabilities = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_label].item() - + if predicted_label in self.model.config.id2label: predicted_emotion = self.model.config.id2label[predicted_label] else: predicted_emotion = f"unknown_{predicted_label}" - + robustness_results.append({ 'base_text': base_text, 'max_length': max_length, @@ -444,10 +444,10 @@ def test_robustness(self): 'success': False, 'error': str(e) }) - + successful = sum(1 for r in robustness_results if r['success']) avg_confidence = np.mean([r['confidence'] for r in robustness_results if r['success']]) - + self.test_results['robustness_tests'] = { 'success_rate': successful / len(robustness_results) * 100, 'avg_confidence': avg_confidence, @@ -455,15 +455,15 @@ def test_robustness(self): 'successful': successful, 'details': robustness_results } - + print(f"๐Ÿ“Š Robustness Test Results: {successful/len(robustness_results)*100:.2f}% success rate, {avg_confidence:.3f} avg confidence") print(f" Successful: {successful}/{len(robustness_results)}") - + def test_real_world_scenarios(self): """Test with real-world scenarios.""" print("\n๐ŸŒ REAL-WORLD SCENARIOS") print("=" * 60) - + real_world_cases = [ # Social media posts ("Just got promoted! Can't believe it!", "excited"), @@ -478,7 +478,7 @@ def test_real_world_scenarios(self): ("Frustrated with the slow internet", "frustrated"), ("Hopeful about the new project", "hopeful"), ("Happy to see old friends", "happy"), - + # Journal entries ("Today I reflected on my journey and felt proud of how far I've come", "proud"), ("The uncertainty of the future is making me anxious", "anxious"), @@ -492,7 +492,7 @@ def test_real_world_scenarios(self): ("Feeling sad about the loss of a loved one", "sad"), ("I'm calm and at peace with myself", "calm"), ("I'm happy with the progress I've made", "happy"), - + # Customer service scenarios ("I'm frustrated with the poor service I received", "frustrated"), ("I'm grateful for the quick resolution", "grateful"), @@ -506,7 +506,7 @@ def test_real_world_scenarios(self): ("I'm sad that I had to go through this", "sad"), ("I'm calm now that everything is sorted", "calm"), ("I'm happy with the outcome", "happy"), - + # Work scenarios ("I'm excited about the new project assignment", "excited"), ("I'm anxious about the upcoming deadline", "anxious"), @@ -521,11 +521,11 @@ def test_real_world_scenarios(self): ("I'm calm during the presentation", "calm"), ("I'm happy with the recognition", "happy") ] - + correct = 0 confidences = [] predictions_by_emotion = defaultdict(list) - + for text, expected in real_world_cases: predicted, confidence, _ = self.predict_emotion(text) is_correct = predicted == expected @@ -538,10 +538,10 @@ def test_real_world_scenarios(self): 'confidence': confidence, 'correct': is_correct }) - + accuracy = correct / len(real_world_cases) * 100 avg_confidence = np.mean(confidences) - + # Analyze performance by emotion in real-world scenarios emotion_performance = {} for emotion, cases in predictions_by_emotion.items(): @@ -554,7 +554,7 @@ def test_real_world_scenarios(self): 'total_cases': len(cases), 'correct': emotion_correct } - + self.test_results['real_world_scenarios'] = { 'accuracy': accuracy, 'avg_confidence': avg_confidence, @@ -562,34 +562,34 @@ def test_real_world_scenarios(self): 'correct': correct, 'emotion_performance': emotion_performance } - + print(f"๐Ÿ“Š Real-World Results: {accuracy:.2f}% accuracy, {avg_confidence:.3f} avg confidence") print(f" Correct: {correct}/{len(real_world_cases)}") - + # Show worst performing emotions worst_emotions = sorted(emotion_performance.items(), key=lambda x: x[1]['accuracy'])[:3] print(f" Worst performing emotions: {', '.join([f'{e[0]} ({e[1]['accuracy']:.1f}%)' for e in worst_emotions])}") - + def analyze_confidence_distribution(self): """Analyze confidence distribution across all tests.""" print("\n๐Ÿ“Š CONFIDENCE ANALYSIS") print("=" * 60) - + # Collect all confidence scores from previous tests all_confidences = [] - + # From basic tests if 'basic_tests' in self.test_results: all_confidences.extend([0.8, 0.9, 0.95]) # Representative values - + # From edge cases if 'edge_cases' in self.test_results: all_confidences.extend([r['confidence'] for r in self.test_results['edge_cases']['details']]) - + # From real-world scenarios if 'real_world_scenarios' in self.test_results: all_confidences.extend([0.85, 0.92, 0.88]) # Representative values - + if all_confidences: confidence_stats = { 'mean': np.mean(all_confidences), @@ -602,9 +602,9 @@ def analyze_confidence_distribution(self): 'low_confidence': sum(1 for c in all_confidences if c < 0.5), 'total': len(all_confidences) } - + self.test_results['confidence_analysis'] = confidence_stats - + print(f"๐Ÿ“Š Confidence Distribution:") print(f" Mean: {confidence_stats['mean']:.3f}") print(f" Median: {confidence_stats['median']:.3f}") @@ -613,27 +613,27 @@ def analyze_confidence_distribution(self): print(f" High confidence (โ‰ฅ0.8): {confidence_stats['high_confidence']}/{confidence_stats['total']} ({confidence_stats['high_confidence']/confidence_stats['total']*100:.1f}%)") print(f" Medium confidence (0.5-0.8): {confidence_stats['medium_confidence']}/{confidence_stats['total']} ({confidence_stats['medium_confidence']/confidence_stats['total']*100:.1f}%)") print(f" Low confidence (<0.5): {confidence_stats['low_confidence']}/{confidence_stats['total']} ({confidence_stats['low_confidence']/confidence_stats['total']*100:.1f}%)") - + def generate_comprehensive_report(self): """Generate a comprehensive test report.""" print("\n๐Ÿ“‹ MEGA COMPREHENSIVE TEST REPORT") print("=" * 80) - + # Calculate overall metrics total_tests = 0 total_correct = 0 all_confidences = [] - + for test_type, results in self.test_results.items(): if 'accuracy' in results: total_tests += results.get('total_tests', 0) total_correct += results.get('correct', 0) if 'avg_confidence' in results: all_confidences.append(results['avg_confidence']) - + overall_accuracy = total_correct / total_tests * 100 if total_tests > 0 else 0 overall_confidence = np.mean(all_confidences) if all_confidences else 0 - + # Generate report report = { 'timestamp': datetime.now().isoformat(), @@ -651,14 +651,14 @@ def generate_comprehensive_report(self): 'deployment_ready': overall_accuracy >= 80 and overall_confidence >= 0.6 } } - + # Save report report_path = f"test_reports/mega_comprehensive_test_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" os.makedirs("test_reports", exist_ok=True) - + with open(report_path, 'w') as f: json.dump(report, f, indent=2) - + # Print summary print(f"๐ŸŽฏ OVERALL PERFORMANCE SUMMARY") print(f" Total Tests: {total_tests}") @@ -667,11 +667,11 @@ def generate_comprehensive_report(self): print(f" Model Status: {report['summary']['model_status']}") print(f" Confidence Status: {report['summary']['confidence_status']}") print(f" Deployment Ready: {'โœ… YES' if report['summary']['deployment_ready'] else 'โŒ NO'}") - + print(f"\n๐Ÿ“ Detailed report saved to: {report_path}") - + return report - + def run_all_tests(self): """Run all comprehensive tests.""" print("๐Ÿš€ STARTING MEGA COMPREHENSIVE MODEL TESTING") @@ -680,11 +680,11 @@ def run_all_tests(self): print(f"๐ŸŽฏ Emotions: {', '.join(self.emotions)}") print(f"โฐ Started at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") print() - + # Load model if not self.load_model(): return False - + # Run all test suites self.test_basic_functionality() self.test_edge_cases() @@ -693,20 +693,20 @@ def run_all_tests(self): self.test_robustness() self.test_real_world_scenarios() self.analyze_confidence_distribution() - + # Generate comprehensive report report = self.generate_comprehensive_report() - + print(f"\n๐ŸŽ‰ MEGA COMPREHENSIVE TESTING COMPLETE!") print("=" * 80) - + return report def main(): """Main function to run mega comprehensive testing.""" tester = MegaComprehensiveModelTester() report = tester.run_all_tests() - + if report: print(f"\nโœ… Testing completed successfully!") print(f"๐Ÿ“Š Final Results:") diff --git a/scripts/testing/mega_test_summary.py b/scripts/testing/mega_test_summary.py index 2954387f4..1bf936dda 100644 --- a/scripts/testing/mega_test_summary.py +++ b/scripts/testing/mega_test_summary.py @@ -8,23 +8,23 @@ def display_mega_test_results(): """Display the mega comprehensive test results.""" - + print("๐ŸŽ‰ MEGA COMPREHENSIVE TEST RESULTS SUMMARY") print("=" * 80) print("๐Ÿ“ Model Tested: deployment/models/default") print("๐ŸŽฏ Emotions: anxious, calm, content, excited, frustrated, grateful, happy, hopeful, overwhelmed, proud, sad, tired") print() - + print("๐Ÿ“Š TEST SUITE RESULTS") print("=" * 50) - + # Basic Functionality Tests print("๐Ÿงช BASIC FUNCTIONALITY TESTS") print(" โœ… Accuracy: 100.00% (24/24)") print(" โœ… Average Confidence: 0.965 (96.5%)") print(" โœ… All basic emotion expressions correctly identified") print() - + # Edge Cases Tests print("๐Ÿ” EDGE CASES AND UNUSUAL INPUTS") print(" โœ… Accuracy: 81.58% (31/38)") @@ -32,7 +32,7 @@ def display_mega_test_results(): print(" โœ… Handles short inputs, long inputs, mixed emotions, negations, questions") print(" โœ… Handles emojis, symbols, capitalization variations, special characters") print() - + # Stress Tests print("๐Ÿ’ช STRESS TESTS") print(" โœ… Success Rate: 100.00% (38/38)") @@ -40,7 +40,7 @@ def display_mega_test_results(): print(" โœ… Handles random noise text, very long texts, special characters") print(" โœ… No crashes or errors under stress conditions") print() - + # Bias Analysis print("โš–๏ธ BIAS ANALYSIS") print(" โœ… Overall Accuracy: 100.00%") @@ -50,7 +50,7 @@ def display_mega_test_results(): print(" โœ… Least Confident: content (0.951)") print(" โœ… No significant bias detected") print() - + # Robustness Tests print("๐Ÿ›ก๏ธ ROBUSTNESS TESTS") print(" โœ… Success Rate: 100.00% (60/60)") @@ -58,7 +58,7 @@ def display_mega_test_results(): print(" โœ… Handles different tokenization lengths (10-200 tokens)") print(" โœ… Consistent performance across input variations") print() - + # Real-World Scenarios print("๐ŸŒ REAL-WORLD SCENARIOS") print(" โœ… Accuracy: 93.75% (45/48)") @@ -66,7 +66,7 @@ def display_mega_test_results(): print(" โœ… Tested: Social media posts, journal entries, customer service, work scenarios") print(" โš ๏ธ Minor issues with: excited, grateful, hopeful (75% accuracy each)") print() - + # Confidence Analysis print("๐Ÿ“Š CONFIDENCE ANALYSIS") print(" โœ… Mean Confidence: 0.839 (83.9%)") @@ -76,10 +76,10 @@ def display_mega_test_results(): print(" โœ… Low Confidence (<0.5): 11.4% of predictions") print(" โœ… Confidence Range: 0.134 - 0.971") print() - + print("๐ŸŽฏ OVERALL PERFORMANCE ASSESSMENT") print("=" * 50) - + print("๐Ÿ† EXCELLENT PERFORMANCE ACROSS ALL METRICS:") print() print("โœ… BASIC FUNCTIONALITY: PERFECT (100% accuracy)") @@ -111,10 +111,10 @@ def display_mega_test_results(): print(" - Handles social media, journal entries, work scenarios") print(" - Minor issues with 3 emotions (excited, grateful, hopeful)") print() - + print("๐Ÿš€ DEPLOYMENT READINESS ASSESSMENT") print("=" * 50) - + print("โœ… DEPLOYMENT STATUS: FULLY READY") print() print("๐ŸŽฏ STRENGTHS:") diff --git a/scripts/testing/setup_model_testing.py b/scripts/testing/setup_model_testing.py index eeed16839..aeca7e876 100644 --- a/scripts/testing/setup_model_testing.py +++ b/scripts/testing/setup_model_testing.py @@ -10,15 +10,15 @@ def check_model_files(): """Check if required model files exist.""" print("๐Ÿ” Checking for model files...") - + required_files = { 'model': 'best_simple_model.pth', 'results': 'simple_training_results.json' } - + missing_files = [] existing_files = {} - + for file_type, filename in required_files.items(): if os.path.exists(filename): size = os.path.getsize(filename) @@ -27,13 +27,13 @@ def check_model_files(): else: missing_files.append(file_type) print(f"โŒ {file_type.capitalize()}: {filename} - MISSING") - + return existing_files, missing_files def create_mock_results(): """Create mock results file for testing if missing.""" print("\n๐Ÿ”ง Creating mock results file for testing...") - + # Mock results based on our training mock_results = { "best_f1": 0.6692, @@ -75,16 +75,16 @@ def create_mock_results(): "neutral": "calm" } } - + with open('simple_training_results.json', 'w') as f: json.dump(mock_results, f, indent=2) - + print("โœ… Created mock results file: simple_training_results.json") def find_model_file(): """Find the model file in common locations.""" print("\n๐Ÿ” Searching for model file...") - + search_locations = [ "best_simple_model.pth", "best_focal_model.pth", # Fallback @@ -92,19 +92,19 @@ def find_model_file(): os.path.expanduser("~/Desktop/best_simple_model.pth"), os.path.expanduser("~/best_simple_model.pth") ] - + for location in search_locations: if os.path.exists(location): size = os.path.getsize(location) print(f"โœ… Found model: {location} ({size:,} bytes)") - + # Copy to current directory if not already here if location != "best_simple_model.pth": shutil.copy2(location, "best_simple_model.pth") print(f"โœ… Copied to: best_simple_model.pth") - + return True - + print("โŒ Model file not found in common locations") return False @@ -112,45 +112,45 @@ def setup_testing(): """Main setup function.""" print("๐Ÿš€ SETTING UP MODEL TESTING") print("=" * 50) - + # Check existing files existing_files, missing_files = check_model_files() - + # Find model file if missing if 'model' in missing_files: if not find_model_file(): print("\nโŒ Cannot proceed without model file!") print("๐Ÿ“‹ Please download best_simple_model.pth from Colab and place it in this directory") return False - + # Create mock results if missing if 'results' in missing_files: create_mock_results() - + print("\nโœ… Setup complete! Ready for testing.") return True def run_quick_test(): """Run a quick test to verify everything works.""" print("\n๐Ÿงช Running quick test...") - + try: import torch import transformers from sklearn.preprocessing import LabelEncoder - + print("โœ… All required libraries available") - + # Test model loading if os.path.exists('best_simple_model.pth'): print("โœ… Model file exists") - + # Try to load a small part to verify it's valid checkpoint = torch.load('best_simple_model.pth', map_location='cpu') print(f"โœ… Model checkpoint loaded with {len(checkpoint)} layers") - + return True - + except ImportError as e: print(f"โŒ Missing library: {e}") print("๐Ÿ“‹ Install with: pip install torch transformers scikit-learn") diff --git a/scripts/testing/simple_model_test.py b/scripts/testing/simple_model_test.py index 265b0b8f1..888f69916 100644 --- a/scripts/testing/simple_model_test.py +++ b/scripts/testing/simple_model_test.py @@ -10,13 +10,13 @@ def test_model_files(): """Test if model files exist and are valid.""" print("๐Ÿงช SIMPLE MODEL TEST") print("=" * 50) - + # Check model file model_file = "best_simple_model.pth" if os.path.exists(model_file): size = os.path.getsize(model_file) print(f"โœ… Model file: {model_file} ({size:,} bytes)") - + # Check if it's a reasonable size (should be ~400MB+) if size > 100_000_000: # 100MB print("โœ… Model file size looks good!") @@ -25,36 +25,36 @@ def test_model_files(): else: print(f"โŒ Model file missing: {model_file}") return False - + # Check results file results_file = "simple_training_results.json" if os.path.exists(results_file): size = os.path.getsize(results_file) print(f"โœ… Results file: {results_file} ({size:,} bytes)") - + # Try to load and parse try: with open(results_file, 'r') as f: results = json.load(f) - + print(f"โœ… Results file is valid JSON") print(f"๐Ÿ“Š F1 Score: {results.get('best_f1', 'N/A')}") print(f"๐Ÿ“Š Emotions: {len(results.get('all_emotions', []))}") - + except json.JSONDecodeError: print("โŒ Results file is not valid JSON") return False else: print(f"โŒ Results file missing: {results_file}") return False - + return True def test_python_environment(): """Test Python environment and libraries.""" print("\n๐Ÿ”ง Testing Python Environment:") print("-" * 30) - + # Test basic imports try: import sys @@ -62,7 +62,7 @@ def test_python_environment(): except ImportError: print("โŒ Cannot import sys") return False - + # Test JSON try: import json @@ -70,7 +70,7 @@ def test_python_environment(): except ImportError: print("โŒ JSON module not available") return False - + # Test OS try: import os @@ -78,29 +78,29 @@ def test_python_environment(): except ImportError: print("โŒ OS module not available") return False - + return True def suggest_next_steps(): """Suggest next steps for testing.""" print("\n๐Ÿ“‹ NEXT STEPS:") print("=" * 30) - + print("1. ๐Ÿ Python Environment:") print(" - You're using Python 3.8.6 but libraries are in Python 3.11") print(" - Options:") print(" a) Use: python3.11 scripts/test_emotion_model.py") print(" b) Install libraries in current Python: pip3 install torch transformers scikit-learn") print(" c) Create virtual environment") - + print("\n2. ๐Ÿงช Model Testing:") print(" - Once Python is fixed, run: python scripts/test_emotion_model.py") print(" - This will test the model with sample journal entries") - + print("\n3. ๐Ÿ“Š Dataset Expansion:") print(" - Run: python scripts/expand_journal_dataset.py") print(" - This will create 1000+ balanced samples") - + print("\n4. ๐Ÿš€ Retraining:") print(" - Use expanded dataset to retrain") print(" - Expect 75-85% F1 score!") @@ -109,22 +109,22 @@ def main(): """Main test function.""" print("๐Ÿš€ SIMPLE MODEL TESTING") print("=" * 50) - + # Test files files_ok = test_model_files() - + # Test environment env_ok = test_python_environment() - + print(f"\n๐Ÿ“Š Test Results:") print(f" Files: {'โœ…' if files_ok else 'โŒ'}") print(f" Environment: {'โœ…' if env_ok else 'โŒ'}") - + if files_ok and env_ok: print("\n๐ŸŽ‰ All tests passed! Ready for full testing.") else: print("\nโš ๏ธ Some issues found. Check above.") - + suggest_next_steps() if __name__ == "__main__": diff --git a/scripts/testing/simple_rate_limiter_test.py b/scripts/testing/simple_rate_limiter_test.py index 0519ecba6..e69de29bb 100644 --- a/scripts/testing/simple_rate_limiter_test.py +++ b/scripts/testing/simple_rate_limiter_test.py @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/scripts/testing/simple_temperature_test.py b/scripts/testing/simple_temperature_test.py index b7b4c7372..0e0199e99 100644 --- a/scripts/testing/simple_temperature_test.py +++ b/scripts/testing/simple_temperature_test.py @@ -60,13 +60,13 @@ def simple_temperature_test(): # Test different temperatures temperatures = [0.5, 1.0, 1.5, 2.0] - + for temp in temperatures: logger.info(f"๐Ÿ“Š Testing temperature: {temp}") - + # Set model temperature model.temperature = temp - + # Evaluate model try: results = evaluate_emotion_classifier( @@ -76,9 +76,9 @@ def simple_temperature_test(): labels=test_labels, device=device ) - + logger.info(f" Temperature {temp}: F1 = {results.get('f1_score', 'N/A'):.4f}") - + except Exception as e: logger.warning(f" Temperature {temp}: Error - {e}") diff --git a/scripts/training/SAMO_Colab_Setup.py b/scripts/training/SAMO_Colab_Setup.py index 955cc7c44..a4b0ad2e1 100644 --- a/scripts/training/SAMO_Colab_Setup.py +++ b/scripts/training/SAMO_Colab_Setup.py @@ -23,13 +23,13 @@ def check_gpu() -> Optional[bool]: try: import torch gpu_available = torch.cuda.is_available() - + if gpu_available: torch.cuda.get_device_name(0) torch.cuda.get_device_properties(0).total_memory / 1e9 else: pass - + return True except ImportError: return False @@ -41,7 +41,7 @@ def clone_repository() -> Optional[bool]: subprocess.run([ "git", "clone", "https://github.com/uelkerd/SAMO--DL.git" ], check=True) - + # Change to repository directory os.chdir("SAMO--DL") return True @@ -55,7 +55,7 @@ def install_dependencies() -> bool: subprocess.run(["pip", "install", "-e", "."], check=True) except subprocess.CalledProcessError: return False - + # Install voice processing libraries voice_packages = [ "pyaudio", @@ -64,13 +64,13 @@ def install_dependencies() -> bool: "openai-whisper", "speechrecognition" ] - + for package in voice_packages: try: subprocess.run(["pip", "install", package], check=True) except subprocess.CalledProcessError: return False - + return True def test_audio_libraries() -> bool: @@ -79,18 +79,18 @@ def test_audio_libraries() -> bool: import soundfile as sf except ImportError: return False - + try: import librosa except ImportError: return False - + try: import whisper whisper.load_model("base") except ImportError: return False - + return True def create_voice_demo() -> bool: @@ -108,27 +108,27 @@ def record_audio(duration=5, sample_rate=16000): chunk = 1024 format = pyaudio.paInt16 channels = 1 - + p = pyaudio.PyAudio() stream = p.open(format=format, channels=channels, rate=sample_rate, input=True, frames_per_buffer=chunk) - + print("๐ŸŽค Recording... Speak now!") frames = [] - + for i in range(0, int(sample_rate / chunk * duration)): data = stream.read(chunk) frames.append(data) - + print("โœ… Recording complete!") - + stream.stop_stream() stream.close() p.terminate() - + return frames def voice_to_text(audio_frames, sample_rate=16000): @@ -139,11 +139,11 @@ def voice_to_text(audio_frames, sample_rate=16000): wf.setsampwidth(2) wf.setframerate(sample_rate) wf.writeframes(b''.join(audio_frames)) - + # Transcribe with Whisper model = whisper.load_model("base") result = model.transcribe("temp_audio.wav") - + return result["text"] def detect_emotion_from_voice(audio_frames, sample_rate=16000): @@ -151,12 +151,12 @@ def detect_emotion_from_voice(audio_frames, sample_rate=16000): # Convert audio frames to numpy array audio_data = np.frombuffer(b''.join(audio_frames), dtype=np.int16) audio_data = audio_data.astype(np.float32) / 32768.0 - + # Extract audio features mfccs = librosa.feature.mfcc(y=audio_data, sr=sample_rate, n_mfcc=13) spectral_centroids = librosa.feature.spectral_centroid(y=audio_data, sr=sample_rate) zero_crossing_rate = librosa.feature.zero_crossing_rate(audio_data) - + # Calculate statistics features = { 'mfcc_mean': np.mean(mfccs), @@ -164,7 +164,7 @@ def detect_emotion_from_voice(audio_frames, sample_rate=16000): 'spectral_centroid_mean': np.mean(spectral_centroids), 'zero_crossing_rate_mean': np.mean(zero_crossing_rate) } - + # Simple emotion mapping if features['spectral_centroid_mean'] > 2000: emotion = "excited" @@ -172,7 +172,7 @@ def detect_emotion_from_voice(audio_frames, sample_rate=16000): emotion = "sad" else: emotion = "neutral" - + return emotion, features # Test voice processing @@ -185,10 +185,10 @@ def detect_emotion_from_voice(audio_frames, sample_rate=16000): print(f"๐Ÿ˜Š Detected emotion: {emotion}") print(f"๐Ÿ“Š Audio features: {features}") ''' - + with open("voice_demo.py", "w") as f: f.write(demo_code) - + return True def create_f1_optimization_script() -> bool: @@ -209,18 +209,18 @@ def create_f1_optimization_script() -> bool: class FocalLoss(nn.Module): """Focal Loss for handling class imbalance.""" - + def __init__(self, alpha=0.25, gamma=2.0, reduction="mean"): super().__init__() self.alpha = alpha self.gamma = gamma self.reduction = reduction - + def forward(self, inputs, targets): bce_loss = nn.functional.binary_cross_entropy_with_logits(inputs, targets, reduction='none') pt = torch.exp(-bce_loss) focal_loss = self.alpha * (1 - pt) ** self.gamma * bce_loss - + if self.reduction == "mean": return focal_loss.mean() elif self.reduction == "sum": @@ -231,38 +231,38 @@ def forward(self, inputs, targets): def optimize_f1_score(): """Optimize F1 score using focal loss and other techniques.""" print("๐Ÿš€ Starting F1 optimization...") - + # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print(f"Using device: {device}") - + # Load dataset data_loader = GoEmotionsDataLoader() datasets = data_loader.prepare_datasets() - + # Create model model = BERTEmotionClassifier() model.to(device) - + # Create focal loss focal_loss = FocalLoss(alpha=0.25, gamma=2.0) - + # Setup optimizer optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5) - + print("โœ… F1 optimization setup complete!") print("๐ŸŽฏ Expected improvement: 13.2% โ†’ 50%+ F1 score") - + return model, focal_loss, optimizer # Run optimization if __name__ == "__main__": model, focal_loss, optimizer = optimize_f1_score() ''' - + with open("f1_optimization.py", "w") as f: f.write(f1_code) - + return True def print_next_steps() -> None: @@ -271,30 +271,30 @@ def print_next_steps() -> None: def main() -> bool: """Main setup function.""" print_header() - + # Check GPU if not check_gpu(): return False - + # Clone repository if not clone_repository(): return False - + # Install dependencies if not install_dependencies(): return False - + # Test audio libraries if not test_audio_libraries(): return False - + # Create demo scripts create_voice_demo() create_f1_optimization_script() - + # Print next steps print_next_steps() - + return True if __name__ == "__main__": diff --git a/scripts/training/add_advanced_features_to_notebook.py b/scripts/training/add_advanced_features_to_notebook.py index 3f063dc9e..48f50988f 100644 --- a/scripts/training/add_advanced_features_to_notebook.py +++ b/scripts/training/add_advanced_features_to_notebook.py @@ -13,11 +13,11 @@ def add_advanced_features(): """Add advanced features to the ultimate notebook.""" - + # Read the existing notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Add focal loss implementation focal_loss_cell = { "cell_type": "markdown", @@ -26,7 +26,7 @@ def add_advanced_features(): "## ๐ŸŽฏ IMPLEMENTING FOCAL LOSS" ] } - + focal_loss_code = { "cell_type": "code", "execution_count": None, @@ -58,7 +58,7 @@ def add_advanced_features(): "print('โœ… Focal Loss implementation ready')" ] } - + # Add class weighting implementation class_weighting_cell = { "cell_type": "markdown", @@ -67,7 +67,7 @@ def add_advanced_features(): "## โš–๏ธ IMPLEMENTING CLASS WEIGHTING" ] } - + class_weighting_code = { "cell_type": "code", "execution_count": None, @@ -96,7 +96,7 @@ def add_advanced_features(): "print(f'โœ… Device: {device}')" ] } - + # Add WeightedLossTrainer weighted_trainer_cell = { "cell_type": "markdown", @@ -105,7 +105,7 @@ def add_advanced_features(): "## ๐Ÿš€ CREATING WEIGHTED LOSS TRAINER" ] } - + weighted_trainer_code = { "cell_type": "code", "execution_count": None, @@ -145,7 +145,7 @@ def add_advanced_features(): "print('โœ… WeightedLossTrainer with focal loss ready')" ] } - + # Add model loading and configuration model_loading_cell = { "cell_type": "markdown", @@ -154,7 +154,7 @@ def add_advanced_features(): "## ๐Ÿ”ง LOADING MODEL WITH PROPER CONFIGURATION" ] } - + model_loading_code = { "cell_type": "code", "execution_count": None, @@ -183,7 +183,7 @@ def add_advanced_features(): "print(f'โœ… label2id: {model.config.label2id}')" ] } - + # Add data preprocessing preprocessing_cell = { "cell_type": "markdown", @@ -192,7 +192,7 @@ def add_advanced_features(): "## ๐Ÿ“ DATA PREPROCESSING" ] } - + preprocessing_code = { "cell_type": "code", "execution_count": None, @@ -221,7 +221,7 @@ def add_advanced_features(): "print(f'โœ… Validation samples: {len(val_dataset)}')" ] } - + # Add training arguments training_args_cell = { "cell_type": "markdown", @@ -230,7 +230,7 @@ def add_advanced_features(): "## โš™๏ธ TRAINING ARGUMENTS" ] } - + training_args_code = { "cell_type": "code", "execution_count": None, @@ -261,7 +261,7 @@ def add_advanced_features(): "print('โœ… Training arguments configured')" ] } - + # Add compute metrics compute_metrics_cell = { "cell_type": "markdown", @@ -270,7 +270,7 @@ def add_advanced_features(): "## ๐Ÿ“Š COMPUTE METRICS" ] } - + compute_metrics_code = { "cell_type": "code", "execution_count": None, @@ -298,7 +298,7 @@ def add_advanced_features(): "print('โœ… Compute metrics function ready')" ] } - + # Add trainer initialization trainer_init_cell = { "cell_type": "markdown", @@ -307,7 +307,7 @@ def add_advanced_features(): "## ๐Ÿš€ INITIALIZING TRAINER" ] } - + trainer_init_code = { "cell_type": "code", "execution_count": None, @@ -330,7 +330,7 @@ def add_advanced_features(): "print('โœ… Trainer initialized with focal loss and class weighting')" ] } - + # Add training training_cell = { "cell_type": "markdown", @@ -339,7 +339,7 @@ def add_advanced_features(): "## ๐Ÿš€ STARTING TRAINING" ] } - + training_code = { "cell_type": "code", "execution_count": None, @@ -361,7 +361,7 @@ def add_advanced_features(): "print('โœ… Training completed successfully!')" ] } - + # Add evaluation evaluation_cell = { "cell_type": "markdown", @@ -370,7 +370,7 @@ def add_advanced_features(): "## ๐Ÿ“Š EVALUATING MODEL" ] } - + evaluation_code = { "cell_type": "code", "execution_count": None, @@ -394,7 +394,7 @@ def add_advanced_features(): " print(f'โš ๏ธ Target not achieved. Need {0.75 - results[\"eval_f1\"]:.3f} more F1 points')" ] } - + # Add advanced validation advanced_validation_cell = { "cell_type": "markdown", @@ -403,7 +403,7 @@ def add_advanced_features(): "## ๐Ÿงช ADVANCED VALIDATION" ] } - + advanced_validation_code = { "cell_type": "code", "execution_count": None, @@ -482,7 +482,7 @@ def add_advanced_features(): " print(f'โŒ Too much bias: {max_bias:.1%} (need <30%)')" ] } - + # Add model saving with verification model_saving_cell = { "cell_type": "markdown", @@ -491,7 +491,7 @@ def add_advanced_features(): "## ๐Ÿ’พ SAVING MODEL WITH VERIFICATION" ] } - + model_saving_code = { "cell_type": "code", "execution_count": None, @@ -580,7 +580,7 @@ def add_advanced_features(): "print('3. Deploy if all tests pass')" ] } - + # Add all cells to the notebook new_cells = [ focal_loss_cell, @@ -608,13 +608,13 @@ def add_advanced_features(): model_saving_cell, model_saving_code ] - + notebook['cells'].extend(new_cells) - + # Save the enhanced notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Enhanced notebook with all advanced features created!') print('๐Ÿ“‹ All features included:') print(' โœ… Configuration preservation') @@ -623,7 +623,7 @@ def add_advanced_features(): print(' โœ… Data augmentation') print(' โœ… Advanced validation') print(' โœ… Model saving with verification') - + return 'notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb' if __name__ == "__main__": diff --git a/scripts/training/bulletproof_training.py b/scripts/training/bulletproof_training.py index 70695c761..859f2f46b 100644 --- a/scripts/training/bulletproof_training.py +++ b/scripts/training/bulletproof_training.py @@ -24,19 +24,19 @@ def validate_environment(): """Validate the environment and clear any corrupted state.""" logger.info("๐Ÿ” Validating environment...") - + # Clear GPU memory if torch.cuda.is_available(): torch.cuda.empty_cache() logger.info("โœ… GPU memory cleared") - + # Check CUDA if torch.cuda.is_available(): logger.info(f"โœ… CUDA available: {torch.cuda.get_device_name()}") logger.info(f"โœ… CUDA memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") else: logger.warning("โš ๏ธ CUDA not available, using CPU") - + # Test basic operations try: test_tensor = torch.randn(2, 3) @@ -45,47 +45,47 @@ def validate_environment(): except Exception as e: logger.error(f"โŒ Basic tensor operations failed: {e}") return False - + return True def create_unified_label_encoder(): """Create a unified label encoder for both datasets.""" logger.info("๐Ÿ”ง Creating unified label encoder...") - + # Load datasets go_emotions = load_dataset("go_emotions", "simplified") with open('data/journal_test_dataset.json', 'r') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) - + # Extract labels go_labels = set() for example in go_emotions['train']: if example['labels']: go_labels.update(example['labels']) - + journal_labels = set(journal_df['emotion'].unique()) - + # Find common labels common_labels = sorted(list(go_labels.intersection(journal_labels))) if not common_labels: logger.warning("โš ๏ธ No common labels found! Using all labels...") common_labels = sorted(list(go_labels.union(journal_labels))) - + logger.info(f"๐Ÿ“Š Using {len(common_labels)} labels: {common_labels}") - + # Create encoder label_encoder = LabelEncoder() label_encoder.fit(common_labels) - + # Save encoder with open('unified_label_encoder.pkl', 'wb') as f: pickle.dump(label_encoder, f) - + # Save mappings label_to_id = {label: idx for idx, label in enumerate(label_encoder.classes_)} id_to_label = {idx: label for label, idx in label_to_id.items()} - + with open('label_mappings.json', 'w') as f: json.dump({ 'label_to_id': label_to_id, @@ -93,22 +93,22 @@ def create_unified_label_encoder(): 'num_labels': len(label_encoder.classes_), 'classes': label_encoder.classes_.tolist() }, f, indent=2) - + logger.info(f"โœ… Label encoder created with {len(label_encoder.classes_)} classes") return label_encoder, label_to_id, id_to_label def prepare_filtered_data(label_encoder, label_to_id): """Prepare filtered data using only common labels.""" logger.info("๐Ÿ“Š Preparing filtered data...") - + # Load datasets go_emotions = load_dataset("go_emotions", "simplified") with open('data/journal_test_dataset.json', 'r') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) - + valid_labels = set(label_encoder.classes_) - + # Filter GoEmotions data go_texts = [] go_labels = [] @@ -119,7 +119,7 @@ def prepare_filtered_data(label_encoder, label_to_id): go_texts.append(example['text']) go_labels.append(label_to_id[label]) break - + # Filter journal data journal_texts = [] journal_labels = [] @@ -127,35 +127,35 @@ def prepare_filtered_data(label_encoder, label_to_id): if row['emotion'] in valid_labels: journal_texts.append(row['content']) journal_labels.append(label_to_id[row['emotion']]) - + logger.info(f"๐Ÿ“Š Filtered GoEmotions: {len(go_texts)} samples") logger.info(f"๐Ÿ“Š Filtered Journal: {len(journal_texts)} samples") - + # Validate label ranges - FIX: Convert to integers for comparison if go_labels: go_label_range = (min(go_labels), max(go_labels)) else: go_label_range = (0, 0) - + if journal_labels: journal_label_range = (min(journal_labels), max(journal_labels)) else: journal_label_range = (0, 0) - + expected_range = (0, len(label_encoder.classes_) - 1) - + logger.info(f"๐Ÿ“Š GoEmotions label range: {go_label_range}") logger.info(f"๐Ÿ“Š Journal label range: {journal_label_range}") logger.info(f"๐Ÿ“Š Expected range: {expected_range}") - + if go_label_range[0] < expected_range[0] or go_label_range[1] > expected_range[1]: logger.error(f"โŒ GoEmotions labels out of range!") return None, None, None, None - + if journal_label_range[0] < expected_range[0] or journal_label_range[1] > expected_range[1]: logger.error(f"โŒ Journal labels out of range!") return None, None, None, None - + logger.info("โœ… All labels within expected range") return go_texts, go_labels, journal_texts, journal_labels @@ -166,30 +166,30 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + # Validate data if len(texts) != len(labels): raise ValueError(f"Texts and labels have different lengths: {len(texts)} vs {len(labels)}") - + # Validate labels for i, label in enumerate(labels): if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {i}: {label}") - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = self.texts[idx] label = self.labels[idx] - + # Validate inputs if not isinstance(text, str) or not text.strip(): raise ValueError(f"Invalid text at index {idx}") - + if not isinstance(label, int) or label < 0: raise ValueError(f"Invalid label at index {idx}: {label}") - + encoding = self.tokenizer( text, truncation=True, @@ -197,7 +197,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -208,84 +208,84 @@ class SimpleEmotionClassifier(nn.Module): """Simple emotion classifier with validation.""" def __init__(self, model_name="bert-base-uncased", num_labels=None): super().__init__() - + if num_labels is None or num_labels <= 0: raise ValueError(f"Invalid num_labels: {num_labels}") - + self.num_labels = num_labels self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(0.3) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + logger.info(f"โœ… Model initialized with {num_labels} labels") - + def forward(self, input_ids, attention_mask): # Validate inputs if input_ids.dim() != 2: raise ValueError(f"Expected input_ids to be 2D, got {input_ids.dim()}D") - + if attention_mask.dim() != 2: raise ValueError(f"Expected attention_mask to be 2D, got {attention_mask.dim()}D") - + outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask) pooled_output = outputs.pooler_output logits = self.classifier(self.dropout(pooled_output)) - + # Validate outputs if logits.shape[-1] != self.num_labels: raise ValueError(f"Expected {self.num_labels} output classes, got {logits.shape[-1]}") - + return logits def train_model_simple(go_texts, go_labels, journal_texts, journal_labels, num_labels): """Simple training function with comprehensive validation.""" logger.info("๐Ÿš€ Starting simple training...") - + # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") logger.info(f"โœ… Using device: {device}") - + # Initialize tokenizer and model tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = SimpleEmotionClassifier(model_name="bert-base-uncased", num_labels=num_labels) model = model.to(device) - + # Create datasets go_dataset = SimpleEmotionDataset(go_texts, go_labels, tokenizer) journal_dataset = SimpleEmotionDataset(journal_texts, journal_labels, tokenizer) - + # Split journal data journal_train_texts, journal_val_texts, journal_train_labels, journal_val_labels = train_test_split( journal_texts, journal_labels, test_size=0.3, random_state=42, stratify=journal_labels ) - + journal_train_dataset = SimpleEmotionDataset(journal_train_texts, journal_train_labels, tokenizer) journal_val_dataset = SimpleEmotionDataset(journal_val_texts, journal_val_labels, tokenizer) - + # Create dataloaders go_loader = DataLoader(go_dataset, batch_size=8, shuffle=True) journal_train_loader = DataLoader(journal_train_dataset, batch_size=8, shuffle=True) journal_val_loader = DataLoader(journal_val_dataset, batch_size=8, shuffle=False) - + logger.info(f"โœ… Training samples: {len(go_dataset)} GoEmotions + {len(journal_train_dataset)} Journal") logger.info(f"โœ… Validation samples: {len(journal_val_dataset)} Journal") - + # Training setup optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5) criterion = nn.CrossEntropyLoss() - + # Training loop num_epochs = 3 # Reduced for testing best_f1 = 0.0 - + for epoch in range(num_epochs): logger.info(f"๐Ÿ”„ Epoch {epoch + 1}/{num_epochs}") - + # Training model.train() total_loss = 0 num_batches = 0 - + # Train on GoEmotions logger.info(" ๐Ÿ“š Training on GoEmotions...") for i, batch in enumerate(go_loader): @@ -294,34 +294,34 @@ def train_model_simple(go_texts, go_labels, journal_texts, journal_labels, num_l if 'input_ids' not in batch or 'attention_mask' not in batch or 'labels' not in batch: logger.warning(f"โš ๏ธ Invalid batch structure at batch {i}") continue - + # Move to device with validation input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + # Validate labels if torch.any(labels >= num_labels) or torch.any(labels < 0): logger.warning(f"โš ๏ธ Invalid labels in batch {i}: {labels}") continue - + # Forward pass optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 50 == 0: logger.info(f" Batch {i}/{len(go_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: logger.error(f"โŒ Error in batch {i}: {e}") continue - + # Train on journal data logger.info(" ๐Ÿ“ Training on journal data...") for i, batch in enumerate(journal_train_loader): @@ -329,98 +329,98 @@ def train_model_simple(go_texts, go_labels, journal_texts, journal_labels, num_l input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + if torch.any(labels >= num_labels) or torch.any(labels < 0): continue - + optimizer.zero_grad() outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() num_batches += 1 - + if i % 10 == 0: logger.info(f" Batch {i}/{len(journal_train_loader)}, Loss: {loss.item():.4f}") - + except Exception as e: logger.error(f"โŒ Error in journal batch {i}: {e}") continue - + # Validation logger.info(" ๐ŸŽฏ Validating...") model.eval() all_preds = [] all_labels = [] - + with torch.no_grad(): for batch in journal_val_loader: try: input_ids = batch['input_ids'].to(device) attention_mask = batch['attention_mask'].to(device) labels = batch['labels'].to(device) - + outputs = model(input_ids=input_ids, attention_mask=attention_mask) preds = torch.argmax(outputs, dim=1) - + all_preds.extend(preds.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - + except Exception as e: logger.error(f"โŒ Error in validation batch: {e}") continue - + # Calculate metrics if all_preds and all_labels: f1_macro = f1_score(all_labels, all_preds, average='macro') accuracy = accuracy_score(all_labels, all_preds) - + avg_loss = total_loss / num_batches if num_batches > 0 else 0 - + logger.info(f" ๐Ÿ“Š Epoch {epoch + 1} Results:") logger.info(f" Average Loss: {avg_loss:.4f}") logger.info(f" Validation F1 (Macro): {f1_macro:.4f}") logger.info(f" Validation Accuracy: {accuracy:.4f}") - + # Save best model if f1_macro > best_f1: best_f1 = f1_macro torch.save(model.state_dict(), 'best_simple_model.pth') logger.info(f" ๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") - + # Clear GPU cache if torch.cuda.is_available(): torch.cuda.empty_cache() - + logger.info(f"๐Ÿ† Training completed! Best F1 Score: {best_f1:.4f}") return best_f1 def main(): """Main function with comprehensive error handling.""" logger.info("๐Ÿš€ Starting bulletproof training for REQ-DL-012...") - + try: # Step 1: Validate environment if not validate_environment(): logger.error("โŒ Environment validation failed") return False - + # Step 2: Create unified label encoder label_encoder, label_to_id, id_to_label = create_unified_label_encoder() - + # Step 3: Prepare filtered data go_texts, go_labels, journal_texts, journal_labels = prepare_filtered_data(label_encoder, label_to_id) - + if go_texts is None: logger.error("โŒ Data preparation failed") return False - + # Step 4: Train model num_labels = len(label_encoder.classes_) best_f1 = train_model_simple(go_texts, go_labels, journal_texts, journal_labels, num_labels) - + # Step 5: Save results results = { 'best_f1': best_f1, @@ -429,16 +429,16 @@ def main(): 'go_samples': len(go_texts), 'journal_samples': len(journal_texts) } - + with open('simple_training_results.json', 'w') as f: json.dump(results, f, indent=2) - + logger.info("โœ… Training completed successfully!") logger.info(f"๐Ÿ“Š Final F1 Score: {best_f1:.4f}") logger.info(f"๐ŸŽฏ Target Met: {'โœ…' if best_f1 >= 0.7 else 'โŒ'}") - + return True - + except Exception as e: logger.error(f"โŒ Training failed: {e}") return False diff --git a/scripts/training/complete_simple_notebook.py b/scripts/training/complete_simple_notebook.py index 752ebcb4e..5426f65a6 100644 --- a/scripts/training/complete_simple_notebook.py +++ b/scripts/training/complete_simple_notebook.py @@ -11,11 +11,11 @@ def complete_simple_notebook(): """Add all missing components to the simple notebook.""" - + # Read the existing notebook with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Add all the missing cells new_cells = [ { @@ -465,14 +465,14 @@ def complete_simple_notebook(): ] } ] - + # Add all new cells notebook['cells'].extend(new_cells) - + # Save the completed notebook with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Completed simple notebook with ALL components!') print('๐Ÿ“‹ Added components:') print(' โœ… Focal Loss implementation') diff --git a/scripts/training/comprehensive_domain_adaptation_training.py b/scripts/training/comprehensive_domain_adaptation_training.py index 2abaa2fc5..4a1af03eb 100644 --- a/scripts/training/comprehensive_domain_adaptation_training.py +++ b/scripts/training/comprehensive_domain_adaptation_training.py @@ -67,11 +67,11 @@ class TrainingConfig: class EnvironmentManager: """Manages environment setup and dependency installation.""" - + def __init__(self): self.is_colab = self._detect_colab() self.installation_success = False - + def _detect_colab(self) -> bool: """Detect if running in Google Colab.""" try: @@ -81,11 +81,11 @@ def _detect_colab(self) -> bool: except ImportError: logger.info("โ„น๏ธ Running in local environment") return False - + def install_dependencies(self) -> bool: """Install dependencies with comprehensive error handling.""" logger.info("๐Ÿ“ฆ Installing dependencies with compatibility fixes...") - + # Define compatible versions - more conservative approach dependencies = { 'torch': '2.0.1', @@ -102,7 +102,7 @@ def install_dependencies(self) -> bool: 'accelerate': '0.20.3', 'wandb': '0.15.8' } - + try: # Step 1: Clean slate - remove conflicting packages logger.info("๐Ÿงน Cleaning existing packages...") @@ -110,7 +110,7 @@ def install_dependencies(self) -> bool: "pip", "uninstall", "torch", "torchvision", "torchaudio", "transformers", "datasets", "-y" ], capture_output=True) - + # Step 2: Install PyTorch with compatible CUDA version logger.info("๐Ÿ”ฅ Installing PyTorch with CUDA support...") result = subprocess.run([ @@ -120,22 +120,22 @@ def install_dependencies(self) -> bool: "--index-url", "https://download.pytorch.org/whl/cu118", "--no-cache-dir" ], capture_output=True, text=True, timeout=600) - + if result.returncode != 0: logger.error(f"โŒ PyTorch installation failed: {result.stderr}") return False - + # Step 3: Install Transformers with compatible version logger.info("๐Ÿค— Installing Transformers...") result = subprocess.run([ "pip", "install", f"transformers=={dependencies['transformers']}", f"datasets=={dependencies['datasets']}", "--no-cache-dir" ], capture_output=True, text=True, timeout=300) - + if result.returncode != 0: logger.error(f"โŒ Transformers installation failed: {result.stderr}") return False - + # Step 4: Install additional dependencies logger.info("๐Ÿ“š Installing additional dependencies...") result = subprocess.run([ @@ -150,11 +150,11 @@ def install_dependencies(self) -> bool: f"wandb=={dependencies['wandb']}", "--no-cache-dir" ], capture_output=True, text=True, timeout=300) - + if result.returncode != 0: logger.error(f"โŒ Additional dependencies installation failed: {result.stderr}") return False - + # Step 5: Apply numpy compatibility fix proactively logger.info("๐Ÿ”ง Applying numpy compatibility fix...") try: @@ -166,32 +166,32 @@ def broadcast_to(array, shape): logger.info(" โœ… Numpy compatibility fix applied proactively") except Exception as e: logger.warning(f"โš ๏ธ Could not apply numpy fix proactively: {e}") - + logger.info("โœ… Dependencies installed successfully") self.installation_success = True return True - + except subprocess.TimeoutExpired: logger.error("โŒ Installation timed out") return False except Exception as e: logger.error(f"โŒ Installation failed: {e}") return False - + def verify_installation(self) -> bool: """Verify that all critical packages are installed correctly.""" logger.info("๐Ÿ” Verifying installation...") - + try: import torch import transformers import datasets - + logger.info(f" PyTorch: {torch.__version__}") logger.info(f" Transformers: {transformers.__version__}") logger.info(f" Datasets: {datasets.__version__}") logger.info(f" CUDA Available: {torch.cuda.is_available()}") - + if torch.cuda.is_available(): logger.info(f" GPU: {torch.cuda.get_device_name(0)}") logger.info(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") @@ -199,7 +199,7 @@ def verify_installation(self) -> bool: logger.info(" โœ… GPU optimized for training") else: logger.warning("โš ๏ธ No GPU available. Training will be slow on CPU.") - + # Test critical imports with numpy compatibility fix try: from transformers import AutoModel, AutoTokenizer @@ -215,18 +215,18 @@ def broadcast_to(array, shape): return np.broadcast_arrays(array, np.empty(shape))[0] np.lib.stride_tricks.broadcast_to = broadcast_to logger.info(" โœ… Numpy compatibility fix applied") - + # Try imports again from transformers import AutoModel, AutoTokenizer logger.info(" โœ… Transformers imports successful after fix") else: raise e - + return True - + except Exception as e: logger.error(f" โŒ Installation verification failed: {e}") - + # Try to fix numpy compatibility issue if "broadcast_to" in str(e): logger.info("๐Ÿ”„ Attempting to fix numpy compatibility issue...") @@ -237,26 +237,26 @@ def broadcast_to(array, shape): return np.broadcast_arrays(array, np.empty(shape))[0] np.lib.stride_tricks.broadcast_to = broadcast_to logger.info("โœ… Numpy compatibility fix applied") - + # Try verification again from transformers import AutoModel, AutoTokenizer logger.info("โœ… Transformers imports successful after fix") return True except Exception as fix_error: logger.error(f"โŒ Could not fix numpy issue: {fix_error}") - + return False class RepositoryManager: """Manages repository setup and file validation.""" - + def __init__(self): self.project_root = None - + def setup_repository(self) -> bool: """Setup the SAMO-DL repository with comprehensive error handling.""" logger.info("๐Ÿ“ Setting up repository...") - + def run_command_safe(command: str, description: str) -> bool: """Execute command with comprehensive error handling.""" logger.info(f"๐Ÿ”„ {description}...") @@ -274,12 +274,12 @@ def run_command_safe(command: str, description: str) -> bool: except Exception as e: logger.error(f" โŒ {description} failed: {e}") return False - + # Clone repository if not exists if not Path('SAMO--DL').exists(): if not run_command_safe('git clone https://github.com/uelkerd/SAMO--DL.git', 'Cloning repository'): return False - + # Change to project directory try: os.chdir('SAMO--DL') @@ -288,163 +288,163 @@ def run_command_safe(command: str, description: str) -> bool: except Exception as e: logger.error(f"โŒ Failed to change directory: {e}") return False - + # Pull latest changes run_command_safe('git pull origin main', 'Pulling latest changes') - + # Verify essential files exist essential_files = [ 'data/journal_test_dataset.json', 'scripts/robust_domain_adaptation_training.py', 'README.md' ] - + missing_files = [] for file_path in essential_files: if not Path(file_path).exists(): missing_files.append(file_path) - + if missing_files: logger.error(f"โš ๏ธ Missing essential files: {missing_files}") return False - + logger.info("โœ… Repository setup completed successfully") return True class DataManager: """Manages data loading and preprocessing with comprehensive error handling.""" - + def __init__(self): self.go_emotions = None self.journal_df = None self.label_encoder = None self.num_labels = 0 - + def load_datasets(self) -> bool: """Load datasets with comprehensive error handling.""" logger.info("๐Ÿ“Š Loading datasets...") - + try: # Load GoEmotions dataset from datasets import load_dataset self.go_emotions = load_dataset("go_emotions", "simplified") logger.info("โœ… GoEmotions dataset loaded") - + # Load journal dataset with open('data/journal_test_dataset.json', 'r', encoding='utf-8') as f: journal_entries = json.load(f) - + import pandas as pd self.journal_df = pd.DataFrame(journal_entries) logger.info(f"โœ… Journal dataset loaded ({len(journal_entries)} entries)") - + return True - + except Exception as e: logger.error(f"โŒ Failed to load datasets: {e}") return False - + def prepare_label_encoder(self) -> bool: """Prepare label encoder for unified emotion classification.""" logger.info("๐Ÿงฌ Preparing label encoder...") - + try: from sklearn.preprocessing import LabelEncoder - + # Get GoEmotions labels go_train = self.go_emotions['train'] go_label_names = go_train.features['labels'].feature.names go_single_labels_int = [label[0] if label else 0 for label in go_train['labels'][:1000]] go_single_labels_str = [go_label_names[i] for i in go_single_labels_int] - + # Get journal labels journal_emotions = self.journal_df['emotion'].tolist() - + # Create unified label encoder self.label_encoder = LabelEncoder() all_emotions = list(set(go_single_labels_str) | set(journal_emotions)) self.label_encoder.fit(all_emotions) - + self.num_labels = len(self.label_encoder.classes_) logger.info(f"๐Ÿ“Š Total emotion classes: {self.num_labels}") logger.info(f"๐Ÿ“Š Classes: {list(self.label_encoder.classes_)}") - + return True - + except Exception as e: logger.error(f"โŒ Failed to prepare label encoder: {e}") return False - + def analyze_domain_gap(self) -> bool: """Analyze domain gap between GoEmotions and journal entries.""" logger.info("๐Ÿ” Analyzing domain gap...") - + try: import numpy as np - + # Get sample texts go_texts = self.go_emotions['train']['text'][:1000] journal_texts = self.journal_df['content'].tolist() - + # Analyze writing styles def analyze_style(texts, domain_name): valid_texts = [text for text in texts if text and isinstance(text, str) and len(text.strip()) > 0] - + if not valid_texts: logger.warning(f"โš ๏ธ No valid texts for {domain_name}") return None - + avg_length = np.mean([len(text.split()) for text in valid_texts]) personal_pronouns = sum(['I ' in text or 'my ' in text or 'me ' in text for text in valid_texts]) / len(valid_texts) reflection_words = sum(['think' in text.lower() or 'feel' in text.lower() or 'believe' in text.lower() for text in valid_texts]) / len(valid_texts) - + logger.info(f"{domain_name} Style Analysis:") logger.info(f" Average length: {avg_length:.1f} words") logger.info(f" Personal pronouns: {personal_pronouns:.1%}") logger.info(f" Reflection words: {reflection_words:.1%}") logger.info(f" Sample size: {len(valid_texts)} texts") - + return { 'avg_length': avg_length, 'personal_pronouns': personal_pronouns, 'reflection_words': reflection_words, 'sample_size': len(valid_texts) } - + go_analysis = analyze_style(go_texts, "GoEmotions (Reddit)") journal_analysis = analyze_style(journal_texts, "Journal Entries") - + if go_analysis and journal_analysis: logger.info("๐ŸŽฏ Key Insights:") logger.info(f"- Journal entries are {journal_analysis['avg_length']/go_analysis['avg_length']:.1f}x longer") logger.info(f"- Journal entries use {journal_analysis['personal_pronouns']/go_analysis['personal_pronouns']:.1f}x more personal pronouns") logger.info(f"- Journal entries contain {journal_analysis['reflection_words']/go_analysis['reflection_words']:.1f}x more reflection words") - + return True else: logger.error("โŒ Domain analysis failed") return False - + except Exception as e: logger.error(f"โŒ Domain analysis failed: {e}") return False class ModelManager: """Manages model architecture and initialization.""" - + def __init__(self, config: TrainingConfig): self.config = config self.model = None self.tokenizer = None self.device = None - + def setup_device(self) -> bool: """Setup device (GPU/CPU) with optimization.""" try: import torch self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - + if torch.cuda.is_available(): logger.info(f"๐Ÿš€ Using GPU: {torch.cuda.get_device_name(0)}") logger.info(f"๐Ÿ’พ GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") @@ -452,64 +452,64 @@ def setup_device(self) -> bool: torch.backends.cudnn.deterministic = False else: logger.warning("โš ๏ธ Using CPU - training will be slow") - + return True - + except Exception as e: logger.error(f"โŒ Device setup failed: {e}") return False - + def initialize_model(self, num_labels: int) -> bool: """Initialize model with comprehensive error handling.""" logger.info(f"๐Ÿ—๏ธ Initializing model with {num_labels} labels...") - + try: import torch import torch.nn as nn from transformers import AutoModel, AutoTokenizer - + # Initialize tokenizer self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_name) logger.info(f"โœ… Tokenizer loaded: {self.config.model_name}") - + # Initialize model self.model = DomainAdaptedEmotionClassifier( model_name=self.config.model_name, num_labels=num_labels, dropout=self.config.dropout ) - + # Move to device self.model = self.model.to(self.device) logger.info(f"โœ… Model moved to {self.device}") - + # Verify model parameters total_params = sum(p.numel() for p in self.model.parameters()) trainable_params = sum(p.numel() for p in self.model.parameters() if p.requires_grad) logger.info(f"๐Ÿ“Š Model parameters: {total_params:,} (trainable: {trainable_params:,})") - + return True - + except Exception as e: logger.error(f"โŒ Model initialization failed: {e}") return False class FocalLoss: """Focal Loss for addressing class imbalance in emotion detection.""" - + def __init__(self, alpha=1, gamma=2, reduction='mean'): import torch.nn as nn self.alpha = alpha self.gamma = gamma self.reduction = reduction - + def __call__(self, inputs, targets): import torch import torch.nn.functional as F ce_loss = F.cross_entropy(inputs, targets, reduction='none') pt = torch.exp(-ce_loss) focal_loss = self.alpha * (1 - pt) ** self.gamma * ce_loss - + if self.reduction == 'mean': return focal_loss.mean() elif self.reduction == 'sum': @@ -519,7 +519,7 @@ def __call__(self, inputs, targets): class DomainAdaptedEmotionClassifier: """BERT-based emotion classifier with domain adaptation capabilities.""" - + def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3): # Validate num_labels if num_labels is None: @@ -527,17 +527,17 @@ def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3) num_labels = 12 elif num_labels <= 0: raise ValueError(f"num_labels must be positive, got {num_labels}") - + logger.info(f"๐Ÿ—๏ธ Initializing DomainAdaptedEmotionClassifier with num_labels = {num_labels}") - + try: import torch.nn as nn from transformers import AutoModel - + self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(dropout) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + # Domain adaptation layer self.domain_classifier = nn.Sequential( nn.Linear(self.bert.config.hidden_size, 512), @@ -545,9 +545,9 @@ def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3) nn.Dropout(0.3), nn.Linear(512, 2) # 2 domains: GoEmotions vs Journal ) - + logger.info(f"โœ… Model initialized successfully with {num_labels} labels") - + except Exception as e: logger.error(f"โŒ Failed to initialize model: {e}") raise @@ -559,21 +559,21 @@ def forward(self, input_ids, attention_mask, domain_labels=None): # Emotion classification emotion_logits = self.classifier(self.dropout(pooled_output)) - + # Domain classification (for domain adaptation) domain_logits = self.domain_classifier(pooled_output) - + if domain_labels is not None: return emotion_logits, domain_logits return emotion_logits - + except Exception as e: logger.error(f"โŒ Forward pass failed: {e}") raise class TrainingManager: """Manages the complete training pipeline.""" - + def __init__(self, config: TrainingConfig, model_manager: ModelManager, data_manager: DataManager): self.config = config self.model_manager = model_manager @@ -583,23 +583,23 @@ def __init__(self, config: TrainingConfig, model_manager: ModelManager, data_man self.criterion = None self.best_f1 = 0.0 self.patience_counter = 0 - + def setup_training(self) -> bool: """Setup training components.""" logger.info("๐ŸŽฏ Setting up training components...") - + try: import torch from torch.optim import AdamW from transformers import get_linear_schedule_with_warmup - + # Setup optimizer self.optimizer = AdamW( self.model_manager.model.parameters(), lr=self.config.learning_rate, weight_decay=self.config.weight_decay ) - + # Setup scheduler total_steps = len(self.data_manager.go_emotions['train']) // self.config.batch_size * self.config.num_epochs self.scheduler = get_linear_schedule_with_warmup( @@ -607,30 +607,30 @@ def setup_training(self) -> bool: num_warmup_steps=self.config.warmup_steps, num_training_steps=total_steps ) - + # Setup loss function self.criterion = FocalLoss( alpha=self.config.focal_alpha, gamma=self.config.focal_gamma ) - + logger.info("โœ… Training components setup completed") return True - + except Exception as e: logger.error(f"โŒ Training setup failed: {e}") return False - + def train(self) -> bool: """Execute the complete training pipeline.""" logger.info("๐Ÿš€ Starting training pipeline...") - + try: # Training loop implementation would go here # This is a placeholder for the actual training implementation logger.info("โœ… Training pipeline ready") return True - + except Exception as e: logger.error(f"โŒ Training failed: {e}") return False @@ -639,68 +639,68 @@ def main(): """Main execution function with comprehensive error handling.""" logger.info("๐Ÿš€ Starting SAMO Deep Learning - Comprehensive Domain Adaptation Training") logger.info("=" * 80) - + # Initialize configuration config = TrainingConfig() - + # Step 1: Environment setup env_manager = EnvironmentManager() if not env_manager.install_dependencies(): logger.error("โŒ Environment setup failed") return False - + if not env_manager.verify_installation(): logger.error("โŒ Installation verification failed") return False - + # Step 2: Repository setup repo_manager = RepositoryManager() if not repo_manager.setup_repository(): logger.error("โŒ Repository setup failed") return False - + # Step 3: Data management data_manager = DataManager() if not data_manager.load_datasets(): logger.error("โŒ Data loading failed") return False - + if not data_manager.prepare_label_encoder(): logger.error("โŒ Label encoder preparation failed") return False - + if not data_manager.analyze_domain_gap(): logger.error("โŒ Domain analysis failed") return False - + # Step 4: Model management model_manager = ModelManager(config) if not model_manager.setup_device(): logger.error("โŒ Device setup failed") return False - + if not model_manager.initialize_model(data_manager.num_labels): logger.error("โŒ Model initialization failed") return False - + # Step 5: Training setup training_manager = TrainingManager(config, model_manager, data_manager) if not training_manager.setup_training(): logger.error("โŒ Training setup failed") return False - + # Step 6: Execute training if not training_manager.train(): logger.error("โŒ Training execution failed") return False - + logger.info("๐ŸŽ‰ Training pipeline completed successfully!") logger.info("๐Ÿ“‹ Next steps:") logger.info(" 1. Evaluate model performance") logger.info(" 2. Save best model") logger.info(" 3. Generate performance report") logger.info(" 4. Update PRD with results") - + return True if __name__ == "__main__": diff --git a/scripts/training/create_bulletproof_colab_notebook.py b/scripts/training/create_bulletproof_colab_notebook.py index 66f7d214a..1fd8a8a9f 100644 --- a/scripts/training/create_bulletproof_colab_notebook.py +++ b/scripts/training/create_bulletproof_colab_notebook.py @@ -11,7 +11,7 @@ def create_bulletproof_colab_notebook(): """Create the bulletproof Colab notebook content""" - + notebook_content = { "cells": [ { @@ -695,11 +695,11 @@ def create_bulletproof_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Write notebook to file with open('notebooks/BULLETPROOF_COMBINED_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook_content, f, indent=2) - + print("โœ… Bulletproof notebook created: notebooks/BULLETPROOF_COMBINED_TRAINING_COLAB.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_colab_expanded_training.py b/scripts/training/create_colab_expanded_training.py index 59cf5ad5e..d1447b54d 100644 --- a/scripts/training/create_colab_expanded_training.py +++ b/scripts/training/create_colab_expanded_training.py @@ -5,7 +5,7 @@ def create_colab_notebook(): """Create a complete Colab notebook for expanded training.""" - + notebook_content = '''{ "cells": [ { @@ -720,11 +720,11 @@ def create_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 }''' - + # Save the notebook with open('notebooks/expanded_dataset_training.ipynb', 'w') as f: f.write(notebook_content) - + print("โœ… Created Colab notebook: notebooks/expanded_dataset_training.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_colab_notebook.py b/scripts/training/create_colab_notebook.py index 44888870b..a1e053060 100644 --- a/scripts/training/create_colab_notebook.py +++ b/scripts/training/create_colab_notebook.py @@ -7,7 +7,7 @@ def create_colab_notebook(): """Create the domain adaptation GPU training notebook.""" - + notebook = { "cells": [ { @@ -657,12 +657,12 @@ def create_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Write the notebook to file notebook_path = "notebooks/domain_adaptation_gpu_training.ipynb" with open(notebook_path, 'w') as f: json.dump(notebook, f, indent=1) - + print(f"โœ… Created Colab notebook: {notebook_path}") print("๐Ÿ“‹ Notebook includes:") print(" - GPU environment setup") diff --git a/scripts/training/create_comprehensive_notebook.py b/scripts/training/create_comprehensive_notebook.py index 53aeac663..c2cb5987b 100644 --- a/scripts/training/create_comprehensive_notebook.py +++ b/scripts/training/create_comprehensive_notebook.py @@ -11,7 +11,7 @@ def create_comprehensive_notebook(): """Create a comprehensive notebook with all advanced features.""" - + notebook_content = { "cells": [ { @@ -582,12 +582,12 @@ def create_comprehensive_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook output_path = "notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb" with open(output_path, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Created comprehensive notebook: {output_path}") print("๐Ÿ“‹ Features included:") print(" โœ… Comprehensive dataset (240 base + augmentation)") @@ -596,7 +596,7 @@ def create_comprehensive_notebook(): print(" โœ… Model architecture fixes") print(" โœ… All advanced features (to be added)") print("\\n๐Ÿš€ This will be a full-featured notebook!") - + return output_path if __name__ == "__main__": diff --git a/scripts/training/create_corrected_specialized_notebook.py b/scripts/training/create_corrected_specialized_notebook.py index b3be8ffb6..a96832eee 100644 --- a/scripts/training/create_corrected_specialized_notebook.py +++ b/scripts/training/create_corrected_specialized_notebook.py @@ -9,7 +9,7 @@ def create_corrected_notebook(): """Create a corrected notebook with proper specialized model usage""" - + notebook_content = '''{ "cells": [ { @@ -619,12 +619,12 @@ def create_corrected_notebook(): "nbformat": 4, "nbformat_minor": 4 }''' - + # Save the notebook notebook_path = Path(__file__).parent.parent / 'notebooks' / 'CORRECTED_SPECIALIZED_TRAINING.ipynb' with open(notebook_path, 'w') as f: f.write(notebook_content) - + print(f"โœ… Created corrected specialized notebook: {notebook_path}") print(f"๐Ÿ“‹ Key improvements:") print(f" 1. Verifies access to j-hartmann/emotion-english-distilroberta-base") diff --git a/scripts/training/create_emotion_specialized_notebook.py b/scripts/training/create_emotion_specialized_notebook.py index 031cb1c7e..85447eeb2 100644 --- a/scripts/training/create_emotion_specialized_notebook.py +++ b/scripts/training/create_emotion_specialized_notebook.py @@ -10,7 +10,7 @@ def create_emotion_specialized_notebook(): """Create the emotion specialized notebook content""" - + notebook_content = { "cells": [ { @@ -481,10 +481,10 @@ def create_emotion_specialized_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + with open('notebooks/EMOTION_SPECIALIZED_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook_content, f, indent=2) - + print("โœ… Emotion specialized notebook created: notebooks/EMOTION_SPECIALIZED_TRAINING_COLAB.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_final_bulletproof_notebook.py b/scripts/training/create_final_bulletproof_notebook.py index d0359a26d..768e97e99 100644 --- a/scripts/training/create_final_bulletproof_notebook.py +++ b/scripts/training/create_final_bulletproof_notebook.py @@ -7,7 +7,7 @@ def create_final_bulletproof_notebook(): """Create a Colab notebook that handles all dependency and path issues""" - + notebook = { "cells": [ { @@ -712,12 +712,12 @@ def create_final_bulletproof_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save notebook output_path = 'notebooks/expanded_dataset_training_final.ipynb' with open(output_path, 'w') as f: json.dump(notebook, f, indent=2) - + print(f"โœ… Created final bulletproof notebook: {output_path}") print("๐Ÿ”ง All issues fixed:") print(" - Fixed NumPy installation command (removed extra quotes)") diff --git a/scripts/training/create_final_colab_notebook.py b/scripts/training/create_final_colab_notebook.py index a400b0c09..ca299cfee 100644 --- a/scripts/training/create_final_colab_notebook.py +++ b/scripts/training/create_final_colab_notebook.py @@ -10,7 +10,7 @@ def create_colab_notebook(): """Create the final Colab notebook content""" - + notebook_content = { "cells": [ { @@ -459,20 +459,20 @@ def create_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + return notebook_content def main(): """Create the notebook file""" print("๐Ÿš€ Creating final Colab notebook...") - + notebook_content = create_colab_notebook() - + # Save to file output_file = "notebooks/FINAL_COMBINED_TRAINING_COLAB.ipynb" with open(output_file, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Notebook created: {output_file}") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_fixed_bulletproof_notebook.py b/scripts/training/create_fixed_bulletproof_notebook.py index 219cd8c78..9dfe86f0a 100644 --- a/scripts/training/create_fixed_bulletproof_notebook.py +++ b/scripts/training/create_fixed_bulletproof_notebook.py @@ -10,7 +10,7 @@ def create_fixed_bulletproof_notebook(): """Create the fixed bulletproof notebook content""" - + notebook_content = { "cells": [ { @@ -450,10 +450,10 @@ def create_fixed_bulletproof_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + with open('notebooks/FIXED_BULLETPROOF_COMBINED_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook_content, f, indent=2) - + print("โœ… Fixed bulletproof notebook created: notebooks/FIXED_BULLETPROOF_COMBINED_TRAINING_COLAB.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_fixed_colab_notebook.py b/scripts/training/create_fixed_colab_notebook.py index f30f8ddca..694c5fe2d 100644 --- a/scripts/training/create_fixed_colab_notebook.py +++ b/scripts/training/create_fixed_colab_notebook.py @@ -10,7 +10,7 @@ def create_fixed_colab_notebook(): """Create the fixed Colab notebook content""" - + notebook_content = { "cells": [ { @@ -439,11 +439,11 @@ def create_fixed_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Write notebook to file with open('notebooks/FIXED_COMBINED_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook_content, f, indent=2) - + print("โœ… Fixed notebook created: notebooks/FIXED_COMBINED_TRAINING_COLAB.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_fixed_notebook.py b/scripts/training/create_fixed_notebook.py index db7a1502e..775a57201 100644 --- a/scripts/training/create_fixed_notebook.py +++ b/scripts/training/create_fixed_notebook.py @@ -11,7 +11,7 @@ def create_fixed_notebook(): """Create a fixed notebook with proper JSON escaping""" - + # Create the notebook structure notebook = { "cells": [ @@ -623,12 +623,12 @@ def create_fixed_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook with proper JSON formatting notebook_path = Path(__file__).parent.parent / 'notebooks' / 'FIXED_SPECIALIZED_TRAINING.ipynb' with open(notebook_path, 'w') as f: json.dump(notebook, f, indent=1) - + print(f"โœ… Created fixed specialized notebook: {notebook_path}") print(f"๐Ÿ“‹ Key improvements:") print(f" 1. Proper JSON formatting (no syntax errors)") diff --git a/scripts/training/create_fixed_specialized_training_notebook.py b/scripts/training/create_fixed_specialized_training_notebook.py index 874bdccfe..30c3a8552 100644 --- a/scripts/training/create_fixed_specialized_training_notebook.py +++ b/scripts/training/create_fixed_specialized_training_notebook.py @@ -14,7 +14,7 @@ def create_fixed_notebook(): """Create a corrected training notebook with proper configuration preservation.""" - + notebook_content = { "cells": [ { @@ -663,12 +663,12 @@ def create_fixed_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook output_path = "notebooks/FIXED_SPECIALIZED_TRAINING_CONFIG_PRESERVATION.ipynb" with open(output_path, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Created fixed training notebook: {output_path}") print("\n๐Ÿ”ง Key fixes implemented:") print("1. โœ… Explicit emotion label mapping before training") @@ -676,7 +676,7 @@ def create_fixed_notebook(): print("3. โœ… Configuration re-setting before saving") print("4. โœ… Saved configuration verification") print("5. โœ… Comprehensive error checking") - + return output_path if __name__ == "__main__": diff --git a/scripts/training/create_improved_expanded_notebook.py b/scripts/training/create_improved_expanded_notebook.py index 84bb4fa86..7ed871b27 100644 --- a/scripts/training/create_improved_expanded_notebook.py +++ b/scripts/training/create_improved_expanded_notebook.py @@ -8,7 +8,7 @@ def create_improved_notebook(): """Create an improved version of the expanded training notebook.""" - + notebook = { "cells": [ { @@ -748,11 +748,11 @@ def create_improved_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the improved notebook with open('notebooks/expanded_dataset_training_improved.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print("โœ… Improved notebook created: 'notebooks/expanded_dataset_training_improved.ipynb'") print("๐Ÿ“‹ Key improvements:") print(" - Fixed JSON syntax errors") diff --git a/scripts/training/create_minimal_working_notebook.py b/scripts/training/create_minimal_working_notebook.py index 215da793b..93d7423c8 100644 --- a/scripts/training/create_minimal_working_notebook.py +++ b/scripts/training/create_minimal_working_notebook.py @@ -11,7 +11,7 @@ def create_minimal_notebook(): """Create a minimal working notebook.""" - + notebook_content = { "cells": [ { @@ -362,12 +362,12 @@ def create_minimal_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook output_path = "notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb" with open(output_path, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Created minimal working notebook: {output_path}") print("๐Ÿ“‹ Features:") print(" โœ… Ultra-minimal training arguments") @@ -375,7 +375,7 @@ def create_minimal_notebook(): print(" โœ… Basic training and evaluation") print(" โœ… Model saving with verification") print("\\n๐Ÿš€ This should work in ANY transformers version!") - + return output_path if __name__ == "__main__": diff --git a/scripts/training/create_model_ensemble_notebook.py b/scripts/training/create_model_ensemble_notebook.py index a5ee53d59..05d545f42 100644 --- a/scripts/training/create_model_ensemble_notebook.py +++ b/scripts/training/create_model_ensemble_notebook.py @@ -10,7 +10,7 @@ def create_model_ensemble_notebook(): """Create the model ensemble notebook content""" - + notebook_content = { "cells": [ { @@ -656,10 +656,10 @@ def create_model_ensemble_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + with open('notebooks/MODEL_ENSEMBLE_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook_content, f, indent=2) - + print("โœ… Model ensemble notebook created: notebooks/MODEL_ENSEMBLE_TRAINING_COLAB.ipynb") print("๐Ÿ“‹ Instructions:") print(" 1. Download the notebook file") diff --git a/scripts/training/create_simple_ultimate_notebook.py b/scripts/training/create_simple_ultimate_notebook.py index 91af37aa3..d19a5a051 100644 --- a/scripts/training/create_simple_ultimate_notebook.py +++ b/scripts/training/create_simple_ultimate_notebook.py @@ -11,7 +11,7 @@ def create_simple_notebook(): """Create a simplified ultimate notebook.""" - + notebook_content = { "cells": [ { @@ -396,12 +396,12 @@ def create_simple_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook output_path = "notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb" with open(output_path, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Created simple ultimate notebook: {output_path}") print("๐Ÿ“‹ Features included:") print(" โœ… Configuration preservation") @@ -410,7 +410,7 @@ def create_simple_notebook(): print(" โœ… Data augmentation") print(" โœ… Simple approach (no datasets library)") print(" โœ… Advanced validation (to be added)") - + return output_path if __name__ == "__main__": diff --git a/scripts/training/create_ultimate_bulletproof_notebook.py b/scripts/training/create_ultimate_bulletproof_notebook.py index ccba22de0..0b4d48d65 100644 --- a/scripts/training/create_ultimate_bulletproof_notebook.py +++ b/scripts/training/create_ultimate_bulletproof_notebook.py @@ -19,7 +19,7 @@ def create_ultimate_notebook(): """Create the ultimate bulletproof training notebook.""" - + notebook_content = { "cells": [ { @@ -400,12 +400,12 @@ def create_ultimate_notebook(): "nbformat": 4, "nbformat_minor": 4 } - + # Save the notebook output_path = "notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb" with open(output_path, 'w') as f: json.dump(notebook_content, f, indent=2) - + print(f"โœ… Created ultimate bulletproof notebook: {output_path}") print("๐Ÿ“‹ Features included:") print(" โœ… Configuration preservation") @@ -413,7 +413,7 @@ def create_ultimate_notebook(): print(" โœ… Class weighting (to be added)") print(" โœ… Data augmentation") print(" โœ… Advanced validation (to be added)") - + return output_path if __name__ == "__main__": diff --git a/scripts/training/debug_colab_compatibility.py b/scripts/training/debug_colab_compatibility.py index 5f3b9b784..37fdfcc5f 100644 --- a/scripts/training/debug_colab_compatibility.py +++ b/scripts/training/debug_colab_compatibility.py @@ -35,7 +35,7 @@ def check_python_version(): print("๐Ÿ Checking Python version...") version = sys.version_info print(f"Python {version.major}.{version.minor}.{version.micro}") - + if version.major == 3 and version.minor >= 8: print("โœ… Python version is compatible") return True @@ -46,11 +46,11 @@ def check_python_version(): def check_gpu_availability(): """Check GPU availability and CUDA compatibility.""" print("๐Ÿ–ฅ๏ธ Checking GPU availability...") - + try: import torch print(f"PyTorch version: {torch.__version__}") - + if torch.cuda.is_available(): print(f"โœ… CUDA available") print(f"GPU: {torch.cuda.get_device_name(0)}") @@ -67,24 +67,24 @@ def check_gpu_availability(): def check_pytorch_installation(): """Check PyTorch installation and compatibility.""" print("๐Ÿ” Checking PyTorch installation...") - + try: import torch print(f"PyTorch: {torch.__version__}") - + # Test basic operations x = torch.randn(2, 2) y = torch.randn(2, 2) z = torch.mm(x, y) print("โœ… Basic PyTorch operations work") - + # Test CUDA operations if available if torch.cuda.is_available(): x_cuda = x.cuda() y_cuda = y.cuda() z_cuda = torch.mm(x_cuda, y_cuda) print("โœ… CUDA operations work") - + return True except Exception as e: print(f"โŒ PyTorch test failed: {e}") @@ -93,20 +93,20 @@ def check_pytorch_installation(): def check_transformers_installation(): """Check Transformers installation and compatibility.""" print("๐Ÿค— Checking Transformers installation...") - + try: import transformers print(f"Transformers: {transformers.__version__}") - + # Test basic imports from transformers import AutoModel, AutoTokenizer print("โœ… Transformers imports successful") - + # Test model loading tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") model = AutoModel.from_pretrained("bert-base-uncased") print("โœ… Model loading successful") - + return True except Exception as e: print(f"โŒ Transformers test failed: {e}") @@ -115,17 +115,17 @@ def check_transformers_installation(): def check_triton_compatibility(): """Check Triton compatibility (common source of errors).""" print("๐Ÿ”ง Checking Triton compatibility...") - + try: import torch - + # Check if Triton is available if hasattr(torch, 'sparse') and hasattr(torch.sparse, '_triton_ops_meta'): print("โœ… Triton ops available") return True else: print("โš ๏ธ Triton ops not available - this may cause issues") - + # Try to import triton directly try: import triton @@ -141,22 +141,22 @@ def check_triton_compatibility(): def fix_pytorch_installation(): """Fix PyTorch installation issues.""" print("๐Ÿ”ง Fixing PyTorch installation...") - + # Uninstall existing PyTorch success, _ = run_command( "pip uninstall torch torchvision torchaudio -y", "Uninstalling existing PyTorch" ) - + if not success: print("โš ๏ธ Failed to uninstall PyTorch") - + # Install compatible PyTorch success, _ = run_command( "pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu118", "Installing compatible PyTorch" ) - + if success: print("โœ… PyTorch installation fixed") return True @@ -167,22 +167,22 @@ def fix_pytorch_installation(): def fix_transformers_installation(): """Fix Transformers installation issues.""" print("๐Ÿ”ง Fixing Transformers installation...") - + # Uninstall existing Transformers success, _ = run_command( "pip uninstall transformers -y", "Uninstalling existing Transformers" ) - + if not success: print("โš ๏ธ Failed to uninstall Transformers") - + # Install compatible Transformers success, _ = run_command( "pip install transformers==4.30.0", "Installing compatible Transformers" ) - + if success: print("โœ… Transformers installation fixed") return True @@ -193,31 +193,31 @@ def fix_transformers_installation(): def test_model_initialization(): """Test model initialization to catch common errors.""" print("๐Ÿงช Testing model initialization...") - + try: import torch from transformers import AutoModel, AutoTokenizer - + # Test tokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") print("โœ… Tokenizer loaded") - + # Test model model = AutoModel.from_pretrained("bert-base-uncased") print("โœ… Model loaded") - + # Test forward pass inputs = tokenizer("Hello world", return_tensors="pt") outputs = model(**inputs) print("โœ… Forward pass successful") - + # Test GPU if available if torch.cuda.is_available(): model = model.cuda() inputs = {k: v.cuda() for k, v in inputs.items()} outputs = model(**inputs) print("โœ… GPU forward pass successful") - + return True except Exception as e: print(f"โŒ Model initialization failed: {e}") @@ -228,20 +228,20 @@ def test_model_initialization(): def check_dataset_loading(): """Check dataset loading capabilities.""" print("๐Ÿ“Š Checking dataset loading...") - + try: from datasets import load_dataset - + # Test loading GoEmotions dataset = load_dataset("go_emotions", "simplified") print(f"โœ… GoEmotions dataset loaded: {len(dataset['train'])} samples") - + # Test journal dataset import json with open('data/journal_test_dataset.json', 'r') as f: journal_data = json.load(f) print(f"โœ… Journal dataset loaded: {len(journal_data)} samples") - + return True except Exception as e: print(f"โŒ Dataset loading failed: {e}") @@ -250,7 +250,7 @@ def check_dataset_loading(): def generate_compatibility_report(): """Generate a comprehensive compatibility report.""" print("๐Ÿ“‹ Generating compatibility report...") - + report = { "python_version": check_python_version(), "gpu_available": check_gpu_availability(), @@ -260,18 +260,18 @@ def generate_compatibility_report(): "model_initialization": test_model_initialization(), "dataset_loading": check_dataset_loading() } - + print("\n" + "="*50) print("COMPATIBILITY REPORT") print("="*50) - + for test, result in report.items(): status = "โœ… PASS" if result else "โŒ FAIL" print(f"{test.replace('_', ' ').title()}: {status}") - + all_passed = all(report.values()) print(f"\nOverall Status: {'โœ… READY' if all_passed else 'โŒ NEEDS FIXES'}") - + if not all_passed: print("\n๐Ÿ”ง Recommended fixes:") if not report["pytorch_working"]: @@ -280,37 +280,37 @@ def generate_compatibility_report(): print("- Run: fix_transformers_installation()") if not report["triton_compatible"]: print("- Consider reinstalling PyTorch with Triton support") - + return report def main(): """Main debugging function.""" print("๐Ÿš€ SAMO Deep Learning - Colab Compatibility Debug") print("="*50) - + # Check if we're in Colab try: import google.colab print("โœ… Running in Google Colab") except ImportError: print("โš ๏ธ Not running in Google Colab") - + # Generate report report = generate_compatibility_report() - + # Offer fixes if not report["pytorch_working"]: print("\n๐Ÿ”ง Would you like to fix PyTorch installation? (y/n)") response = input().lower() if response == 'y': fix_pytorch_installation() - + if not report["transformers_working"]: print("\n๐Ÿ”ง Would you like to fix Transformers installation? (y/n)") response = input().lower() if response == 'y': fix_transformers_installation() - + print("\n๐ŸŽฏ Debug complete!") print("๐Ÿ“‹ If issues persist, try:") print(" 1. Restart Colab runtime") diff --git a/scripts/training/debug_training_loss.py b/scripts/training/debug_training_loss.py index 7ff4f581f..3731f1953 100644 --- a/scripts/training/debug_training_loss.py +++ b/scripts/training/debug_training_loss.py @@ -254,7 +254,7 @@ def main(): dev_mode=True ) datasets = trainer.prepare_data(dev_mode=True) - + logits, predictions, labels = debug_model_outputs(datasets) if logits is None: return False diff --git a/scripts/training/final_combined_training.py b/scripts/training/final_combined_training.py index 0d278c1a2..4321c1afe 100644 --- a/scripts/training/final_combined_training.py +++ b/scripts/training/final_combined_training.py @@ -36,14 +36,14 @@ def load_combined_dataset(): """Load and combine journal and CMU-MOSEI datasets""" print("๐Ÿ“Š Loading combined dataset...") - + combined_samples = [] - + # Load original journal dataset (150 high-quality samples) try: with open('data/journal_test_dataset.json', 'r') as f: journal_data = json.load(f) - + for item in journal_data: combined_samples.append({ 'text': item['text'], @@ -53,12 +53,12 @@ def load_combined_dataset(): print(f"โœ… Loaded {len(journal_data)} journal samples") except Exception as e: print(f"โš ๏ธ Could not load journal data: {e}") - + # Load CMU-MOSEI dataset try: with open('data/cmu_mosei_balanced_dataset.json', 'r') as f: cmu_data = json.load(f) - + for item in cmu_data: combined_samples.append({ 'text': item['text'], @@ -68,16 +68,16 @@ def load_combined_dataset(): print(f"โœ… Loaded {len(cmu_data)} CMU-MOSEI samples") except Exception as e: print(f"โš ๏ธ Could not load CMU-MOSEI data: {e}") - + # Load expanded journal dataset as backup try: with open('data/expanded_journal_dataset.json', 'r') as f: expanded_data = json.load(f) - + # Only use a subset to avoid synthetic data issues subset_size = min(200, len(expanded_data)) selected_samples = np.random.choice(expanded_data, size=subset_size, replace=False) - + for item in selected_samples: combined_samples.append({ 'text': item['text'], @@ -87,37 +87,37 @@ def load_combined_dataset(): print(f"โœ… Loaded {subset_size} expanded journal samples") except Exception as e: print(f"โš ๏ธ Could not load expanded journal data: {e}") - + print(f"๐Ÿ“Š Total combined samples: {len(combined_samples)}") - + # Show emotion distribution emotion_counts = {} for sample in combined_samples: emotion = sample['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - + print("๐Ÿ“Š Emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") - + return combined_samples class EmotionDataset(Dataset): """Custom dataset for emotion classification""" - + def __init__(self, texts, labels, tokenizer, max_length=128): self.texts = texts self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = str(self.texts[idx]) label = self.labels[idx] - + encoding = self.tokenizer( text, truncation=True, @@ -125,7 +125,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -136,10 +136,10 @@ def compute_metrics(eval_pred): """Compute F1 score and accuracy""" predictions, labels = eval_pred predictions = np.argmax(predictions, axis=1) - + f1 = f1_score(labels, predictions, average='weighted') accuracy = accuracy_score(labels, predictions) - + return { 'f1': f1, 'accuracy': accuracy @@ -151,48 +151,48 @@ def main(): print("๐Ÿ”ง Current Best: 67%") print("๐Ÿ“ˆ Expected Improvement: 8-18%") print() - + # Load combined dataset samples = load_combined_dataset() - + if not samples: print("โŒ No samples loaded!") return - + # Prepare data texts = [sample['text'] for sample in samples] emotions = [sample['emotion'] for sample in samples] - + # Encode labels label_encoder = LabelEncoder() labels = label_encoder.fit_transform(emotions) - + print(f"๐ŸŽฏ Number of labels: {len(label_encoder.classes_)}") print(f"๐Ÿ“Š Labels: {list(label_encoder.classes_)}") - + # Split data train_texts, test_texts, train_labels, test_labels = train_test_split( texts, labels, test_size=0.2, random_state=42, stratify=labels ) - + print(f"๐Ÿ“ˆ Training samples: {len(train_texts)}") print(f"๐Ÿงช Test samples: {len(test_labels)}") - + # Initialize tokenizer and model print("๐Ÿ”ง Initializing model...") model_name = "bert-base-uncased" tokenizer = AutoTokenizer.from_pretrained(model_name) - + model = AutoModelForSequenceClassification.from_pretrained( model_name, num_labels=len(label_encoder.classes_), problem_type="single_label_classification" ) - + # Create datasets train_dataset = EmotionDataset(train_texts, train_labels, tokenizer) test_dataset = EmotionDataset(test_texts, test_labels, tokenizer) - + # Training arguments optimized for performance training_args = TrainingArguments( output_dir="./emotion_model_combined", @@ -216,7 +216,7 @@ def main(): learning_rate=2e-5, # Optimal learning rate gradient_accumulation_steps=2, # Effective batch size = 32 ) - + # Initialize trainer trainer = Trainer( model=model, @@ -226,22 +226,22 @@ def main(): compute_metrics=compute_metrics, callbacks=[EarlyStoppingCallback(early_stopping_patience=3)] ) - + # Train model print("๐Ÿš€ Starting training...") trainer.train() - + # Evaluate final model print("๐Ÿ“Š Evaluating final model...") results = trainer.evaluate() - + print(f"๐Ÿ† Final F1 Score: {results['eval_f1']:.4f} ({results['eval_f1']*100:.2f}%)") print(f"๐ŸŽฏ Target achieved: {'โœ… YES!' if results['eval_f1'] >= 0.75 else 'โŒ Not yet'}") - + # Save model trainer.save_model("./emotion_model_final_combined") print("๐Ÿ’พ Model saved to ./emotion_model_final_combined") - + # Test on sample texts print("\n๐Ÿงช Testing on sample texts...") test_texts = [ @@ -251,7 +251,7 @@ def main(): "I'm grateful for all the support.", "I'm tired and need some rest." ] - + model.eval() with torch.no_grad(): for text in test_texts: @@ -260,12 +260,12 @@ def main(): probs = torch.softmax(outputs.logits, dim=1) predicted_label = torch.argmax(probs, dim=1).item() confidence = torch.max(probs).item() - + predicted_emotion = label_encoder.inverse_transform([predicted_label])[0] print(f"Text: {text}") print(f"Predicted: {predicted_emotion} (confidence: {confidence:.3f})") print() - + print("๐ŸŽ‰ Training completed!") print(f"๐Ÿ“ˆ Final F1 Score: {results['eval_f1']*100:.2f}%") print(f"๐ŸŽฏ Target: 75-85%") diff --git a/scripts/training/final_expanded_training.py b/scripts/training/final_expanded_training.py index 435792b4d..3556704f9 100644 --- a/scripts/training/final_expanded_training.py +++ b/scripts/training/final_expanded_training.py @@ -65,14 +65,14 @@ def __init__(self, texts, labels, tokenizer, max_length=128): self.labels = labels self.tokenizer = tokenizer self.max_length = max_length - + def __len__(self): return len(self.texts) - + def __getitem__(self, idx): text = str(self.texts[idx]) label = self.labels[idx] - + encoding = self.tokenizer( text, truncation=True, @@ -80,7 +80,7 @@ def __getitem__(self, idx): max_length=self.max_length, return_tensors='pt' ) - + return { 'input_ids': encoding['input_ids'].flatten(), 'attention_mask': encoding['attention_mask'].flatten(), @@ -128,10 +128,10 @@ def __getitem__(self, idx): def compute_metrics(eval_pred): predictions, labels = eval_pred predictions = np.argmax(predictions, axis=1) - + f1 = f1_score(labels, predictions, average='weighted') accuracy = accuracy_score(labels, predictions) - + return { 'f1': f1, 'accuracy': accuracy @@ -190,7 +190,7 @@ def compute_metrics(eval_pred): for i, (text, expected) in enumerate(zip(test_samples, expected_emotions), 1): # Tokenize inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True, max_length=128) - + # Predict with torch.no_grad(): outputs = model(**inputs) @@ -198,17 +198,17 @@ def compute_metrics(eval_pred): predicted_idx = torch.argmax(probabilities, dim=1).item() confidence = probabilities[0][predicted_idx].item() predicted_emotion = label_encoder.inverse_transform([predicted_idx])[0] - + # Get top 3 predictions top_3_indices = torch.topk(probabilities[0], 3).indices top_3_emotions = label_encoder.inverse_transform(top_3_indices.cpu().numpy()) top_3_probs = torch.topk(probabilities[0], 3).values.cpu().numpy() - + # Check if correct is_correct = predicted_emotion == expected if is_correct: correct_predictions += 1 - + print(f"{i}. Text: {text}") print(f" Predicted: {predicted_emotion} (confidence: {confidence:.3f})") print(f" Expected: {expected}") diff --git a/scripts/training/fix_imports_in_notebook.py b/scripts/training/fix_imports_in_notebook.py index b65d8d307..3dfcbc164 100644 --- a/scripts/training/fix_imports_in_notebook.py +++ b/scripts/training/fix_imports_in_notebook.py @@ -11,11 +11,11 @@ def fix_imports(): """Add missing imports to the ultimate notebook.""" - + # Read the existing notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Find the imports cell and update it for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'import torch' in ''.join(cell['source']): @@ -38,11 +38,11 @@ def fix_imports(): "print(f'CUDA available: {torch.cuda.is_available()}')" ] break - + # Save the updated notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Fixed imports in ultimate notebook!') print('๐Ÿ“‹ Added missing imports:') print(' โœ… f1_score, accuracy_score, precision_score, recall_score') diff --git a/scripts/training/fix_notebook_json.py b/scripts/training/fix_notebook_json.py index c3ff9a2f0..2e82c9d76 100644 --- a/scripts/training/fix_notebook_json.py +++ b/scripts/training/fix_notebook_json.py @@ -7,11 +7,11 @@ def fix_notebook_json(): """Fix JSON syntax errors in the notebook.""" - + # Read the notebook as text with open('notebooks/expanded_dataset_training.ipynb', 'r') as f: content = f.read() - + # Fix unescaped quotes in strings # Replace "I'm" with "I\\'m" and similar patterns content = re.sub(r'"I\'m', r'"I\\\'m', content) @@ -32,16 +32,16 @@ def fix_notebook_json(): content = re.sub(r'"shouldn\'t', r'"shouldn\\\'t', content) content = re.sub(r'"mightn\'t', r'"mightn\\\'t', content) content = re.sub(r'"mustn\'t', r'"mustn\\\'t', content) - + # Fix other common contractions content = re.sub(r'"(\w+)\'(\w+)"', r'"\\1\\\'\\2"', content) - + # Write the fixed content with open('notebooks/expanded_dataset_training_fixed.ipynb', 'w') as f: f.write(content) - + print("โœ… Fixed notebook saved as 'notebooks/expanded_dataset_training_fixed.ipynb'") - + # Test if the JSON is valid try: import json diff --git a/scripts/training/fix_preprocessing_in_notebook.py b/scripts/training/fix_preprocessing_in_notebook.py index 1bc9eae51..5190e68b8 100644 --- a/scripts/training/fix_preprocessing_in_notebook.py +++ b/scripts/training/fix_preprocessing_in_notebook.py @@ -11,11 +11,11 @@ def fix_preprocessing(): """Fix the preprocessing function in the ultimate notebook.""" - + # Read the existing notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Find and replace the preprocessing cell for i, cell in enumerate(notebook['cells']): if cell['cell_type'] == 'code' and 'def preprocess_function' in ''.join(cell['source']): @@ -67,7 +67,7 @@ def fix_preprocessing(): "print('โœ… Data structure verified!')" ] break - + # Also add a data collator cell after the training arguments data_collator_cell = { "cell_type": "markdown", @@ -76,7 +76,7 @@ def fix_preprocessing(): "## ๐Ÿ”ง DATA COLLATOR" ] } - + data_collator_code = { "cell_type": "code", "execution_count": None, @@ -95,7 +95,7 @@ def fix_preprocessing(): "print('โœ… Data collator configured')" ] } - + # Find the training arguments cell and add the data collator after it for i, cell in enumerate(notebook['cells']): if cell['cell_type'] == 'code' and 'TrainingArguments(' in ''.join(cell['source']): @@ -103,7 +103,7 @@ def fix_preprocessing(): notebook['cells'].insert(i + 2, data_collator_cell) notebook['cells'].insert(i + 3, data_collator_code) break - + # Update the trainer initialization to include the data collator for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'WeightedLossTrainer(' in ''.join(cell['source']): @@ -126,11 +126,11 @@ def fix_preprocessing(): "print('โœ… Trainer initialized with focal loss and class weighting')" ] break - + # Save the updated notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Fixed preprocessing in ultimate notebook!') print('๐Ÿ“‹ Changes made:') print(' โœ… Updated preprocessing function with proper tokenization') diff --git a/scripts/training/fix_training_arguments.py b/scripts/training/fix_training_arguments.py index a9dcebb1b..a286c3ea3 100644 --- a/scripts/training/fix_training_arguments.py +++ b/scripts/training/fix_training_arguments.py @@ -11,11 +11,11 @@ def fix_training_arguments(): """Fix the training arguments in the simple notebook.""" - + # Read the existing notebook with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + # Find and replace the training arguments cell for cell in notebook['cells']: if cell['cell_type'] == 'code' and 'TrainingArguments(' in ''.join(cell['source']): @@ -43,11 +43,11 @@ def fix_training_arguments(): "print('โœ… Training arguments configured')" ] break - + # Save the updated notebook with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print('โœ… Fixed training arguments in simple notebook!') print('๐Ÿ“‹ Changes made:') print(' โœ… Removed evaluation_strategy parameter') diff --git a/scripts/training/fixed_focal_training.py b/scripts/training/fixed_focal_training.py index 2c5becf52..f64fadc1c 100644 --- a/scripts/training/fixed_focal_training.py +++ b/scripts/training/fixed_focal_training.py @@ -72,7 +72,7 @@ def create_proper_training_data(): # Create diverse training data with proper emotion labels training_data = [] - + # Joy examples joy_examples = [ "I'm so happy today! Everything is going great!", @@ -86,7 +86,7 @@ def create_proper_training_data(): "I'm delighted with how things turned out!", "This brings me so much joy!" ] - + # Sadness examples sadness_examples = [ "I'm feeling really down today.", @@ -100,7 +100,7 @@ def create_proper_training_data(): "Everything is going wrong.", "I'm so upset about this situation." ] - + # Anger examples anger_examples = [ "I'm so angry about this!", @@ -114,7 +114,7 @@ def create_proper_training_data(): "This is driving me crazy!", "I'm really annoyed and angry!" ] - + # Fear examples fear_examples = [ "I'm really scared about what might happen.", @@ -128,7 +128,7 @@ def create_proper_training_data(): "I'm terrified of the outcome.", "This is making me really nervous." ] - + # Love examples love_examples = [ "I love you so much!", @@ -142,7 +142,7 @@ def create_proper_training_data(): "I love spending time with you.", "You're the love of my life." ] - + # Disgust examples disgust_examples = [ "This is absolutely disgusting!", @@ -156,7 +156,7 @@ def create_proper_training_data(): "This is really sickening.", "I'm really grossed out." ] - + # Surprise examples surprise_examples = [ "Oh my God! I can't believe this!", @@ -170,7 +170,7 @@ def create_proper_training_data(): "I'm really surprised by this!", "This is astonishing!" ] - + # Neutral examples neutral_examples = [ "The weather is cloudy today.", @@ -190,37 +190,37 @@ def create_proper_training_data(): labels = [0] * 28 labels[emotion_names.index("joy")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in sadness_examples: labels = [0] * 28 labels[emotion_names.index("sadness")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in anger_examples: labels = [0] * 28 labels[emotion_names.index("anger")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in fear_examples: labels = [0] * 28 labels[emotion_names.index("fear")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in love_examples: labels = [0] * 28 labels[emotion_names.index("love")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in disgust_examples: labels = [0] * 28 labels[emotion_names.index("disgust")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in surprise_examples: labels = [0] * 28 labels[emotion_names.index("surprise")] = 1 training_data.append({"text": text, "labels": labels}) - + for text in neutral_examples: labels = [0] * 28 labels[emotion_names.index("neutral")] = 1 @@ -228,31 +228,31 @@ def create_proper_training_data(): # Shuffle the data random.shuffle(training_data) - + # Split into train/val/test total_samples = len(training_data) train_size = int(0.7 * total_samples) val_size = int(0.15 * total_samples) - + train_data = training_data[:train_size] val_data = training_data[train_size:train_size + val_size] test_data = training_data[train_size + val_size:] - + logger.info(f"โœ… Created {len(train_data)} training, {len(val_data)} validation, {len(test_data)} test samples") - + return train_data, val_data, test_data def create_dataloader(data, model, batch_size=8): """Create a simple dataloader for the data.""" dataloader = [] - + for i in range(0, len(data), batch_size): batch = data[i:i + batch_size] - + texts = [item["text"] for item in batch] labels = [item["labels"] for item in batch] - + # Tokenize tokenized = model.tokenizer( texts, @@ -261,43 +261,43 @@ def create_dataloader(data, model, batch_size=8): max_length=512, return_tensors="pt" ) - + dataloader.append({ "input_ids": tokenized["input_ids"], "attention_mask": tokenized["attention_mask"], "labels": torch.tensor(labels, dtype=torch.float32) }) - + return dataloader def train_model(model, train_data, val_data, device, epochs=10): """Train the model with focal loss.""" logger.info("๐Ÿš€ Starting model training...") - + model.to(device) optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5) criterion = FocalLoss() - + best_val_loss = float('inf') - + for epoch in range(epochs): model.train() total_loss = 0 - + for batch in tqdm(train_data, desc=f"Epoch {epoch + 1}/{epochs}"): input_ids = batch["input_ids"].to(device) attention_mask = batch["attention_mask"].to(device) labels = batch["labels"].to(device) - + optimizer.zero_grad() outputs = model(input_ids, attention_mask) loss = criterion(outputs, labels) loss.backward() optimizer.step() - + total_loss += loss.item() - + # Validation model.eval() val_loss = 0 @@ -306,77 +306,77 @@ def train_model(model, train_data, val_data, device, epochs=10): input_ids = batch["input_ids"].to(device) attention_mask = batch["attention_mask"].to(device) labels = batch["labels"].to(device) - + outputs = model(input_ids, attention_mask) loss = criterion(outputs, labels) val_loss += loss.item() - + avg_train_loss = total_loss / len(train_data) avg_val_loss = val_loss / len(val_data) - + logger.info(f"Epoch {epoch + 1}: Train Loss: {avg_train_loss:.4f}, Val Loss: {avg_val_loss:.4f}") - + # Save best model if avg_val_loss < best_val_loss: best_val_loss = avg_val_loss torch.save(model.state_dict(), "best_focal_model.pth") logger.info(f"โœ… Saved best model with val loss: {best_val_loss:.4f}") - + return model def evaluate_model(model, test_data, device): """Evaluate the model with different thresholds.""" logger.info("๐Ÿ“Š Evaluating model with different thresholds...") - + model.eval() all_predictions = [] all_labels = [] - + with torch.no_grad(): for batch in test_data: input_ids = batch["input_ids"].to(device) attention_mask = batch["attention_mask"].to(device) labels = batch["labels"].to(device) - + outputs = model(input_ids, attention_mask) predictions = torch.sigmoid(outputs) - + all_predictions.append(predictions.cpu().numpy()) all_labels.append(labels.cpu().numpy()) - + all_predictions = np.concatenate(all_predictions, axis=0) all_labels = np.concatenate(all_labels, axis=0) - + # Test different thresholds thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] best_f1 = 0 best_threshold = 0.5 - + for threshold in thresholds: binary_predictions = (all_predictions > threshold).astype(int) - + # Calculate metrics f1 = f1_score(all_labels, binary_predictions, average='weighted', zero_division=0) precision = precision_score(all_labels, binary_predictions, average='weighted', zero_division=0) recall = recall_score(all_labels, binary_predictions, average='weighted', zero_division=0) - + logger.info(f"Threshold {threshold}: F1={f1:.4f}, Precision={precision:.4f}, Recall={recall:.4f}") - + if f1 > best_f1: best_f1 = f1 best_threshold = threshold - + logger.info(f"๐ŸŽฏ Best threshold: {best_threshold} with F1: {best_f1:.4f}") - + # Final evaluation with best threshold binary_predictions = (all_predictions > best_threshold).astype(int) final_f1 = f1_score(all_labels, binary_predictions, average='weighted', zero_division=0) final_precision = precision_score(all_labels, binary_predictions, average='weighted', zero_division=0) final_recall = recall_score(all_labels, binary_predictions, average='weighted', zero_division=0) - + logger.info(f"๐Ÿ† Final Results - F1: {final_f1:.4f}, Precision: {final_precision:.4f}, Recall: {final_recall:.4f}") - + return { "f1": final_f1, "precision": final_precision, @@ -388,40 +388,40 @@ def evaluate_model(model, test_data, device): def main(): """Main training function.""" logger.info("๐ŸŽฏ Starting Fixed Focal Loss Training") - + # Setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") logger.info(f"๐Ÿ–ฅ๏ธ Using device: {device}") - + # Create directories Path("models").mkdir(exist_ok=True) Path("results").mkdir(exist_ok=True) - + # Create proper training data train_data, val_data, test_data = create_proper_training_data() - + # Create model model = SimpleBERTClassifier() logger.info(f"๐Ÿค– Created model with {sum(p.numel() for p in model.parameters())} parameters") - + # Create dataloaders train_dataloader = create_dataloader(train_data, model, batch_size=8) val_dataloader = create_dataloader(val_data, model, batch_size=8) test_dataloader = create_dataloader(test_data, model, batch_size=8) - + # Train model trained_model = train_model(model, train_dataloader, val_dataloader, device, epochs=5) - + # Load best model trained_model.load_state_dict(torch.load("best_focal_model.pth")) - + # Evaluate model results = evaluate_model(trained_model, test_dataloader, device) - + # Save results with open("results/focal_training_results.json", "w") as f: json.dump(results, f, indent=2) - + # Final summary logger.info("๐ŸŽ‰ Training completed successfully!") logger.info(f"๐Ÿ“Š Final F1 Score: {results['f1']:.4f}") diff --git a/scripts/training/full_dataset_focal_training.py b/scripts/training/full_dataset_focal_training.py index f92018dd7..183cc871a 100644 --- a/scripts/training/full_dataset_focal_training.py +++ b/scripts/training/full_dataset_focal_training.py @@ -131,11 +131,11 @@ def full_dataset_focal_training(): # Training loop model.train() train_losses = [] - + for epoch in range(3): logger.info(f"๐Ÿ“š Epoch {epoch + 1}/3") epoch_loss = 0.0 - + for batch_idx, batch in enumerate(train_dataloader): input_ids, attention_mask, batch_labels = batch input_ids = input_ids.to(device) diff --git a/scripts/training/full_focal_training.py b/scripts/training/full_focal_training.py index a39eebc6b..e5a3738a2 100644 --- a/scripts/training/full_focal_training.py +++ b/scripts/training/full_focal_training.py @@ -116,7 +116,7 @@ def full_focal_training(): model.train() for epoch in range(3): logger.info(f"๐Ÿ“š Epoch {epoch + 1}/3") - + for batch_idx, batch in enumerate(train_dataloader): input_ids, attention_mask, batch_labels = batch input_ids = input_ids.to(device) diff --git a/scripts/training/full_scale_focal_training.py b/scripts/training/full_scale_focal_training.py index 740c80006..372fc7681 100644 --- a/scripts/training/full_scale_focal_training.py +++ b/scripts/training/full_scale_focal_training.py @@ -134,7 +134,7 @@ def full_scale_focal_training(): for epoch in range(5): logger.info(f"๐Ÿ“š Epoch {epoch + 1}/5") epoch_loss = 0.0 - + for batch_idx, batch in enumerate(train_dataloader): input_ids, attention_mask, batch_labels = batch input_ids = input_ids.to(device) diff --git a/scripts/training/improve_expanded_training_notebook.py b/scripts/training/improve_expanded_training_notebook.py index 60273cc1b..10a1af353 100644 --- a/scripts/training/improve_expanded_training_notebook.py +++ b/scripts/training/improve_expanded_training_notebook.py @@ -9,25 +9,25 @@ def improve_notebook(): """Improve the expanded training notebook with enhancements.""" - + # Read the current notebook with open('notebooks/expanded_dataset_training.ipynb', 'r') as f: notebook = json.load(f) - + # Find the training function cell training_cell_idx = None for i, cell in enumerate(notebook['cells']): if cell['cell_type'] == 'code' and 'train_expanded_model' in str(cell['source']): training_cell_idx = i break - + if training_cell_idx is None: print("โŒ Could not find training function cell") return - + # Get the training function source training_source = notebook['cells'][training_cell_idx]['source'] - + # Add GPU optimizations after device setup device_pattern = r'print\(f"โœ… Using device: \{device\}"\)' gpu_optimizations = ''' @@ -38,19 +38,19 @@ def improve_notebook(): torch.backends.cudnn.deterministic = False print(f"๐Ÿ“Š GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") print(f"๐Ÿ“Š Available Memory: {torch.cuda.memory_allocated(0) / 1e9:.1f} GB") - + # Clear GPU cache if torch.cuda.is_available(): torch.cuda.empty_cache() ''' - + # Replace the device setup new_source = re.sub( device_pattern, f'print(f"โœ… Using device: {{device}}")\n{gpu_optimizations}', training_source ) - + # Add early stopping early_stopping_pattern = r'if f1_macro > best_f1:' early_stopping_code = ''' @@ -58,59 +58,59 @@ def improve_notebook(): if epoch > 2 and f1_macro < best_f1 * 0.95: print(f"๐Ÿ›‘ Early stopping triggered. F1 dropped below 95% of best.") break - + if f1_macro > best_f1:''' - + new_source = re.sub(early_stopping_pattern, early_stopping_code, new_source) - + # Add learning rate scheduling lr_scheduler_pattern = r'optimizer = torch\.optim\.AdamW\(model\.parameters\(\), lr=2e-5\)' lr_scheduler_code = '''optimizer = torch.optim.AdamW(model.parameters(), lr=2e-5) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=2, verbose=True)''' - + new_source = re.sub(lr_scheduler_pattern, lr_scheduler_code, new_source) - + # Add scheduler step scheduler_step_pattern = r'print\(f"๐Ÿ’พ New best model saved! F1: \{best_f1:.4f\}"\)' scheduler_step_code = '''print(f"๐Ÿ’พ New best model saved! F1: {best_f1:.4f}") scheduler.step(f1_macro)''' - + new_source = re.sub(scheduler_step_pattern, scheduler_step_code, new_source) - + # Add mixed precision training mixed_precision_pattern = r'import torch\.nn as nn' mixed_precision_code = '''import torch.nn as nn from torch.cuda.amp import autocast, GradScaler''' - + new_source = re.sub(mixed_precision_pattern, mixed_precision_code, new_source) - + # Add scaler initialization scaler_init_pattern = r'criterion = nn\.CrossEntropyLoss\(\)' scaler_init_code = '''criterion = nn.CrossEntropyLoss() scaler = GradScaler()''' - + new_source = re.sub(scaler_init_pattern, scaler_init_code, new_source) - + # Add mixed precision training loop training_loop_pattern = r'optimizer\.zero_grad\(\)\s+outputs = model\(input_ids=input_ids, attention_mask=attention_mask\)\s+loss = criterion\(outputs, labels\)\s+loss\.backward\(\)\s+optimizer\.step\(\)' training_loop_code = '''optimizer.zero_grad() with autocast(): outputs = model(input_ids=input_ids, attention_mask=attention_mask) loss = criterion(outputs, labels) - + scaler.scale(loss).backward() scaler.step(optimizer) scaler.update()''' - + new_source = re.sub(training_loop_pattern, training_loop_code, new_source) - + # Update the cell notebook['cells'][training_cell_idx]['source'] = new_source - + # Save the improved notebook with open('notebooks/expanded_dataset_training_improved.ipynb', 'w') as f: json.dump(notebook, f, indent=2) - + print("โœ… Improved notebook saved as 'notebooks/expanded_dataset_training_improved.ipynb'") print("๐Ÿ“‹ Improvements added:") print(" - GPU optimizations (cudnn benchmark, memory management)") diff --git a/scripts/training/robust_domain_adaptation_training.py b/scripts/training/robust_domain_adaptation_training.py index f605aee6f..a2d304584 100644 --- a/scripts/training/robust_domain_adaptation_training.py +++ b/scripts/training/robust_domain_adaptation_training.py @@ -25,7 +25,7 @@ def setup_environment(): """Setup the environment with proper dependency management.""" print("๐Ÿ”ง Setting up robust environment...") - + # Check if we're in Colab try: import google.colab @@ -34,47 +34,47 @@ def setup_environment(): except ImportError: print("โ„น๏ธ Running in local environment") is_colab = False - + # Install dependencies with proper version management print("๐Ÿ“ฆ Installing dependencies with compatibility fixes...") - + # Step 1: Clean slate - remove conflicting packages subprocess.run([ "pip", "uninstall", "torch", "torchvision", "torchaudio", "transformers", "datasets", "-y" ], capture_output=True) - + # Step 2: Install PyTorch with compatible CUDA version subprocess.run([ "pip", "install", "torch==2.1.0", "torchvision==0.16.0", "torchaudio==2.1.0", "--index-url", "https://download.pytorch.org/whl/cu118", "--no-cache-dir" ]) - + # Step 3: Install Transformers with compatible version subprocess.run([ "pip", "install", "transformers==4.30.0", "datasets==2.13.0", "--no-cache-dir" ]) - + # Step 4: Install additional dependencies subprocess.run([ "pip", "install", "evaluate", "scikit-learn", "pandas", "numpy", "matplotlib", "seaborn", "accelerate", "wandb", "--no-cache-dir" ]) - + print("โœ… Dependencies installed successfully") return is_colab def verify_installation(): """Verify that all critical packages are installed correctly.""" print("๐Ÿ” Verifying installation...") - + try: import torch import transformers print(f" PyTorch: {torch.__version__}") print(f" Transformers: {transformers.__version__}") print(f" CUDA Available: {torch.cuda.is_available()}") - + if torch.cuda.is_available(): print(f" GPU: {torch.cuda.get_device_name(0)}") print(f" Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") @@ -82,13 +82,13 @@ def verify_installation(): print(" โœ… GPU optimized for training") else: print("โš ๏ธ No GPU available. Training will be slow on CPU.") - + # Test critical imports from transformers import AutoModel, AutoTokenizer print(" โœ… Transformers imports successful") - + return True - + except Exception as e: print(f" โŒ Installation verification failed: {e}") return False @@ -96,7 +96,7 @@ def verify_installation(): def setup_repository(): """Setup the SAMO-DL repository.""" print("๐Ÿ“ Setting up repository...") - + def run_command(command: str, description: str) -> bool: """Execute command with error handling.""" print(f"๐Ÿ”„ {description}...") @@ -111,15 +111,15 @@ def run_command(command: str, description: str) -> bool: except Exception as e: print(f" โŒ {description} failed: {e}") return False - + # Clone repository if not exists if not Path('SAMO--DL').exists(): run_command('git clone https://github.com/uelkerd/SAMO--DL.git', 'Cloning repository') - + # Change to project directory os.chdir('SAMO--DL') print(f"๐Ÿ“ Working directory: {os.getcwd()}") - + # Pull latest changes run_command('git pull origin main', 'Pulling latest changes') @@ -153,16 +153,16 @@ def analyze_writing_style(texts: List[str], domain_name: str) -> Optional[Dict[s if not texts: print(f"โš ๏ธ No texts provided for {domain_name}") return None - + # Filter out None or empty texts valid_texts = [text for text in texts if text and isinstance(text, str)] - + if not valid_texts: print(f"โš ๏ธ No valid texts found for {domain_name}") return None - + import numpy as np - + avg_length = np.mean([len(text.split()) for text in valid_texts]) personal_pronouns = sum(['I ' in text or 'my ' in text or 'me ' in text for text in valid_texts]) / len(valid_texts) reflection_words = sum(['think' in text.lower() or 'feel' in text.lower() or 'believe' in text.lower() @@ -182,14 +182,14 @@ def analyze_writing_style(texts: List[str], domain_name: str) -> Optional[Dict[s def perform_domain_analysis(): """Perform domain gap analysis between GoEmotions and journal entries.""" print("๐Ÿ“Š Loading datasets for domain analysis...") - + # Load GoEmotions dataset go_emotions = safe_load_dataset("go_emotions", "simplified") if go_emotions: go_texts = go_emotions['train']['text'][:1000] # Sample for analysis else: go_texts = [] - + # Load journal dataset journal_entries = safe_load_json('data/journal_test_dataset.json') if journal_entries: @@ -198,19 +198,19 @@ def perform_domain_analysis(): journal_texts = journal_df['content'].tolist() else: journal_texts = [] - + # Analyze domains if data is available if go_texts and journal_texts: print("\n๐Ÿ” Domain Gap Analysis:") go_analysis = analyze_writing_style(go_texts, "GoEmotions (Reddit)") journal_analysis = analyze_writing_style(journal_texts, "Journal Entries") - + if go_analysis and journal_analysis: print("\n๐ŸŽฏ Key Insights:") print(f"- Journal entries are {journal_analysis['avg_length']/go_analysis['avg_length']:.1f}x longer") print(f"- Journal entries use {journal_analysis['personal_pronouns']/go_analysis['personal_pronouns']:.1f}x more personal pronouns") print(f"- Journal entries contain {journal_analysis['reflection_words']/go_analysis['reflection_words']:.1f}x more reflection words") - + return go_emotions, journal_df else: print("โš ๏ธ Cannot perform domain analysis - missing data") @@ -218,7 +218,7 @@ def perform_domain_analysis(): class FocalLoss: """Focal Loss for addressing class imbalance in emotion detection.""" - + def __init__(self, alpha=1, gamma=2, reduction='mean'): import torch.nn as nn import torch.nn.functional as F @@ -226,7 +226,7 @@ def __init__(self, alpha=1, gamma=2, reduction='mean'): self.gamma = gamma self.reduction = reduction self.F = F - + def __call__(self, inputs, targets): ce_loss = self.F.cross_entropy(inputs, targets, reduction='none') pt = torch.exp(-ce_loss) @@ -241,25 +241,25 @@ def __call__(self, inputs, targets): class DomainAdaptedEmotionClassifier: """BERT-based emotion classifier with domain adaptation capabilities.""" - + def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3): import torch.nn as nn from transformers import AutoModel - + # ROBUST: Validate num_labels if num_labels is None: print("โš ๏ธ num_labels not provided, using default value of 12") num_labels = 12 elif num_labels <= 0: raise ValueError(f"num_labels must be positive, got {num_labels}") - + print(f"๐Ÿ—๏ธ Initializing DomainAdaptedEmotionClassifier with num_labels = {num_labels}") - + try: self.bert = AutoModel.from_pretrained(model_name) self.dropout = nn.Dropout(dropout) self.classifier = nn.Linear(self.bert.config.hidden_size, num_labels) - + # Domain adaptation layer self.domain_classifier = nn.Sequential( nn.Linear(self.bert.config.hidden_size, 512), @@ -267,9 +267,9 @@ def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3) nn.Dropout(0.3), nn.Linear(512, 2) # 2 domains: GoEmotions vs Journal ) - + print(f"โœ… Model initialized successfully with {num_labels} labels") - + except Exception as e: print(f"โŒ Failed to initialize model: {e}") raise @@ -281,14 +281,14 @@ def forward(self, input_ids, attention_mask, domain_labels=None): # Emotion classification emotion_logits = self.classifier(self.dropout(pooled_output)) - + # Domain classification (for domain adaptation) domain_logits = self.domain_classifier(pooled_output) - + if domain_labels is not None: return emotion_logits, domain_logits return emotion_logits - + except Exception as e: print(f"โŒ Forward pass failed: {e}") raise @@ -297,26 +297,26 @@ def safe_model_initialization(model_name: str, num_labels: int, device: str): """Safely initialize model with error handling.""" try: print(f"๐Ÿ—๏ธ Initializing model with {model_name}...") - + # Initialize tokenizer from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(model_name) print(f"โœ… Tokenizer loaded: {model_name}") - + # Initialize model model = DomainAdaptedEmotionClassifier(model_name=model_name, num_labels=num_labels) - + # Move to device import torch model = model.to(device) print(f"โœ… Model moved to {device}") - + # Verify model parameters total_params = sum(p.numel() for p in model.parameters()) print(f"๐Ÿ“Š Model parameters: {total_params:,}") - + return model, tokenizer - + except Exception as e: print(f"โŒ Model initialization failed: {e}") raise @@ -325,32 +325,32 @@ def main(): """Main execution function.""" print("๐Ÿš€ Starting SAMO Deep Learning - Robust Domain Adaptation Training") print("=" * 70) - + # Step 1: Setup environment is_colab = setup_environment() - + # Step 2: Verify installation if not verify_installation(): print("โŒ Installation verification failed. Please restart and try again.") return - + # Step 3: Setup repository setup_repository() - + # Step 4: Perform domain analysis go_emotions, journal_df = perform_domain_analysis() - + if go_emotions is None or journal_df is None: print("โŒ Cannot proceed without datasets") return - + # Step 5: Initialize model (example) import torch device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - + # This would be called when we have the label encoder ready # model, tokenizer = safe_model_initialization("bert-base-uncased", num_labels, device) - + print("\nโœ… Setup completed successfully!") print("๐ŸŽฏ Ready for domain adaptation training") print("\n๐Ÿ“‹ Next steps:") diff --git a/scripts/training/setup_colab_environment.py b/scripts/training/setup_colab_environment.py index e33c1902a..18e13be62 100644 --- a/scripts/training/setup_colab_environment.py +++ b/scripts/training/setup_colab_environment.py @@ -32,7 +32,7 @@ def detect_colab_environment(): def install_dependencies(): """Install all required dependencies.""" logger.info("๐Ÿ“ฆ Installing dependencies...") - + # Core ML dependencies packages = [ "torch>=2.1.0,<2.2.0", @@ -61,7 +61,7 @@ def install_dependencies(): "python-dotenv>=1.0.0,<2.0.0", "accelerate>=0.20.0,<1.0.0", ] - + for package in packages: try: logger.info(f"๐Ÿ“ฆ Installing {package}...") @@ -71,37 +71,37 @@ def install_dependencies(): except subprocess.CalledProcessError as e: logger.error(f"โŒ Failed to install {package}: {e}") return False - + return True def setup_gpu_environment(): """Set up GPU environment for optimal performance.""" logger.info("๐Ÿ–ฅ๏ธ Setting up GPU environment...") - + try: import torch - + if torch.cuda.is_available(): logger.info(f"๐ŸŽฎ GPU detected: {torch.cuda.get_device_name(0)}") logger.info(f"๐ŸŽฎ GPU count: {torch.cuda.device_count()}") logger.info(f"๐ŸŽฎ CUDA version: {torch.version.cuda}") - + # Set environment variables for optimal GPU performance os.environ["CUDA_LAUNCH_BLOCKING"] = "1" os.environ["TOKENIZERS_PARALLELISM"] = "false" - + # Test GPU functionality device = torch.device("cuda") test_tensor = torch.randn(100, 100).to(device) result = torch.matmul(test_tensor, test_tensor.T) logger.info(f"โœ… GPU test successful, result shape: {result.shape}") - + return True else: logger.warning("โš ๏ธ No GPU available, using CPU") return True - + except ImportError: logger.error("โŒ PyTorch not available for GPU setup") return False @@ -113,7 +113,7 @@ def setup_gpu_environment(): def create_colab_notebook(): """Create a Colab-ready notebook template.""" logger.info("๐Ÿ““ Creating Colab notebook template...") - + notebook_content = '''{ "cells": [ { @@ -211,10 +211,10 @@ def create_colab_notebook(): "nbformat": 4, "nbformat_minor": 4 }''' - + with open("samo_dl_colab_setup.ipynb", "w") as f: f.write(notebook_content) - + logger.info("โœ… Colab notebook template created: samo_dl_colab_setup.ipynb") return True @@ -222,7 +222,7 @@ def create_colab_notebook(): def run_ci_pipeline(): """Run the CI pipeline to verify everything is working.""" logger.info("๐Ÿš€ Running CI pipeline verification...") - + try: result = subprocess.run( [sys.executable, "scripts/ci/run_full_ci_pipeline.py"], @@ -230,7 +230,7 @@ def run_ci_pipeline(): text=True, timeout=600 # 10 minute timeout ) - + if result.returncode == 0: logger.info("โœ… CI pipeline verification passed") logger.info("๐Ÿ“Š CI Results:") @@ -240,7 +240,7 @@ def run_ci_pipeline(): logger.error("โŒ CI pipeline verification failed") logger.error(result.stderr) return False - + except subprocess.TimeoutExpired: logger.error("โฐ CI pipeline verification timed out") return False @@ -253,36 +253,36 @@ def main(): """Main setup function.""" logger.info("๐Ÿš€ Starting Colab Environment Setup") logger.info("=" * 50) - + # Detect environment is_colab = detect_colab_environment() - + # Install dependencies if not install_dependencies(): logger.error("โŒ Dependency installation failed") sys.exit(1) - + # Setup GPU environment if not setup_gpu_environment(): logger.error("โŒ GPU environment setup failed") sys.exit(1) - + # Create Colab notebook if is_colab: create_colab_notebook() - + # Run CI pipeline verification if not run_ci_pipeline(): logger.error("โŒ CI pipeline verification failed") sys.exit(1) - + logger.info("๐ŸŽ‰ Colab environment setup completed successfully!") logger.info("=" * 50) logger.info("๐Ÿ“‹ Next steps:") logger.info("1. Upload the repository to Colab") logger.info("2. Run the CI pipeline: python scripts/ci/run_full_ci_pipeline.py") logger.info("3. Start developing with GPU acceleration!") - + if is_colab: logger.info("๐Ÿ““ Colab notebook template created: samo_dl_colab_setup.ipynb") diff --git a/scripts/training/summarize_comprehensive_notebook.py b/scripts/training/summarize_comprehensive_notebook.py index fdaf4daca..fe57d825c 100644 --- a/scripts/training/summarize_comprehensive_notebook.py +++ b/scripts/training/summarize_comprehensive_notebook.py @@ -11,28 +11,28 @@ def summarize_comprehensive_notebook(): """Summarize the comprehensive notebook.""" - + # Read the notebook with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + print("๐Ÿš€ COMPREHENSIVE ULTIMATE TRAINING NOTEBOOK SUMMARY") print("=" * 60) print() - + # Count cells by type markdown_cells = [cell for cell in notebook['cells'] if cell['cell_type'] == 'markdown'] code_cells = [cell for cell in notebook['cells'] if cell['cell_type'] == 'code'] - + print(f"๐Ÿ“Š NOTEBOOK STATISTICS:") print(f" Total cells: {len(notebook['cells'])}") print(f" Markdown cells: {len(markdown_cells)}") print(f" Code cells: {len(code_cells)}") print() - + print("๐ŸŽฏ ALL FEATURES INCLUDED:") print("=" * 40) - + features = [ "โœ… Configuration preservation (prevents 8.3% vs 75% discrepancy)", "โœ… Focal loss (handles class imbalance)", @@ -50,14 +50,14 @@ def summarize_comprehensive_notebook(): "โœ… Evaluation and metrics", "โœ… Unseen data testing" ] - + for feature in features: print(f" {feature}") - + print() print("๐Ÿ“‹ CELL BREAKDOWN:") print("=" * 30) - + cell_titles = [ "Title and Overview", "Package Installation", @@ -78,10 +78,10 @@ def summarize_comprehensive_notebook(): "Advanced Validation and Bias Analysis", "Model Saving with Verification" ] - + for i, title in enumerate(cell_titles, 1): print(f" {i:2d}. {title}") - + print() print("๐ŸŽฏ KEY ADVANTAGES:") print("=" * 30) @@ -95,10 +95,10 @@ def summarize_comprehensive_notebook(): "๐Ÿš€ Ready for production deployment", "๐Ÿ“‹ Complete training pipeline from start to finish" ] - + for advantage in advantages: print(f" {advantage}") - + print() print("๐Ÿ“ FILE LOCATION:") print(f" notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb") diff --git a/scripts/training/summarize_ultimate_notebook.py b/scripts/training/summarize_ultimate_notebook.py index d6c83271e..cd484730b 100644 --- a/scripts/training/summarize_ultimate_notebook.py +++ b/scripts/training/summarize_ultimate_notebook.py @@ -10,21 +10,21 @@ def summarize_notebook(): """Summarize the ultimate notebook contents.""" - + print("๐Ÿš€ ULTIMATE BULLETPROOF TRAINING NOTEBOOK SUMMARY") print("=" * 60) print() - + # Read the notebook with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: notebook = json.load(f) - + print("๐Ÿ“‹ NOTEBOOK OVERVIEW:") print(" ๐Ÿ“ File: notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb") print(f" ๐Ÿ“Š Total cells: {len(notebook['cells'])}") print(" ๐ŸŽฏ Target: 75-85% F1 score with consistent performance") print() - + print("โœ… ALL FEATURES INCLUDED:") print(" ๐Ÿ”ง Configuration preservation (prevents 8.3% vs 75% discrepancy)") print(" ๐ŸŽฏ Focal loss implementation (handles class imbalance)") @@ -33,7 +33,7 @@ def summarize_notebook(): print(" ๐Ÿงช Advanced validation (proper testing)") print(" ๐Ÿ’พ Model saving with verification") print() - + print("๐Ÿ” CELL BREAKDOWN:") cell_count = 0 for cell in notebook['cells']: @@ -58,7 +58,7 @@ def summarize_notebook(): print(f" {cell_count:2d}. ๐Ÿš€ Training Execution") elif 'model.save_pretrained' in code_text: print(f" {cell_count:2d}. ๐Ÿ’พ Model Saving with Verification") - + print() print("๐ŸŽฏ KEY IMPROVEMENTS FROM PREVIOUS ITERATIONS:") print(" โœ… Fixed model configuration preservation") @@ -68,7 +68,7 @@ def summarize_notebook(): print(" โœ… Advanced validation on diverse examples") print(" โœ… Comprehensive model saving with verification") print() - + print("๐Ÿ“‹ USAGE INSTRUCTIONS:") print(" 1. Download the notebook file") print(" 2. Upload to Google Colab") @@ -76,7 +76,7 @@ def summarize_notebook(): print(" 4. Run all cells") print(" 5. Expect 75-85% F1 score!") print() - + print("๐Ÿ”ง TECHNICAL SPECIFICATIONS:") print(" ๐Ÿ—๏ธ Model: j-hartmann/emotion-english-distilroberta-base") print(" ๐ŸŽฏ Emotions: 12 classes (anxious, calm, content, excited, etc.)") @@ -85,7 +85,7 @@ def summarize_notebook(): print(" ๐Ÿงช Validation: Advanced testing on diverse examples") print(" ๐Ÿ’พ Output: Verified model with proper configuration") print() - + print("๐ŸŽ‰ THIS IS THE ULTIMATE BULLETPROOF VERSION!") print(" Combines ALL successful techniques from previous iterations") print(" Addresses ALL known issues and limitations") diff --git a/scripts/training/validate_improved_notebook.py b/scripts/training/validate_improved_notebook.py index eda4c6a03..d664aaf60 100644 --- a/scripts/training/validate_improved_notebook.py +++ b/scripts/training/validate_improved_notebook.py @@ -8,9 +8,9 @@ def validate_notebook(): """Validate the improved notebook for Colab execution.""" - + print("๐Ÿ” Validating improved notebook...") - + # Load the notebook try: with open('notebooks/expanded_dataset_training_improved.ipynb', 'r') as f: @@ -19,22 +19,22 @@ def validate_notebook(): except Exception as e: print(f"โŒ Notebook JSON error: {e}") return False - + # Check notebook structure cells = notebook['cells'] print(f"๐Ÿ“Š Notebook has {len(cells)} cells") - + # Validate cell types markdown_cells = [c for c in cells if c['cell_type'] == 'markdown'] code_cells = [c for c in cells if c['cell_type'] == 'code'] - + print(f"๐Ÿ“ Markdown cells: {len(markdown_cells)}") print(f"๐Ÿ’ป Code cells: {len(code_cells)}") - + # Check for critical components cell_sources = [str(c.get('source', '')) for c in cells] all_source = ' '.join(cell_sources) - + # Critical checks checks = [ ("Repository cloning", "git clone https://github.com/uelkerd/SAMO--DL.git"), @@ -48,17 +48,17 @@ def validate_notebook(): ("Model testing", "test_new_model"), ("Results download", "files.download"), ] - + print("\n๐Ÿ” Critical component checks:") all_passed = True - + for check_name, check_content in checks: if check_content in all_source: print(f" โœ… {check_name}") else: print(f" โŒ {check_name}") all_passed = False - + # Check for JSON syntax issues print("\n๐Ÿ” JSON syntax validation:") try: @@ -69,7 +69,7 @@ def validate_notebook(): except Exception as e: print(f" โŒ JSON escaping issues: {e}") all_passed = False - + # Check for GPU optimizations gpu_optimizations = [ "torch.backends.cudnn.benchmark = True", @@ -79,7 +79,7 @@ def validate_notebook(): "num_workers=2", "pin_memory=True" ] - + print("\n๐Ÿ” GPU optimization checks:") for opt in gpu_optimizations: if opt in all_source: @@ -87,7 +87,7 @@ def validate_notebook(): else: print(f" โŒ {opt}") all_passed = False - + # Check for training optimizations training_optimizations = [ "GradScaler()", @@ -98,7 +98,7 @@ def validate_notebook(): "ReduceLROnPlateau", "Early stopping triggered" ] - + print("\n๐Ÿ” Training optimization checks:") for opt in training_optimizations: if opt in all_source: @@ -106,14 +106,14 @@ def validate_notebook(): else: print(f" โŒ {opt}") all_passed = False - + # Summary print(f"\n๐Ÿ“Š Validation Summary:") print(f" Total cells: {len(cells)}") print(f" Code cells: {len(code_cells)}") print(f" Markdown cells: {len(markdown_cells)}") print(f" All checks passed: {'โœ…' if all_passed else 'โŒ'}") - + if all_passed: print("\n๐ŸŽ‰ Notebook is ready for Colab execution!") print("๐Ÿ“‹ Next steps:") @@ -123,7 +123,7 @@ def validate_notebook(): print(" 4. Expect 75-85% F1 score!") else: print("\nโš ๏ธ Notebook needs fixes before Colab execution") - + return all_passed if __name__ == "__main__": diff --git a/scripts/validation/check_dependencies.py b/scripts/validation/check_dependencies.py index f1f8149d8..ef9decad5 100644 --- a/scripts/validation/check_dependencies.py +++ b/scripts/validation/check_dependencies.py @@ -13,39 +13,39 @@ class DependencyChecker: """Checker for dependency usage in the codebase.""" - + def __init__(self, requirements_path: str = "requirements.txt"): self.requirements_path = Path(requirements_path) self.project_root = Path(__file__).parent.parent.parent self.unused_deps = [] self.missing_deps = [] - + def check_dependencies(self) -> bool: """Check if all dependencies are used in the codebase.""" print("๐Ÿ” Checking dependency usage...") - + # Read requirements.txt if not self.requirements_path.exists(): print(f"โŒ Requirements file not found: {self.requirements_path}") return False - + required_deps = self._parse_requirements() used_deps = self._find_used_dependencies() - + # Check for unused dependencies for dep in required_deps: if dep not in used_deps: self.unused_deps.append(dep) - + # Check for missing dependencies (optional) # This would require more complex analysis - + return len(self.unused_deps) == 0 - + def _parse_requirements(self) -> Set[str]: """Parse requirements.txt and extract package names.""" deps = set() - + with open(self.requirements_path, 'r') as f: for line in f: line = line.strip() @@ -53,34 +53,34 @@ def _parse_requirements(self) -> Set[str]: # Extract package name (remove version constraints) package = re.split(r'[<>=!~]', line)[0].strip() deps.add(package) - + return deps - + def _find_used_dependencies(self) -> Set[str]: """Find all dependencies used in the codebase.""" used_deps = set() - + # Common Python file extensions python_extensions = {'.py', '.pyx', '.pyi'} - + # Directories to scan scan_dirs = ['src', 'scripts', 'tests', 'deployment'] - + for scan_dir in scan_dirs: dir_path = self.project_root / scan_dir if dir_path.exists(): for file_path in dir_path.rglob('*'): if file_path.suffix in python_extensions: self._scan_file_for_imports(file_path, used_deps) - + return used_deps - + def _scan_file_for_imports(self, file_path: Path, used_deps: Set[str]) -> None: """Scan a Python file for import statements.""" try: with open(file_path, 'r', encoding='utf-8') as f: content = f.read() - + # Find import statements import_patterns = [ r'^import\s+(\w+)', @@ -88,7 +88,7 @@ def _scan_file_for_imports(self, file_path: Path, used_deps: Set[str]) -> None: r'^\s+import\s+(\w+)', r'^\s+from\s+(\w+)' ] - + for pattern in import_patterns: matches = re.findall(pattern, content, re.MULTILINE) for match in matches: @@ -98,15 +98,15 @@ def _scan_file_for_imports(self, file_path: Path, used_deps: Set[str]) -> None: # Extract base package name base_package = package.split('.')[0] used_deps.add(base_package) - + except Exception as e: print(f"โš ๏ธ Warning: Could not scan {file_path}: {e}") - + def print_results(self) -> None: """Print dependency check results.""" print(f"\n๐Ÿ“Š Dependency Usage Check Results") print("=" * 50) - + if self.unused_deps: print(f"\nโš ๏ธ Potentially Unused Dependencies ({len(self.unused_deps)}):") for dep in sorted(self.unused_deps): @@ -114,7 +114,7 @@ def print_results(self) -> None: print("\n๐Ÿ’ก Consider removing these dependencies if they're not needed.") else: print("\nโœ… All dependencies appear to be used in the codebase!") - + if self.missing_deps: print(f"\nโŒ Missing Dependencies ({len(self.missing_deps)}):") for dep in sorted(self.missing_deps): @@ -123,7 +123,7 @@ def print_results(self) -> None: def main(): """Main function to run dependency usage check.""" checker = DependencyChecker() - + if checker.check_dependencies(): checker.print_results() if checker.unused_deps: diff --git a/scripts/validation/validate_security_config.py b/scripts/validation/validate_security_config.py index 9d438eee0..90d68be4e 100644 --- a/scripts/validation/validate_security_config.py +++ b/scripts/validation/validate_security_config.py @@ -13,225 +13,225 @@ class SecurityConfigValidator: """Validator for security configuration files.""" - + def __init__(self, config_path: str = "configs/security.yaml"): self.config_path = Path(config_path) self.errors = [] self.warnings = [] - + def validate(self) -> bool: """Validate the security configuration file.""" print("๐Ÿ” Validating security configuration...") - + # Check if file exists if not self.config_path.exists(): self.errors.append(f"Security configuration file not found: {self.config_path}") return False - + try: with open(self.config_path, 'r') as f: config = yaml.safe_load(f) except yaml.YAMLError as e: self.errors.append(f"Invalid YAML in security configuration: {e}") return False - + # Validate required sections self._validate_required_sections(config) - + # Validate API security settings self._validate_api_security(config.get('api', {})) - + # Validate security headers self._validate_security_headers(config.get('security_headers', {})) - + # Validate logging configuration self._validate_logging(config.get('logging', {})) - + # Validate environment settings self._validate_environment(config.get('environment', {})) - + # Validate dependency security self._validate_dependencies(config.get('dependencies', {})) - + # Validate model security self._validate_model_security(config.get('model', {})) - + # Validate database security self._validate_database_security(config.get('database', {})) - + # Validate deployment security self._validate_deployment_security(config.get('deployment', {})) - + return len(self.errors) == 0 - + def _validate_required_sections(self, config: Dict[str, Any]) -> None: """Validate that all required sections are present.""" required_sections = [ 'api', 'security_headers', 'logging', 'environment', 'dependencies', 'model', 'database', 'deployment' ] - + for section in required_sections: if section not in config: self.errors.append(f"Missing required section: {section}") - + def _validate_api_security(self, api_config: Dict[str, Any]) -> None: """Validate API security configuration.""" if not api_config: self.errors.append("API configuration is empty") return - + # Check rate limiting rate_limiting = api_config.get('rate_limiting', {}) if not rate_limiting.get('enabled', False): self.warnings.append("Rate limiting is disabled - security risk") - + # Check CORS cors = api_config.get('cors', {}) if not cors.get('enabled', False): self.warnings.append("CORS is disabled - may cause issues") - + # Check authentication auth = api_config.get('authentication', {}) if not auth.get('enabled', False): self.errors.append("Authentication is disabled - security risk") - + # Check input validation input_validation = api_config.get('input_validation', {}) if not input_validation: self.errors.append("Input validation configuration is missing") - + def _validate_security_headers(self, headers_config: Dict[str, Any]) -> None: """Validate security headers configuration.""" if not headers_config.get('enabled', False): self.warnings.append("Security headers are disabled") return - + headers = headers_config.get('headers', {}) required_headers = [ 'X-Content-Type-Options', 'X-Frame-Options', 'X-XSS-Protection' ] - + for header in required_headers: if header not in headers: self.warnings.append(f"Missing recommended security header: {header}") - + def _validate_logging(self, logging_config: Dict[str, Any]) -> None: """Validate logging configuration.""" if not logging_config: self.errors.append("Logging configuration is missing") return - + # Check security events logging security_events = logging_config.get('security_events', {}) if not security_events.get('enabled', False): self.warnings.append("Security events logging is disabled") - + # Check request logging requests = logging_config.get('requests', {}) if not requests.get('enabled', False): self.warnings.append("Request logging is disabled") - + # Check error logging errors = logging_config.get('errors', {}) if not errors.get('enabled', False): self.warnings.append("Error logging is disabled") - + def _validate_environment(self, env_config: Dict[str, Any]) -> None: """Validate environment configuration.""" if not env_config: self.errors.append("Environment configuration is missing") return - + # Check required environment variables required_vars = env_config.get('required_vars', []) if not required_vars: self.warnings.append("No required environment variables specified") - + # Check sensitive variables sensitive_vars = env_config.get('sensitive_vars', []) if not sensitive_vars: self.warnings.append("No sensitive variables specified for masking") - + # Check environment-specific settings for env in ['production', 'development', 'testing']: env_settings = env_config.get(env, {}) if not env_settings: self.warnings.append(f"No settings specified for {env} environment") - + def _validate_dependencies(self, deps_config: Dict[str, Any]) -> None: """Validate dependency security configuration.""" if not deps_config: self.errors.append("Dependency security configuration is missing") return - + scanning = deps_config.get('scanning', {}) if not scanning.get('enabled', False): self.warnings.append("Dependency security scanning is disabled") - + tools = scanning.get('tools', []) if not tools: self.warnings.append("No security scanning tools specified") - + def _validate_model_security(self, model_config: Dict[str, Any]) -> None: """Validate model security configuration.""" if not model_config: self.errors.append("Model security configuration is missing") return - + loading = model_config.get('loading', {}) if not loading.get('validate_model_files', False): self.warnings.append("Model file validation is disabled") - + inference = model_config.get('inference', {}) if not inference: self.warnings.append("Model inference security settings are missing") - + def _validate_database_security(self, db_config: Dict[str, Any]) -> None: """Validate database security configuration.""" if not db_config: self.errors.append("Database security configuration is missing") return - + connection = db_config.get('connection', {}) if not connection.get('use_ssl', False): self.errors.append("Database SSL is disabled - security risk") - + data_protection = db_config.get('data_protection', {}) if not data_protection.get('encrypt_sensitive_data', False): self.warnings.append("Sensitive data encryption is disabled") - + def _validate_deployment_security(self, deploy_config: Dict[str, Any]) -> None: """Validate deployment security configuration.""" if not deploy_config: self.errors.append("Deployment security configuration is missing") return - + container = deploy_config.get('container', {}) if not container.get('run_as_non_root', False): self.errors.append("Container not configured to run as non-root - security risk") - + network = deploy_config.get('network', {}) if not network.get('use_https', False): self.errors.append("HTTPS is disabled - security risk") - + def print_results(self) -> None: """Print validation results.""" print(f"\n๐Ÿ“Š Security Configuration Validation Results") print("=" * 50) - + if self.errors: print(f"\nโŒ Errors ({len(self.errors)}):") for error in self.errors: print(f" - {error}") - + if self.warnings: print(f"\nโš ๏ธ Warnings ({len(self.warnings)}):") for warning in self.warnings: print(f" - {warning}") - + if not self.errors and not self.warnings: print("\nโœ… Security configuration is valid!") elif not self.errors: @@ -242,7 +242,7 @@ def print_results(self) -> None: def main(): """Main function to run security configuration validation.""" validator = SecurityConfigValidator() - + if validator.validate(): validator.print_results() if validator.errors: diff --git a/src/api_rate_limiter.py b/src/api_rate_limiter.py index e96a4c4cf..d705bbf67 100644 --- a/src/api_rate_limiter.py +++ b/src/api_rate_limiter.py @@ -43,7 +43,7 @@ class RateLimitConfig: class TokenBucketRateLimiter: """ Token bucket rate limiter with security enhancements. - + Features: - Token bucket algorithm for smooth rate limiting - IP-based rate limiting with whitelist/blacklist @@ -52,7 +52,7 @@ class TokenBucketRateLimiter: - Automatic blocking of abusive clients - Request fingerprinting for advanced detection """ - + def __init__(self, config: RateLimitConfig): self.config = config self.buckets: Dict[str, float] = defaultdict(lambda: config.burst_size) @@ -61,34 +61,34 @@ def __init__(self, config: RateLimitConfig): self.concurrent_requests: Dict[str, int] = defaultdict(int) self.request_history: Dict[str, Deque] = defaultdict(lambda: deque(maxlen=100)) self.lock = threading.RLock() - + # Initialize whitelist/blacklist if config.whitelisted_ips is None: config.whitelisted_ips = set() if config.blacklisted_ips is None: config.blacklisted_ips = set() - + def _get_client_key(self, client_ip: str, user_agent: str = "") -> str: """Generate a unique client key for rate limiting.""" # Create a fingerprint based on IP and user agent fingerprint = f"{client_ip}:{user_agent}" return hashlib.sha256(fingerprint.encode()).hexdigest() - + def _is_ip_allowed(self, client_ip: str) -> bool: """Check if IP is allowed based on whitelist/blacklist.""" # Allow test clients to pass through if client_ip in ["testclient", "127.0.0.1", "localhost"]: return True - + try: ip = ipaddress.ip_address(client_ip) - + # Check blacklist first if self.config.enable_ip_blacklist: if client_ip in self.config.blacklisted_ips: logger.warning(f"Blocked request from blacklisted IP: {client_ip}") return False - + # Check whitelist if self.config.enable_ip_whitelist: if client_ip in self.config.whitelisted_ips: @@ -96,13 +96,13 @@ def _is_ip_allowed(self, client_ip: str) -> bool: else: logger.warning(f"Blocked request from non-whitelisted IP: {client_ip}") return False - + return True - + except ValueError: logger.error(f"Invalid IP address: {client_ip}") return False - + def _is_client_blocked(self, client_key: str) -> bool: """Check if client is currently blocked.""" if client_key in self.blocked_clients: @@ -113,74 +113,74 @@ def _is_client_blocked(self, client_key: str) -> bool: # Remove expired block del self.blocked_clients[client_key] return False - + def _analyze_user_agent(self, user_agent: str) -> int: """Analyze user agent for suspicious patterns. Returns score (0-10).""" if not user_agent: return 0 - + score = 0 ua_lower = user_agent.lower() - + # High-risk patterns (score +3 each) high_risk_patterns = [ 'sqlmap', 'nikto', 'nmap', 'scanner', 'crawler', 'spider', 'bot', 'automation', 'script', 'python-requests', 'curl', 'wget', 'httrack', 'grabber', 'harvester' ] - + # Medium-risk patterns (score +2 each) medium_risk_patterns = [ 'headless', 'phantom', 'selenium', 'webdriver', 'automated', 'testing', 'monitoring', 'healthcheck', 'pingdom', 'uptimerobot' ] - + # Low-risk patterns (score +1 each) low_risk_patterns = [ 'bot', 'crawler', 'spider', 'indexer', 'feed', 'rss', 'aggregator', 'monitor', 'checker' ] - + # Check high-risk patterns for pattern in high_risk_patterns: if pattern in ua_lower: score += 3 logger.debug(f"High-risk UA pattern detected: {pattern}") - + # Check medium-risk patterns for pattern in medium_risk_patterns: if pattern in ua_lower: score += 2 logger.debug(f"Medium-risk UA pattern detected: {pattern}") - + # Check low-risk patterns for pattern in low_risk_patterns: if pattern in ua_lower: score += 1 logger.debug(f"Low-risk UA pattern detected: {pattern}") - + # Bonus for suspicious combinations if any(pattern in ua_lower for pattern in ['bot', 'crawler']) and any(pattern in ua_lower for pattern in ['python', 'curl', 'wget']): score += 2 logger.debug("Suspicious UA combination detected") - + return min(score, 10) # Cap at 10 - + def _analyze_request_patterns(self, client_key: str, client_ip: str) -> int: """Analyze request patterns for suspicious behavior. Returns score (0-10).""" score = 0 history = self.request_history[client_key] current_time = time.time() - + if len(history) < 5: # Need minimum data for analysis return 0 - + # Remove old requests recent_history = [req_time for req_time in history if current_time - req_time <= self.config.anomaly_detection_window] - + if len(recent_history) < 3: return 0 - + # Check for burst patterns (many requests in short time) burst_windows = [1.0, 5.0, 10.0] # 1s, 5s, 10s windows for window in burst_windows: @@ -188,73 +188,73 @@ def _analyze_request_patterns(self, client_key: str, client_ip: str) -> int: if len(burst_requests) > window * 2: # More than 2 requests per second score += 2 logger.debug(f"Burst pattern detected: {len(burst_requests)} requests in {window}s") - + # Check for regular intervals (automated behavior) if len(recent_history) >= 5: intervals = [] for i in range(1, len(recent_history)): intervals.append(recent_history[i] - recent_history[i-1]) - + # Check if intervals are too regular (automated) if len(intervals) >= 3: avg_interval = sum(intervals) / len(intervals) variance = sum((x - avg_interval) ** 2 for x in intervals) / len(intervals) - + if variance < 0.1 and avg_interval < 2.0: # Very regular, fast intervals score += 3 logger.debug(f"Regular interval pattern detected: avg={avg_interval:.2f}s, variance={variance:.2f}") - + # Check for sustained high rate minute_requests = [req_time for req_time in recent_history if current_time - req_time <= 60.0] if len(minute_requests) > 50: # More than 50 requests per minute score += 2 logger.debug(f"Sustained high rate: {len(minute_requests)} requests per minute") - + return min(score, 10) # Cap at 10 - + def _detect_abuse(self, client_key: str, client_ip: str, user_agent: str = "") -> bool: """Enhanced abuse detection with user agent and pattern analysis.""" # Basic rate-based detection (existing logic) history = self.request_history[client_key] current_time = time.time() - + # Remove old requests (older than 1 hour) while history and current_time - history[0] > 3600: history.popleft() - + # Check for rapid-fire requests recent_requests = [req_time for req_time in history if current_time - req_time <= self.config.rapid_fire_window] if len(recent_requests) > self.config.rapid_fire_threshold: logger.warning(f"Rate-based abuse detected: {len(recent_requests)} requests in {self.config.rapid_fire_window}s from {client_ip}") return True - + # Check for sustained high rate minute_requests = [req_time for req_time in history if current_time - req_time <= self.config.sustained_rate_window] if len(minute_requests) > self.config.sustained_rate_threshold: logger.warning(f"Rate-based abuse detected: {len(minute_requests)} requests in {self.config.sustained_rate_window}s from {client_ip}") return True - + # Enhanced anomaly detection if self.config.enable_user_agent_analysis: ua_score = self._analyze_user_agent(user_agent) if ua_score >= self.config.suspicious_user_agent_score_threshold: logger.warning(f"User agent abuse detected: score {ua_score} from {client_ip} (UA: {user_agent[:100]})") return True - + if self.config.enable_request_pattern_analysis: pattern_score = self._analyze_request_patterns(client_key, client_ip) if pattern_score >= self.config.request_pattern_score_threshold: logger.warning(f"Pattern-based abuse detected: score {pattern_score} from {client_ip}") return True - + return False - + def _refill_bucket(self, client_key: str): """Refill the token bucket for a client.""" current_time = time.time() last_refill_time = self.last_refill[client_key] time_passed = current_time - last_refill_time - + # Calculate tokens to add tokens_to_add = (time_passed / 60.0) * self.config.requests_per_minute self.buckets[client_key] = min( @@ -262,11 +262,11 @@ def _refill_bucket(self, client_key: str): self.buckets[client_key] + tokens_to_add ) self.last_refill[client_key] = current_time - + def allow_request(self, client_ip: str, user_agent: str = "") -> Tuple[bool, str, Dict]: """ Check if request should be allowed. - + Returns: Tuple of (allowed, reason, metadata) """ @@ -274,13 +274,13 @@ def allow_request(self, client_ip: str, user_agent: str = "") -> Tuple[bool, str # Check IP allowlist/blocklist if not self._is_ip_allowed(client_ip): return False, "IP not allowed", {"ip": client_ip} - + client_key = self._get_client_key(client_ip, user_agent) - + # Check if client is blocked if self._is_client_blocked(client_key): return False, "Client blocked", {"client_key": client_key, "ip": client_ip} - + # Check concurrent requests if self.concurrent_requests[client_key] >= self.config.max_concurrent_requests: return False, "Too many concurrent requests", { @@ -288,16 +288,16 @@ def allow_request(self, client_ip: str, user_agent: str = "") -> Tuple[bool, str "concurrent": self.concurrent_requests[client_key], "max": self.config.max_concurrent_requests } - + # Enhanced abuse detection with user agent if self._detect_abuse(client_key, client_ip, user_agent): self.blocked_clients[client_key] = time.time() + self.config.block_duration_seconds logger.warning(f"Blocked abusive client {client_key} from {client_ip} for {self.config.block_duration_seconds}s") return False, "Abuse detected", {"client_key": client_key, "ip": client_ip} - + # Refill bucket self._refill_bucket(client_key) - + # Check if tokens available if self.buckets[client_key] < 0.999999: # Use small epsilon to handle floating-point precision return False, "Rate limit exceeded", { @@ -305,29 +305,29 @@ def allow_request(self, client_ip: str, user_agent: str = "") -> Tuple[bool, str "tokens": self.buckets[client_key], "rate_limit": self.config.requests_per_minute } - + # Consume token self.buckets[client_key] -= 1.0 - + # Update request history self.request_history[client_key].append(time.time()) - + # Increment concurrent requests self.concurrent_requests[client_key] += 1 - + return True, "Request allowed", { "client_key": client_key, "tokens_remaining": self.buckets[client_key], "concurrent_requests": self.concurrent_requests[client_key] } - + def release_request(self, client_ip: str, user_agent: str = ""): """Release a concurrent request slot.""" with self.lock: client_key = self._get_client_key(client_ip, user_agent) if client_key in self.concurrent_requests: self.concurrent_requests[client_key] = max(0, self.concurrent_requests[client_key] - 1) - + def get_stats(self) -> Dict: """Get rate limiter statistics.""" with self.lock: @@ -343,31 +343,31 @@ def get_stats(self) -> Dict: "block_duration_seconds": self.config.block_duration_seconds } } - + def add_to_blacklist(self, ip: str): """Add IP to blacklist.""" with self.lock: self.config.blacklisted_ips.add(ip) logger.info(f"Added {ip} to blacklist") - + def remove_from_blacklist(self, ip: str): """Remove IP from blacklist.""" with self.lock: self.config.blacklisted_ips.discard(ip) logger.info(f"Removed {ip} from blacklist") - + def add_to_whitelist(self, ip: str): """Add IP to whitelist.""" with self.lock: self.config.whitelisted_ips.add(ip) logger.info(f"Added {ip} to whitelist") - + def remove_from_whitelist(self, ip: str): """Remove IP from whitelist.""" with self.lock: self.config.whitelisted_ips.discard(ip) logger.info(f"Removed {ip} from whitelist") - + def reset_state(self): """Reset all rate limiter state for testing.""" with self.lock: @@ -384,7 +384,7 @@ def add_rate_limiting(app, requests_per_minute=100, burst_size=10, max_concurren """Add rate limiting middleware to FastAPI app.""" from fastapi import Request from fastapi.responses import JSONResponse - + # Create rate limiter instance config = RateLimitConfig( requests_per_minute=requests_per_minute, @@ -396,24 +396,24 @@ def add_rate_limiting(app, requests_per_minute=100, burst_size=10, max_concurren sustained_rate_threshold=sustained_rate_threshold ) rate_limiter = TokenBucketRateLimiter(config) - + # Store rate limiter instance on app for testing app.state.rate_limiter = rate_limiter - + @app.middleware("http") async def rate_limit_middleware(request: Request, call_next): """Rate limiting middleware.""" client_ip = request.client.host if request.client else "unknown" user_agent = request.headers.get("user-agent", "") - + # Bypass rate limiting for test environment if "test" in user_agent.lower() or "pytest" in user_agent.lower() or "testclient" in user_agent.lower(): response = await call_next(request) return response - + # Check rate limit allowed, reason, meta = rate_limiter.allow_request(client_ip, user_agent) - + if not allowed: return JSONResponse( status_code=429, @@ -423,11 +423,11 @@ async def rate_limit_middleware(request: Request, call_next): "retry_after": meta.get("retry_after", 60) } ) - + # Add rate limit headers response = await call_next(request) response.headers["X-RateLimit-Limit"] = str(config.requests_per_minute) response.headers["X-RateLimit-Remaining"] = str(meta.get("tokens_remaining", 0)) response.headers["X-RateLimit-Reset"] = str(meta.get("reset_time", 0)) - + return response diff --git a/src/input_sanitizer.py b/src/input_sanitizer.py index ecc8a1b0c..acf851701 100644 --- a/src/input_sanitizer.py +++ b/src/input_sanitizer.py @@ -31,7 +31,7 @@ class SanitizationConfig: class InputSanitizer: """ Comprehensive input sanitization and validation. - + Features: - XSS protection - SQL injection protection @@ -42,10 +42,10 @@ class InputSanitizer: - Length limits - Pattern blocking """ - + def __init__(self, config: SanitizationConfig): self.config = config - + # Initialize default blocked patterns if config.blocked_patterns is None: config.blocked_patterns = { @@ -56,63 +56,63 @@ def __init__(self, config: SanitizationConfig): r']*>', r']*>', r']*>', - + # SQL injection patterns r'(\b(union|select|insert|update|delete|drop|create|alter|exec|execute)\b)', r'(\b(or|and)\b\s+\d+\s*=\s*\d+)', r'(\b(union|select)\b.*?\bfrom\b)', r'(\b(insert|update|delete)\b.*?\binto\b)', - + # Path traversal patterns r'\.\./', r'\.\.\\', r'%2e%2e%2f', r'%2e%2e%5c', - + # Command injection patterns r'(\b(cmd|command|exec|system|eval|exec)\b)', r'(\b(popen|subprocess|os\.system)\b)', r'(\b(shell|bash|sh|powershell)\b)', r'(\b(rm|del|format|mkfs)\b)', - + # Other dangerous patterns r'(\b(import|__import__)\b)', r'(\b(eval|exec|compile)\b)', r'(\b(open|file|read|write)\b)', r'(\b(subprocess|multiprocessing)\b)', } - + # Initialize allowed HTML tags if config.allowed_html_tags is None: config.allowed_html_tags = { 'p', 'br', 'strong', 'em', 'u', 'i', 'b', 'span', 'div' } - + def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[str]]: """ Sanitize text input. - + Args: text: Input text to sanitize context: Context for sanitization (e.g., "emotion", "general") - + Returns: Tuple of (sanitized_text, warnings) """ warnings = [] - + if not isinstance(text, str): raise ValueError(f"Input must be a string, got {type(text)}") - + # Check length if len(text) > self.config.max_text_length: warnings.append(f"Text truncated from {len(text)} to {self.config.max_text_length} characters") text = text[:self.config.max_text_length] - + # Unicode normalization if self.config.enable_unicode_normalization: text = unicodedata.normalize('NFKC', text) - + # Check for blocked patterns if self.config.enable_xss_protection or self.config.enable_sql_injection_protection: for pattern in self.config.blocked_patterns: @@ -120,37 +120,37 @@ def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[ warnings.append(f"Blocked pattern detected: {pattern}") # Replace with safe alternative text = re.sub(pattern, '[BLOCKED]', text, flags=re.IGNORECASE) - + # HTML escaping for XSS protection if self.config.enable_xss_protection: text = html.escape(text) - + # Remove null bytes and control characters text = ''.join(char for char in text if ord(char) >= 32 or char in '\n\r\t') - + # Strip leading/trailing whitespace text = text.strip() - + return text, warnings - + def sanitize_json(self, data: Any, max_depth: int = 10) -> Tuple[Any, List[str]]: """ Sanitize JSON data recursively. - + Args: data: JSON data to sanitize max_depth: Maximum recursion depth - + Returns: Tuple of (sanitized_data, warnings) """ warnings = [] - + def _sanitize_recursive(obj: Any, depth: int = 0) -> Any: if depth > max_depth: warnings.append(f"Maximum recursion depth {max_depth} exceeded") return None - + if isinstance(obj, str): sanitized, obj_warnings = self.sanitize_text(obj) warnings.extend(obj_warnings) @@ -164,34 +164,34 @@ def _sanitize_recursive(obj: Any, depth: int = 0) -> Any: else: warnings.append(f"Unsupported type {type(obj)} converted to string") return str(obj) - + return _sanitize_recursive(data), warnings - + def validate_emotion_request(self, data: Dict) -> Tuple[Dict, List[str]]: """ Validate and sanitize emotion detection request. - + Args: data: Request data - + Returns: Tuple of (sanitized_data, warnings) """ warnings = [] sanitized_data = {} - + # Validate text field if 'text' not in data: raise ValueError("Missing required field 'text'") - + text = data['text'] if not isinstance(text, str): raise ValueError("Field 'text' must be a string") - + sanitized_text, text_warnings = self.sanitize_text(text, "emotion") sanitized_data['text'] = sanitized_text warnings.extend(text_warnings) - + # Validate optional fields if 'confidence_threshold' in data: try: @@ -202,48 +202,48 @@ def validate_emotion_request(self, data: Dict) -> Tuple[Dict, List[str]]: warnings.append("confidence_threshold must be between 0.0 and 1.0") except (ValueError, TypeError): warnings.append("confidence_threshold must be a number") - + return sanitized_data, warnings - + def validate_batch_request(self, data: Dict) -> Tuple[Dict, List[str]]: """ Validate and sanitize batch emotion detection request. - + Args: data: Request data - + Returns: Tuple of (sanitized_data, warnings) """ warnings = [] sanitized_data = {} - + # Validate texts field if 'texts' not in data: raise ValueError("Missing required field 'texts'") - + texts = data['texts'] if not isinstance(texts, list): raise ValueError("Field 'texts' must be a list") - + # Check batch size if len(texts) > self.config.max_batch_size: warnings.append(f"Batch size {len(texts)} exceeds maximum {self.config.max_batch_size}") texts = texts[:self.config.max_batch_size] - + # Sanitize each text sanitized_texts = [] for i, text in enumerate(texts): if not isinstance(text, str): warnings.append(f"Text at index {i} is not a string, skipping") continue - + sanitized_text, text_warnings = self.sanitize_text(text, "emotion") sanitized_texts.append(sanitized_text) warnings.extend([f"Text {i}: {w}" for w in text_warnings]) - + sanitized_data['texts'] = sanitized_texts - + # Validate optional fields if 'confidence_threshold' in data: try: @@ -254,90 +254,90 @@ def validate_batch_request(self, data: Dict) -> Tuple[Dict, List[str]]: warnings.append("confidence_threshold must be between 0.0 and 1.0") except (ValueError, TypeError): warnings.append("confidence_threshold must be a number") - + return sanitized_data, warnings - + def validate_content_type(self, content_type: str) -> bool: """ Validate content type header. - + Args: content_type: Content type header value - + Returns: True if valid, False otherwise """ if not self.config.enable_content_type_validation: return True - + # Check for JSON content type if not content_type or 'application/json' not in content_type.lower(): return False - + return True - + def sanitize_headers(self, headers: Dict[str, str]) -> Tuple[Dict[str, str], List[str]]: """ Sanitize HTTP headers. - + Args: headers: HTTP headers - + Returns: Tuple of (sanitized_headers, warnings) """ warnings = [] sanitized_headers = {} - + for key, value in headers.items(): if not isinstance(key, str) or not isinstance(value, str): warnings.append(f"Invalid header type: {key}") continue - + # Sanitize header name and value sanitized_key, key_warnings = self.sanitize_text(key, "header") sanitized_value, value_warnings = self.sanitize_text(value, "header") - + sanitized_headers[sanitized_key] = sanitized_value warnings.extend(key_warnings) warnings.extend(value_warnings) - + return sanitized_headers, warnings - + def detect_anomalies(self, data: Any) -> List[str]: """ Detect potential security anomalies in data. - + Args: data: Data to analyze - + Returns: List of detected anomalies """ anomalies = [] - + def _analyze_recursive(obj: Any, path: str = ""): if isinstance(obj, str): # Check for suspicious patterns if len(obj) > 1000: anomalies.append(f"Large string at {path}: {len(obj)} characters") - + if re.search(r'[<>"\']', obj): anomalies.append(f"Potential HTML/script content at {path}") - + if re.search(r'\b(union|select|insert|update|delete)\b', obj, re.IGNORECASE): anomalies.append(f"Potential SQL injection at {path}") - + elif isinstance(obj, dict): for key, value in obj.items(): _analyze_recursive(value, f"{path}.{key}" if path else key) elif isinstance(obj, list): for i, item in enumerate(obj): _analyze_recursive(item, f"{path}[{i}]") - + _analyze_recursive(data) return anomalies - + def get_sanitization_stats(self) -> Dict: """Get sanitization statistics.""" return { diff --git a/src/models/emotion_detection/dataset_loader.py b/src/models/emotion_detection/dataset_loader.py index 8c094f5ed..6b10a2289 100644 --- a/src/models/emotion_detection/dataset_loader.py +++ b/src/models/emotion_detection/dataset_loader.py @@ -92,11 +92,11 @@ def clean_text(self, text: str) -> str: # Remove excessive whitespace while preserving structure text = re.sub(r'\s+', ' ', text.strip()) - + # The dataset is already split by HuggingFace # Tokenize with BERT tokenizer # Use the same logic as analyze_dataset_statistics - + return text def tokenize_batch(self, texts: List[str]) -> Dict[str, torch.Tensor]: @@ -110,7 +110,7 @@ def tokenize_batch(self, texts: List[str]) -> Dict[str, torch.Tensor]: """ # Clean texts first cleaned_texts = [self.clean_text(text) for text in texts] - + # Tokenize with BERT tokenizer encoded = self.tokenizer( cleaned_texts, @@ -119,7 +119,7 @@ def tokenize_batch(self, texts: List[str]) -> Dict[str, torch.Tensor]: max_length=self.max_length, return_tensors="pt", ) - + return encoded @@ -151,7 +151,7 @@ def __init__( self.test_size = test_size self.val_size = val_size self.random_state = random_state - + self.preprocessor = GoEmotionsPreprocessor(model_name, max_length) self.dataset = None self.train_dataset = None @@ -182,11 +182,11 @@ def analyze_dataset_statistics(self) -> Dict[str, Any]: self.download_dataset() stats = {} - + # Basic statistics stats["total_samples"] = len(self.dataset["train"]) stats["num_emotions"] = len(GOEMOTIONS_EMOTIONS) - + # Emotion distribution emotion_counts = Counter() for example in self.dataset["train"]: @@ -194,17 +194,17 @@ def analyze_dataset_statistics(self) -> Dict[str, Any]: for label in labels: if 0 <= label < len(GOEMOTIONS_EMOTIONS): emotion_counts[label] += 1 - + stats["emotion_distribution"] = dict(emotion_counts) stats["most_common_emotions"] = emotion_counts.most_common(10) stats["least_common_emotions"] = emotion_counts.most_common()[:-11:-1] - + # Text length statistics text_lengths = [len(example["text"]) for example in self.dataset["train"]] stats["avg_text_length"] = np.mean(text_lengths) stats["max_text_length"] = np.max(text_lengths) stats["min_text_length"] = np.min(text_lengths) - + logger.info(f"Dataset statistics: {stats}") return stats @@ -228,10 +228,10 @@ def compute_class_weights(self) -> np.ndarray: # Compute inverse frequency weights total_samples = len(self.dataset["train"]) class_weights = total_samples / (len(GOEMOTIONS_EMOTIONS) * emotion_counts) - + # Handle zero counts class_weights[emotion_counts == 0] = 1.0 - + logger.info(f"Computed class weights: min={class_weights.min():.3f}, max={class_weights.max():.3f}") return class_weights @@ -249,19 +249,19 @@ def create_train_val_test_splits(self) -> tuple: test_size=self.test_size + self.val_size, seed=self.random_state, ) - + # Split validation from test val_test = train_val_test["test"].train_test_split( test_size=self.val_size / (self.test_size + self.val_size), seed=self.random_state, ) - + train_data = train_val_test["train"] val_data = val_test["train"] test_data = val_test["test"] - + logger.info(f"Created splits - Train: {len(train_data)}, Val: {len(val_data)}, Test: {len(test_data)}") - + return train_data, val_data, test_data def prepare_datasets(self, force_download: bool = False) -> dict: @@ -278,13 +278,13 @@ def prepare_datasets(self, force_download: bool = False) -> dict: # Create splits train_data, val_data, test_data = self.create_train_val_test_splits() - + # Compute class weights class_weights = self.compute_class_weights() - + # Analyze statistics stats = self.analyze_dataset_statistics() - + return { "train_data": train_data, "val_data": val_data, diff --git a/src/models/secure_loader/integrity_checker.py b/src/models/secure_loader/integrity_checker.py index 5ec4cada5..f3b976f30 100644 --- a/src/models/secure_loader/integrity_checker.py +++ b/src/models/secure_loader/integrity_checker.py @@ -19,7 +19,7 @@ class IntegrityChecker: """Model integrity checker for secure model loading. - + Provides comprehensive integrity verification including: - SHA-256 checksums - File format validation @@ -29,13 +29,13 @@ class IntegrityChecker: def __init__(self, trusted_checksums_file: Optional[str] = None): """Initialize integrity checker. - + Args: trusted_checksums_file: Path to file containing trusted checksums """ self.trusted_checksums_file = trusted_checksums_file self.trusted_checksums = self._load_trusted_checksums() - + # Security constraints self.max_file_size = 2 * 1024 * 1024 * 1024 # 2GB max self.allowed_extensions = {'.pt', '.pth', '.bin', '.safetensors'} @@ -46,14 +46,14 @@ def __init__(self, trusted_checksums_file: Optional[str] = None): def _load_trusted_checksums(self) -> Dict[str, str]: """Load trusted checksums from file. - + Returns: Dictionary mapping file paths to expected checksums """ if not self.trusted_checksums_file or not os.path.exists(self.trusted_checksums_file): logger.warning("No trusted checksums file found, using empty trust store") return {} - + try: with open(self.trusted_checksums_file, 'r') as f: return json.load(f) @@ -63,15 +63,15 @@ def _load_trusted_checksums(self) -> Dict[str, str]: def calculate_checksum(self, file_path: str) -> str: """Calculate SHA-256 checksum of a file. - + Args: file_path: Path to the file - + Returns: SHA-256 checksum as hex string """ sha256_hash = hashlib.sha256() - + try: with open(file_path, 'rb') as f: for chunk in iter(lambda: f.read(4096), b""): @@ -83,10 +83,10 @@ def calculate_checksum(self, file_path: str) -> str: def validate_file_size(self, file_path: str) -> bool: """Validate file size is within acceptable limits. - + Args: file_path: Path to the file - + Returns: True if file size is acceptable """ @@ -102,10 +102,10 @@ def validate_file_size(self, file_path: str) -> bool: def validate_file_extension(self, file_path: str) -> bool: """Validate file extension is allowed. - + Args: file_path: Path to the file - + Returns: True if file extension is allowed """ @@ -117,93 +117,93 @@ def validate_file_extension(self, file_path: str) -> bool: def scan_for_malicious_content(self, file_path: str) -> Tuple[bool, list]: """Scan file for potentially malicious content. - + Args: file_path: Path to the file - + Returns: Tuple of (is_safe, list_of_findings) """ findings = [] - + try: with open(file_path, 'rb') as f: content = f.read() - + for pattern in self.blocked_patterns: if pattern in content: findings.append(f"Found blocked pattern: {pattern}") - + except Exception as e: logger.error(f"Failed to scan file {file_path}: {e}") findings.append(f"Scan failed: {e}") - + return len(findings) == 0, findings def verify_checksum(self, file_path: str, expected_checksum: Optional[str] = None) -> bool: """Verify file checksum against expected value. - + Args: file_path: Path to the file expected_checksum: Expected checksum (if None, uses trusted checksums) - + Returns: True if checksum matches """ try: actual_checksum = self.calculate_checksum(file_path) - + if expected_checksum: return actual_checksum == expected_checksum - + # Check against trusted checksums if file_path in self.trusted_checksums: return actual_checksum == self.trusted_checksums[file_path] - + logger.warning(f"No expected checksum provided for {file_path}") return False - + except Exception as e: logger.error(f"Failed to verify checksum for {file_path}: {e}") return False def validate_model_structure(self, model_path: str) -> bool: """Validate PyTorch model structure. - + Args: model_path: Path to the model file - + Returns: True if model structure is valid """ try: # Load model in a controlled environment model_data = torch.load(model_path, map_location='cpu', weights_only=True) - + # Basic structure validation if not isinstance(model_data, dict): logger.error(f"Model {model_path} is not a valid state dict") return False - + # Check for required keys in state dict required_keys = ['state_dict', 'config', 'model_name'] for key in required_keys: if key not in model_data: logger.warning(f"Model {model_path} missing key: {key}") - + return True - + except Exception as e: logger.error(f"Failed to validate model structure for {model_path}: {e}") return False def comprehensive_validation(self, file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, Dict]: """Perform comprehensive file validation. - + Args: file_path: Path to the file expected_checksum: Expected checksum - + Returns: Tuple of (is_valid, validation_results) """ @@ -216,33 +216,33 @@ def comprehensive_validation(self, file_path: str, expected_checksum: Optional[s 'structure_valid': False, 'findings': [] } - + # File size validation results['size_valid'] = self.validate_file_size(file_path) if not results['size_valid']: results['findings'].append("File size exceeds limit") - + # Extension validation results['extension_valid'] = self.validate_file_extension(file_path) if not results['extension_valid']: results['findings'].append("File extension not allowed") - + # Checksum validation results['checksum_valid'] = self.verify_checksum(file_path, expected_checksum) if not results['checksum_valid']: results['findings'].append("Checksum verification failed") - + # Content safety scan is_safe, findings = self.scan_for_malicious_content(file_path) results['content_safe'] = is_safe results['findings'].extend(findings) - + # Model structure validation (only for model files) if Path(file_path).suffix.lower() in {'.pt', '.pth'}: results['structure_valid'] = self.validate_model_structure(file_path) if not results['structure_valid']: results['findings'].append("Model structure validation failed") - + # Overall validation result is_valid = all([ results['size_valid'], @@ -250,8 +250,8 @@ def comprehensive_validation(self, file_path: str, expected_checksum: Optional[s results['checksum_valid'], results['content_safe'] ]) - + if Path(file_path).suffix.lower() in {'.pt', '.pth'}: is_valid = is_valid and results['structure_valid'] - + return is_valid, results \ No newline at end of file diff --git a/src/models/secure_loader/model_validator.py b/src/models/secure_loader/model_validator.py index de5b51a52..226e1063e 100644 --- a/src/models/secure_loader/model_validator.py +++ b/src/models/secure_loader/model_validator.py @@ -19,7 +19,7 @@ class ModelValidator: """Model validator for secure model loading. - + Provides comprehensive model validation including: - Model structure validation - Version compatibility checks @@ -32,7 +32,7 @@ def __init__(self, max_model_size_mb: int = 2048, required_config_keys: Optional[List[str]] = None): """Initialize model validator. - + Args: allowed_model_types: List of allowed model types max_model_size_mb: Maximum model size in MB @@ -45,7 +45,7 @@ def __init__(self, self.required_config_keys = required_config_keys or [ 'model_name', 'num_emotions', 'hidden_dropout_prob' ] - + # Version compatibility matrix self.version_compatibility = { 'torch': '>=1.9.0', @@ -55,10 +55,10 @@ def __init__(self, def validate_model_structure(self, model: nn.Module) -> Tuple[bool, Dict]: """Validate model structure. - + Args: model: PyTorch model to validate - + Returns: Tuple of (is_valid, validation_info) """ @@ -68,20 +68,20 @@ def validate_model_structure(self, model: nn.Module) -> Tuple[bool, Dict]: 'layers': [], 'issues': [] } - + try: # Check model type if type(model).__name__ not in self.allowed_model_types: validation_info['issues'].append(f"Model type {type(model).__name__} not allowed") - + # Count parameters param_count = sum(p.numel() for p in model.parameters()) validation_info['parameter_count'] = param_count - + # Check for reasonable parameter count if param_count > 500_000_000: # 500M parameters validation_info['issues'].append("Model has too many parameters") - + # Analyze model layers for name, module in model.named_modules(): if isinstance(module, (nn.Linear, nn.Conv2d, nn.LSTM, nn.Transformer)): @@ -90,26 +90,26 @@ def validate_model_structure(self, model: nn.Module) -> Tuple[bool, Dict]: 'type': type(module).__name__, 'parameters': sum(p.numel() for p in module.parameters()) }) - + # Check for required methods required_methods = ['forward', 'eval', 'train'] for method in required_methods: if not hasattr(model, method): validation_info['issues'].append(f"Missing required method: {method}") - + is_valid = len(validation_info['issues']) == 0 return is_valid, validation_info - + except Exception as e: validation_info['issues'].append(f"Validation error: {e}") return False, validation_info def validate_model_config(self, config: Dict[str, Any]) -> Tuple[bool, Dict]: """Validate model configuration. - + Args: config: Model configuration dictionary - + Returns: Tuple of (is_valid, validation_info) """ @@ -119,44 +119,44 @@ def validate_model_config(self, config: Dict[str, Any]) -> Tuple[bool, Dict]: 'invalid_values': [], 'issues': [] } - + try: # Check required keys for key in self.required_config_keys: if key not in config: validation_info['missing_keys'].append(key) - + # Validate specific config values if 'num_emotions' in config: num_emotions = config['num_emotions'] if not isinstance(num_emotions, int) or num_emotions <= 0: validation_info['invalid_values'].append(f"num_emotions: {num_emotions}") - + if 'hidden_dropout_prob' in config: dropout = config['hidden_dropout_prob'] if not isinstance(dropout, (int, float)) or dropout < 0 or dropout > 1: validation_info['invalid_values'].append(f"hidden_dropout_prob: {dropout}") - + # Check for issues if validation_info['missing_keys']: validation_info['issues'].append(f"Missing required keys: {validation_info['missing_keys']}") - + if validation_info['invalid_values']: validation_info['issues'].append(f"Invalid values: {validation_info['invalid_values']}") - + is_valid = len(validation_info['issues']) == 0 return is_valid, validation_info - + except Exception as e: validation_info['issues'].append(f"Config validation error: {e}") return False, validation_info def validate_model_file(self, model_path: str) -> Tuple[bool, Dict]: """Validate model file. - + Args: model_path: Path to the model file - + Returns: Tuple of (is_valid, validation_info) """ @@ -168,35 +168,35 @@ def validate_model_file(self, model_path: str) -> Tuple[bool, Dict]: 'loadable': False, 'issues': [] } - + try: # Check file existence if not os.path.exists(model_path): validation_info['issues'].append("Model file does not exist") return False, validation_info - + validation_info['file_exists'] = True - + # Check file size file_size = os.path.getsize(model_path) file_size_mb = file_size / (1024 * 1024) validation_info['file_size_mb'] = file_size_mb - + if file_size_mb > self.max_model_size_mb: validation_info['issues'].append(f"Model file too large: {file_size_mb:.2f}MB") - + # Check if file is readable if not os.access(model_path, os.R_OK): validation_info['issues'].append("Model file is not readable") return False, validation_info - + validation_info['is_readable'] = True - + # Try to load the model try: model_data = torch.load(model_path, map_location='cpu', weights_only=True) validation_info['loadable'] = True - + # Validate model data structure if not isinstance(model_data, dict): validation_info['issues'].append("Model file is not a valid state dict") @@ -204,26 +204,26 @@ def validate_model_file(self, model_path: str) -> Tuple[bool, Dict]: # Check for required keys if 'state_dict' not in model_data: validation_info['issues'].append("Model file missing state_dict") - + if 'config' not in model_data: validation_info['issues'].append("Model file missing config") - + except Exception as e: validation_info['issues'].append(f"Failed to load model: {e}") - + is_valid = len(validation_info['issues']) == 0 return is_valid, validation_info - + except Exception as e: validation_info['issues'].append(f"File validation error: {e}") return False, validation_info def validate_version_compatibility(self, model_config: Dict[str, Any]) -> Tuple[bool, Dict]: """Validate version compatibility. - + Args: model_config: Model configuration - + Returns: Tuple of (is_valid, validation_info) """ @@ -233,17 +233,17 @@ def validate_version_compatibility(self, model_config: Dict[str, Any]) -> Tuple[ 'compatibility_issues': [], 'issues': [] } - + try: # Get current versions import torch import transformers - + validation_info['current_versions'] = { 'torch': torch.__version__, 'transformers': transformers.__version__ } - + # Check version compatibility for package, required_version in self.version_compatibility.items(): if package in validation_info['current_versions']: @@ -255,25 +255,25 @@ def validate_version_compatibility(self, model_config: Dict[str, Any]) -> Tuple[ validation_info['compatibility_issues'].append(f"PyTorch version {current_version} may not be compatible") elif package == 'transformers' and not current_version.startswith('4.'): validation_info['compatibility_issues'].append(f"Transformers version {current_version} may not be compatible") - + # Check for issues if validation_info['compatibility_issues']: validation_info['issues'].extend(validation_info['compatibility_issues']) - + is_valid = len(validation_info['issues']) == 0 return is_valid, validation_info - + except Exception as e: validation_info['issues'].append(f"Version validation error: {e}") return False, validation_info def validate_model_performance(self, model: nn.Module, test_input: torch.Tensor) -> Tuple[bool, Dict]: """Validate model performance with test input. - + Args: model: PyTorch model test_input: Test input tensor - + Returns: Tuple of (is_valid, validation_info) """ @@ -283,41 +283,41 @@ def validate_model_performance(self, model: nn.Module, test_input: torch.Tensor) 'output_shape': None, 'issues': [] } - + try: import time - + # Set model to eval mode model.eval() - + # Measure forward pass time start_time = time.time() with torch.no_grad(): output = model(test_input) end_time = time.time() - + validation_info['forward_pass_time'] = end_time - start_time validation_info['output_shape'] = list(output.shape) - + # Check performance constraints if validation_info['forward_pass_time'] > 5.0: # 5 seconds validation_info['issues'].append("Forward pass too slow") - + # Check output shape if output.dim() != 2: # Expected 2D output for classification validation_info['issues'].append("Unexpected output shape") - + # Measure memory usage if hasattr(torch.cuda, 'memory_allocated'): memory_mb = torch.cuda.memory_allocated() / (1024 * 1024) validation_info['memory_usage_mb'] = memory_mb - + if memory_mb > 2048: # 2GB validation_info['issues'].append("Memory usage too high") - + is_valid = len(validation_info['issues']) == 0 return is_valid, validation_info - + except Exception as e: validation_info['issues'].append(f"Performance validation error: {e}") return False, validation_info @@ -328,13 +328,13 @@ def comprehensive_validation(self, model_config: Dict[str, Any], test_input: Optional[torch.Tensor] = None) -> Tuple[bool, Dict]: """Perform comprehensive model validation. - + Args: model_path: Path to the model file model_class: Model class model_config: Model configuration test_input: Optional test input for performance validation - + Returns: Tuple of (is_valid, comprehensive_validation_info) """ @@ -347,60 +347,60 @@ def comprehensive_validation(self, 'overall_valid': False, 'issues': [] } - + try: # 1. File validation file_valid, file_info = self.validate_model_file(model_path) comprehensive_info['file_validation'] = file_info if not file_valid: comprehensive_info['issues'].extend(file_info['issues']) - + # 2. Config validation config_valid, config_info = self.validate_model_config(model_config) comprehensive_info['config_validation'] = config_info if not config_valid: comprehensive_info['issues'].extend(config_info['issues']) - + # 3. Version validation version_valid, version_info = self.validate_version_compatibility(model_config) comprehensive_info['version_validation'] = version_info if not version_valid: comprehensive_info['issues'].extend(version_info['issues']) - + # 4. Structure validation (if file is valid) if file_valid: try: model_data = torch.load(model_path, map_location='cpu', weights_only=True) - + # Filter model_config to only include valid constructor parameters import inspect constructor_params = inspect.signature(model_class.__init__).parameters valid_params = {k: v for k, v in model_config.items() if k in constructor_params} model = model_class(**valid_params) - + if 'state_dict' in model_data: model.load_state_dict(model_data['state_dict']) - + structure_valid, structure_info = self.validate_model_structure(model) comprehensive_info['structure_validation'] = structure_info if not structure_valid: comprehensive_info['issues'].extend(structure_info['issues']) - + # 5. Performance validation (if structure is valid and test input provided) if structure_valid and test_input is not None: perf_valid, perf_info = self.validate_model_performance(model, test_input) comprehensive_info['performance_validation'] = perf_info if not perf_valid: comprehensive_info['issues'].extend(perf_info['issues']) - + except Exception as e: comprehensive_info['issues'].append(f"Model loading error: {e}") - + # Overall validation result comprehensive_info['overall_valid'] = len(comprehensive_info['issues']) == 0 - + return comprehensive_info['overall_valid'], comprehensive_info - + except Exception as e: comprehensive_info['issues'].append(f"Comprehensive validation error: {e}") return False, comprehensive_info \ No newline at end of file diff --git a/src/models/secure_loader/sandbox_executor.py b/src/models/secure_loader/sandbox_executor.py index 48f345065..0e0da05fe 100644 --- a/src/models/secure_loader/sandbox_executor.py +++ b/src/models/secure_loader/sandbox_executor.py @@ -37,7 +37,7 @@ def to_dict(self): class SandboxExecutor: """Sandbox executor for secure model loading. - + Provides isolated execution environment with: - Resource limits (CPU, memory, time) - Restricted file system access @@ -51,7 +51,7 @@ def __init__(self, max_wall_time: int = 60, allow_network: bool = False): """Initialize sandbox executor. - + Args: max_memory_mb: Maximum memory usage in MB max_cpu_time: Maximum CPU time in seconds @@ -62,13 +62,13 @@ def __init__(self, self.max_cpu_time = max_cpu_time self.max_wall_time = max_wall_time self.allow_network = allow_network - + # Restricted operations self.blocked_modules = { 'subprocess', 'os', 'sys', 'builtins', 'importlib', 'pickle', 'marshal', 'code', 'types' } - + # Restricted functions self.blocked_functions = { 'eval', 'exec', 'compile', 'open', 'file', @@ -81,15 +81,15 @@ def _set_resource_limits(self): # Memory limit (soft and hard) memory_limit = self.max_memory_mb * 1024 * 1024 # Convert to bytes resource.setrlimit(resource.RLIMIT_AS, (memory_limit, memory_limit)) - + # CPU time limit resource.setrlimit(resource.RLIMIT_CPU, (self.max_cpu_time, self.max_cpu_time)) - + # File size limit resource.setrlimit(resource.RLIMIT_FSIZE, (1024 * 1024 * 1024, 1024 * 1024 * 1024)) # 1GB - + logger.debug(f"Resource limits set: memory={self.max_memory_mb}MB, cpu={self.max_cpu_time}s") - + except Exception as e: logger.error(f"Failed to set resource limits: {e}") @@ -152,12 +152,12 @@ def _disable_network(self): try: import socket original_socket = socket.socket - + def blocked_socket(*args, **kwargs): raise PermissionError("Network access is not allowed in sandbox") - + socket.socket = blocked_socket - + except ImportError: pass # socket module not available @@ -184,63 +184,63 @@ def execute_safely(self, func: Callable, *args, **kwargs) -> Tuple[Any, Dict]: def load_model_safely(self, model_path: str, model_class: type, **kwargs) -> Any: """Load a model safely in the sandbox. - + Args: model_path: Path to the model file model_class: Model class to instantiate **kwargs: Additional arguments for model loading - + Returns: Loaded model instance """ def load_model(): # Use torch.load with weights_only=True for additional safety model_data = torch.load(model_path, map_location='cpu', weights_only=True) - + # Filter kwargs to only include valid constructor parameters import inspect constructor_params = inspect.signature(model_class.__init__).parameters valid_params = {k: v for k, v in kwargs.items() if k in constructor_params} - + # Create model instance model = model_class(**valid_params) - + # Load state dict if available if 'state_dict' in model_data: model.load_state_dict(model_data['state_dict']) - + return model - + result, execution_info = self.execute_safely(load_model) logger.info(f"Model loaded safely: {execution_info}") return result, execution_info def validate_model_safely(self, model_path: str) -> Tuple[bool, Dict]: """Validate a model safely in the sandbox. - + Args: model_path: Path to the model file - + Returns: Tuple of (is_valid, validation_info) """ def validate_model(): # Load model data model_data = torch.load(model_path, map_location='cpu', weights_only=True) - + # Basic validation if not isinstance(model_data, dict): return False, {"error": "Model is not a valid state dict"} - + # Check for required keys required_keys = ['state_dict'] missing_keys = [key for key in required_keys if key not in model_data] - + if missing_keys: return False, {"error": f"Missing required keys: {missing_keys}"} - + return True, {"message": "Model validation successful"} - + try: result, execution_info = self.execute_safely(validate_model) return result @@ -249,17 +249,17 @@ def validate_model(): def get_resource_usage(self) -> Dict[str, float]: """Get current resource usage. - + Returns: Dictionary with resource usage information """ try: import psutil - + process = psutil.Process() memory_info = process.memory_info() cpu_percent = process.cpu_percent() - + return { 'memory_mb': memory_info.rss / 1024 / 1024, 'cpu_percent': cpu_percent, @@ -274,10 +274,10 @@ def cleanup(self): try: # Cancel any pending alarms signal.alarm(0) - + # Clear any cached models if hasattr(torch, 'cuda'): torch.cuda.empty_cache() - + except Exception as e: logger.error(f"Cleanup error: {e}") \ No newline at end of file diff --git a/src/models/secure_loader/secure_model_loader.py b/src/models/secure_loader/secure_model_loader.py index 14d1ae916..0bd924386 100644 --- a/src/models/secure_loader/secure_model_loader.py +++ b/src/models/secure_loader/secure_model_loader.py @@ -22,7 +22,7 @@ class SecureModelLoader: """Secure model loader with defense-in-depth security. - + Provides comprehensive secure model loading with: - Integrity verification (checksums, file validation) - Sandboxed execution (resource limits, isolation) @@ -39,7 +39,7 @@ def __init__(self, max_cache_size_mb: int = 1024, audit_log_file: Optional[str] = None): """Initialize secure model loader. - + Args: trusted_checksums_file: Path to trusted checksums file enable_sandbox: Whether to enable sandboxed execution @@ -53,32 +53,32 @@ def __init__(self, self.cache_dir = cache_dir or os.path.join(os.getcwd(), '.model_cache') self.max_cache_size_mb = max_cache_size_mb self.audit_log_file = audit_log_file - + # Initialize security components self.integrity_checker = IntegrityChecker(trusted_checksums_file) self.sandbox_executor = SandboxExecutor() if enable_sandbox else None self.model_validator = ModelValidator() - + # Model cache self.model_cache = {} self.cache_metadata = {} - + # Audit log self.audit_logger = self._setup_audit_logger() - + # Create cache directory if enable_caching: os.makedirs(self.cache_dir, exist_ok=True) def _setup_audit_logger(self) -> logging.Logger: """Set up audit logger. - + Returns: Configured audit logger """ audit_logger = logging.getLogger('secure_model_loader.audit') audit_logger.setLevel(logging.INFO) - + if self.audit_log_file: handler = logging.FileHandler(self.audit_log_file) formatter = logging.Formatter( @@ -86,12 +86,12 @@ def _setup_audit_logger(self) -> logging.Logger: ) handler.setFormatter(formatter) audit_logger.addHandler(handler) - + return audit_logger def _log_audit_event(self, event_type: str, details: Dict[str, Any]): """Log audit event. - + Args: event_type: Type of audit event details: Event details @@ -101,88 +101,88 @@ def _log_audit_event(self, event_type: str, details: Dict[str, Any]): 'event_type': event_type, 'details': details } - + self.audit_logger.info(f"AUDIT: {audit_entry}") logger.info(f"Audit event: {event_type} - {details}") def _get_cache_key(self, model_path: str, model_class: type, **kwargs) -> str: """Generate cache key for model. - + Args: model_path: Path to model file model_class: Model class **kwargs: Model parameters - + Returns: Cache key string """ import hashlib - + # Create cache key from model path, class, and parameters key_data = f"{model_path}:{model_class.__name__}:{sorted(kwargs.items())}" return hashlib.sha256(key_data.encode()).hexdigest() def _is_cached(self, cache_key: str) -> bool: """Check if model is cached. - + Args: cache_key: Cache key - + Returns: True if model is cached """ if not self.enable_caching: return False - + return cache_key in self.model_cache def _load_from_cache(self, cache_key: str) -> Optional[nn.Module]: """Load model from cache. - + Args: cache_key: Cache key - + Returns: Cached model or None """ if not self.enable_caching or cache_key not in self.model_cache: return None - + self._log_audit_event('cache_hit', {'cache_key': cache_key}) logger.info(f"Loading model from cache: {cache_key}") return self.model_cache[cache_key] def _save_to_cache(self, cache_key: str, model: nn.Module): """Save model to cache. - + Args: cache_key: Cache key model: Model to cache """ if not self.enable_caching: return - + # Check cache size current_size = sum( os.path.getsize(os.path.join(self.cache_dir, f)) for f in os.listdir(self.cache_dir) if os.path.isfile(os.path.join(self.cache_dir, f)) ) / (1024 * 1024) # Convert to MB - + if current_size > self.max_cache_size_mb: logger.warning("Cache size limit exceeded, clearing old entries") self._clear_cache() - + # Save model to cache cache_file = os.path.join(self.cache_dir, f"{cache_key}.pt") torch.save(model.state_dict(), cache_file) - + self.model_cache[cache_key] = model self.cache_metadata[cache_key] = { 'timestamp': time.time(), 'file_path': cache_file } - + self._log_audit_event('cache_save', { 'cache_key': cache_key, 'cache_file': cache_file @@ -192,16 +192,16 @@ def _clear_cache(self): """Clear model cache.""" if not self.enable_caching: return - + # Remove cache files for cache_key, metadata in self.cache_metadata.items(): if os.path.exists(metadata['file_path']): os.remove(metadata['file_path']) - + # Clear memory cache self.model_cache.clear() self.cache_metadata.clear() - + self._log_audit_event('cache_clear', {}) def load_model(self, @@ -211,14 +211,14 @@ def load_model(self, test_input: Optional[torch.Tensor] = None, **kwargs) -> Tuple[nn.Module, Dict[str, Any]]: """Load model securely. - + Args: model_path: Path to model file model_class: Model class to instantiate expected_checksum: Expected checksum for integrity verification test_input: Optional test input for performance validation **kwargs: Additional arguments for model class - + Returns: Tuple of (loaded_model, loading_info) """ @@ -233,11 +233,11 @@ def load_model(self, 'sandbox_execution': {}, 'issues': [] } - + try: # Generate cache key cache_key = self._get_cache_key(model_path, model_class, **kwargs) - + # Check cache first if self._is_cached(cache_key): model = self._load_from_cache(cache_key) @@ -250,18 +250,18 @@ def load_model(self, 'loading_time': loading_info['loading_time'] }) return model, loading_info - + # 1. Integrity check logger.info(f"Performing integrity check for {model_path}") integrity_valid, integrity_info = self.integrity_checker.comprehensive_validation( model_path, expected_checksum ) loading_info['integrity_check'] = integrity_info - + if not integrity_valid: loading_info['issues'].extend(integrity_info['findings']) raise ValueError(f"Integrity check failed: {integrity_info['findings']}") - + # 2. Model validation logger.info(f"Validating model {model_path}") # Filter out non-model-config parameters @@ -270,11 +270,11 @@ def load_model(self, model_path, model_class, model_config, test_input ) loading_info['validation'] = validation_info - + if not validation_valid: loading_info['issues'].extend(validation_info['issues']) raise ValueError(f"Model validation failed: {validation_info['issues']}") - + # 3. Load model (with or without sandbox) logger.info(f"Loading model {model_path}") if self.enable_sandbox and self.sandbox_executor: @@ -285,45 +285,45 @@ def load_model(self, else: # Load without sandbox (less secure but faster) model_data = torch.load(model_path, map_location='cpu', weights_only=True) - + # Filter kwargs to only include valid constructor parameters import inspect constructor_params = inspect.signature(model_class.__init__).parameters valid_params = {k: v for k, v in kwargs.items() if k in constructor_params} model = model_class(**valid_params) - + if 'state_dict' in model_data: model.load_state_dict(model_data['state_dict']) - + # 4. Cache model if self.enable_caching: self._save_to_cache(cache_key, model) - + # 5. Final validation model.eval() - + loading_info['loading_time'] = time.time() - start_time - + self._log_audit_event('model_loaded', { 'model_path': model_path, 'cache_used': False, 'loading_time': loading_info['loading_time'], 'model_type': type(model).__name__ }) - + logger.info(f"Model loaded successfully in {loading_info['loading_time']:.2f}s") return model, loading_info - + except Exception as e: loading_info['loading_time'] = time.time() - start_time loading_info['issues'].append(f"Loading failed: {e}") - + self._log_audit_event('model_load_failed', { 'model_path': model_path, 'error': str(e), 'loading_time': loading_info['loading_time'] }) - + logger.error(f"Failed to load model {model_path}: {e}") raise @@ -334,13 +334,13 @@ def validate_model(self, test_input: Optional[torch.Tensor] = None, **kwargs) -> Tuple[bool, Dict[str, Any]]: """Validate model without loading it. - + Args: model_path: Path to model file model_class: Model class test_input: Optional test input **kwargs: Model parameters - + Returns: Tuple of (is_valid, validation_info) """ @@ -351,58 +351,58 @@ def validate_model(self, 'overall_valid': False, 'issues': [] } - + try: # Integrity check integrity_valid, integrity_info = self.integrity_checker.comprehensive_validation( model_path, expected_checksum ) validation_info['integrity_check'] = integrity_info - + if not integrity_valid: validation_info['issues'].extend(integrity_info['findings']) - + # Model validation - filter out non-model-config parameters model_config = {k: v for k, v in kwargs.items() if k not in ['expected_checksum']} validation_valid, model_validation_info = self.model_validator.comprehensive_validation( model_path, model_class, model_config, test_input ) validation_info['validation'] = model_validation_info - + if not validation_valid: validation_info['issues'].extend(model_validation_info['issues']) - + # Overall validation result validation_info['overall_valid'] = integrity_valid and validation_valid - + self._log_audit_event('model_validated', { 'model_path': model_path, 'is_valid': validation_info['overall_valid'], 'issues': validation_info['issues'] }) - + return validation_info['overall_valid'], validation_info - + except Exception as e: validation_info['issues'].append(f"Validation error: {e}") validation_info['overall_valid'] = False - + self._log_audit_event('model_validation_failed', { 'model_path': model_path, 'error': str(e) }) - + return False, validation_info def get_cache_info(self) -> Dict[str, Any]: """Get cache information. - + Returns: Cache information dictionary """ if not self.enable_caching: return {'enabled': False} - + cache_size = 0 if os.path.exists(self.cache_dir): cache_size = sum( @@ -410,7 +410,7 @@ def get_cache_info(self) -> Dict[str, Any]: for f in os.listdir(self.cache_dir) if os.path.isfile(os.path.join(self.cache_dir, f)) ) / (1024 * 1024) # Convert to MB - + return { 'enabled': True, 'cache_dir': self.cache_dir, @@ -429,6 +429,6 @@ def cleanup(self): """Clean up resources.""" if self.sandbox_executor: self.sandbox_executor.cleanup() - + self._log_audit_event('cleanup', {}) logger.info("Secure model loader cleanup completed") \ No newline at end of file diff --git a/src/monitoring/dashboard.py b/src/monitoring/dashboard.py index 53f982a0e..a82a104e0 100644 --- a/src/monitoring/dashboard.py +++ b/src/monitoring/dashboard.py @@ -68,11 +68,11 @@ class APIMetrics: class MonitoringDashboard: """Comprehensive monitoring dashboard for SAMO Deep Learning API.""" - + def __init__(self, history_size: int = 1000): self.history_size = history_size self.start_time = time.time() - + # Metrics storage self.system_metrics_history = deque(maxlen=history_size) self.model_metrics = defaultdict(lambda: ModelMetrics( @@ -93,17 +93,17 @@ def __init__(self, history_size: int = 1000): active_connections=0, uptime_seconds=0.0 ) - + # Request tracking self.request_times = deque(maxlen=history_size) self.error_log = deque(maxlen=history_size) self.total_errors = 0 # Track total errors for accurate error rate - + # Performance tracking self.response_times = deque(maxlen=history_size) - + logger.info("Monitoring dashboard initialized") - + def update_system_metrics(self) -> SystemMetrics: """Update and store current system metrics.""" try: @@ -111,12 +111,12 @@ def update_system_metrics(self) -> SystemMetrics: cpu_percent = psutil.cpu_percent(interval=None) memory = psutil.virtual_memory() disk = psutil.disk_usage('/') - + # Network metrics network = psutil.net_io_counters() network_sent_mb = network.bytes_sent / (1024 * 1024) network_recv_mb = network.bytes_recv / (1024 * 1024) - + metrics = SystemMetrics( timestamp=time.time(), cpu_percent=cpu_percent, @@ -127,26 +127,26 @@ def update_system_metrics(self) -> SystemMetrics: network_sent_mb=network_sent_mb, network_recv_mb=network_recv_mb ) - + self.system_metrics_history.append(metrics) return metrics - + except Exception as exc: logger.error(f"Failed to update system metrics: {exc}") return None - + def record_model_request(self, model_name: str, success: bool, response_time_ms: float): """Record a model request for metrics tracking.""" metrics = self.model_metrics[model_name] metrics.model_name = model_name metrics.total_requests += 1 - + if success: metrics.successful_requests += 1 else: metrics.failed_requests += 1 metrics.error_count += 1 - + # Update average response time if metrics.average_response_time_ms == 0: metrics.average_response_time_ms = response_time_ms @@ -155,67 +155,67 @@ def record_model_request(self, model_name: str, success: bool, response_time_ms: (metrics.average_response_time_ms * (metrics.total_requests - 1) + response_time_ms) / metrics.total_requests ) - + metrics.last_used = time.time() - + def record_api_request(self, response_time_ms: float, success: bool): """Record an API request for metrics tracking.""" self.api_metrics.total_requests += 1 self.request_times.append(time.time()) self.response_times.append(response_time_ms) - + if not success: self.error_log.append({ "timestamp": time.time(), "error": "API request failed" }) self.total_errors += 1 - + # Update metrics self._update_api_metrics() - + def _update_api_metrics(self): """Update API metrics based on recent data.""" current_time = time.time() - + # Calculate requests per minute one_minute_ago = current_time - 60 recent_requests = sum(bool(t > one_minute_ago) for t in self.request_times) self.api_metrics.requests_per_minute = recent_requests - + # Calculate average response time if self.response_times: self.api_metrics.average_response_time_ms = sum(self.response_times) / len(self.response_times) - + # Calculate error rate if self.api_metrics.total_requests > 0: self.api_metrics.error_rate = self.total_errors / self.api_metrics.total_requests - + # Update uptime self.api_metrics.uptime_seconds = current_time - self.start_time - + def set_model_loaded_status(self, model_name: str, is_loaded: bool): """Set the loaded status of a model.""" if model_name in self.model_metrics: self.model_metrics[model_name].is_loaded = is_loaded - + def get_comprehensive_metrics(self) -> Dict[str, Any]: """Get comprehensive monitoring metrics.""" # Update system metrics current_system_metrics = self.update_system_metrics() - + # Update API metrics self._update_api_metrics() - + # Prepare model metrics model_metrics_dict = {model_name: asdict(metrics) for model_name, metrics in self.model_metrics.items()} - + # Calculate trends trends = self._calculate_trends() - + # Health status health_status = self._calculate_health_status() - + return { "timestamp": time.time(), "health_status": health_status, @@ -225,72 +225,72 @@ def get_comprehensive_metrics(self) -> Dict[str, Any]: "trends": trends, "alerts": self._generate_alerts() } - + def _calculate_trends(self) -> Dict[str, Any]: """Calculate performance trends.""" if len(self.system_metrics_history) < 2: return {} - + recent_metrics = list(self.system_metrics_history)[-10:] # Last 10 measurements - + cpu_trend = "stable" memory_trend = "stable" - + if len(recent_metrics) >= 2: cpu_values = [m.cpu_percent for m in recent_metrics] memory_values = [m.memory_percent for m in recent_metrics] - + # Simple trend calculation cpu_slope = (cpu_values[-1] - cpu_values[0]) / len(cpu_values) memory_slope = (memory_values[-1] - memory_values[0]) / len(memory_values) - + if cpu_slope > 5: cpu_trend = "increasing" elif cpu_slope < -5: cpu_trend = "decreasing" - + if memory_slope > 2: memory_trend = "increasing" elif memory_slope < -2: memory_trend = "decreasing" - + return { "cpu_trend": cpu_trend, "memory_trend": memory_trend, "response_time_trend": "stable" # Could be enhanced with more sophisticated analysis } - + def _calculate_health_status(self) -> str: """Calculate overall system health status.""" if not self.system_metrics_history: return "unknown" - + current_metrics = self.system_metrics_history[-1] - + # Check critical thresholds first if (current_metrics.cpu_percent > CRITICAL_CPU_THRESHOLD or current_metrics.memory_percent > CRITICAL_MEMORY_THRESHOLD or current_metrics.disk_percent > CRITICAL_DISK_THRESHOLD): return "critical" - + # Check warning thresholds if (current_metrics.cpu_percent > WARNING_CPU_THRESHOLD or current_metrics.memory_percent > WARNING_MEMORY_THRESHOLD or current_metrics.disk_percent > WARNING_DISK_THRESHOLD or self.api_metrics.error_rate > CRITICAL_ERROR_RATE_THRESHOLD): return "warning" - + return "healthy" - + def _generate_alerts(self) -> List[Dict[str, Any]]: """Generate alerts based on current metrics.""" alerts = [] - + if not self.system_metrics_history: return alerts - + current_metrics = self.system_metrics_history[-1] - + # System alerts if current_metrics.cpu_percent > CRITICAL_CPU_THRESHOLD: alerts.append({ @@ -298,21 +298,21 @@ def _generate_alerts(self) -> List[Dict[str, Any]]: "message": f"High CPU usage: {current_metrics.cpu_percent:.1f}%", "timestamp": current_metrics.timestamp }) - + if current_metrics.memory_percent > CRITICAL_MEMORY_THRESHOLD: alerts.append({ "level": "critical", "message": f"High memory usage: {current_metrics.memory_percent:.1f}%", "timestamp": current_metrics.timestamp }) - + if current_metrics.disk_percent > CRITICAL_DISK_THRESHOLD: alerts.append({ "level": "critical", "message": f"Low disk space: {100 - current_metrics.disk_percent:.1f}% free", "timestamp": current_metrics.timestamp }) - + # API alerts if self.api_metrics.error_rate > CRITICAL_ERROR_RATE_THRESHOLD: alerts.append({ @@ -320,7 +320,7 @@ def _generate_alerts(self) -> List[Dict[str, Any]]: "message": f"High error rate: {self.api_metrics.error_rate:.1%}", "timestamp": time.time() }) - + # Model alerts for model_name, metrics in self.model_metrics.items(): if metrics.error_count > MODEL_ERROR_COUNT_THRESHOLD: @@ -329,31 +329,31 @@ def _generate_alerts(self) -> List[Dict[str, Any]]: "message": f"High error count for {model_name}: {metrics.error_count} errors", "timestamp": time.time() }) - + return alerts - + def get_historical_data(self, hours: int = 24) -> Dict[str, Any]: """Get historical data for the specified time period.""" cutoff_time = time.time() - (hours * 3600) - + # Filter system metrics historical_system = [ asdict(metrics) for metrics in self.system_metrics_history if metrics.timestamp > cutoff_time ] - + # Filter response times historical_response_times = [ rt for rt in self.response_times if rt > cutoff_time ] - + return { "system_metrics": historical_system, "response_times": historical_response_times, "period_hours": hours } - + def reset_metrics(self): """Reset all metrics (useful for testing).""" self.system_metrics_history.clear() @@ -362,7 +362,7 @@ def reset_metrics(self): self.error_log.clear() self.response_times.clear() self.start_time = time.time() - + logger.info("Monitoring metrics reset") # Global dashboard instance diff --git a/src/security/jwt_manager.py b/src/security/jwt_manager.py index 641b7bc98..c90f3ce81 100644 --- a/src/security/jwt_manager.py +++ b/src/security/jwt_manager.py @@ -47,12 +47,12 @@ class TokenResponse(BaseModel): class JWTManager: """Comprehensive JWT token management system""" - + def __init__(self, secret_key: str = SECRET_KEY, algorithm: str = ALGORITHM): self.secret_key = secret_key self.algorithm = algorithm self.blacklisted_tokens: dict = {} # Changed to dict: {token: exp_datetime} - + def create_access_token(self, user_data: Dict[str, Any]) -> str: """Create a new access token""" payload = { @@ -64,7 +64,7 @@ def create_access_token(self, user_data: Dict[str, Any]) -> str: "iat": datetime.utcnow() } return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) - + def create_refresh_token(self, user_data: Dict[str, Any]) -> str: """Create a new refresh token""" payload = { @@ -77,24 +77,24 @@ def create_refresh_token(self, user_data: Dict[str, Any]) -> str: "type": "refresh" } return jwt.encode(payload, self.secret_key, algorithm=self.algorithm) - + def create_token_pair(self, user_data: Dict[str, Any]) -> TokenResponse: """Create both access and refresh tokens""" access_token = self.create_access_token(user_data) refresh_token = self.create_refresh_token(user_data) - + return TokenResponse( access_token=access_token, refresh_token=refresh_token, expires_in=ACCESS_TOKEN_EXPIRE_MINUTES * 60 ) - + def verify_token(self, token: str) -> Optional[TokenPayload]: """Verify and decode a token""" try: if token in self.blacklisted_tokens: return None - + payload = jwt.decode(token, self.secret_key, algorithms=[self.algorithm]) return TokenPayload(**payload) except jwt.ExpiredSignatureError: @@ -106,13 +106,13 @@ def verify_token(self, token: str) -> Optional[TokenPayload]: except Exception as e: logger.error(f"Token verification error: {str(e)}") return None - + def refresh_access_token(self, refresh_token: str) -> Optional[str]: """Refresh an access token using a valid refresh token""" payload = self.verify_token(refresh_token) if not payload or getattr(payload, "type", None) != "refresh": return None - + user_data = { "user_id": payload.user_id, "username": payload.username, @@ -120,7 +120,7 @@ def refresh_access_token(self, refresh_token: str) -> Optional[str]: "permissions": payload.permissions } return self.create_access_token(user_data) - + def blacklist_token(self, token: str) -> bool: """Add a token to the blacklist""" try: @@ -130,32 +130,32 @@ def blacklist_token(self, token: str) -> bool: return True except jwt.InvalidTokenError: return False - + def is_token_blacklisted(self, token: str) -> bool: """Check if a token is blacklisted""" return token in self.blacklisted_tokens - + def get_user_permissions(self, token: str) -> List[str]: """Extract user permissions from token""" payload = self.verify_token(token) return payload.permissions if payload else [] - + def has_permission(self, token: str, required_permission: str) -> bool: """Check if user has a specific permission""" permissions = self.get_user_permissions(token) return required_permission in permissions - + def cleanup_expired_tokens(self) -> int: """Clean up expired tokens from blacklist""" initial_count = len(self.blacklisted_tokens) current_time = datetime.utcnow() - + tokens_to_remove = set() # self.blacklisted_tokens is now a dict: {token: exp_datetime} for token, exp_datetime in self.blacklisted_tokens.items(): if exp_datetime and exp_datetime < current_time: tokens_to_remove.add(token) - + for token in tokens_to_remove: self.blacklisted_tokens.pop(token, None) return initial_count - len(self.blacklisted_tokens) diff --git a/src/security_headers.py b/src/security_headers.py index f20667e2d..5e6fd5946 100644 --- a/src/security_headers.py +++ b/src/security_headers.py @@ -43,7 +43,7 @@ class SecurityHeadersConfig: class SecurityHeadersMiddleware: """ Flask middleware for adding security headers and implementing security policies. - + Features: - Content Security Policy (CSP) - HTTP Strict Transport Security (HSTS) @@ -56,7 +56,7 @@ class SecurityHeadersMiddleware: - Request correlation - Security monitoring """ - + def __init__(self, app: Flask, config: SecurityHeadersConfig): self.app = app self.config = config @@ -68,14 +68,14 @@ def __init__(self, app: Flask, config: SecurityHeadersConfig): self.csp_policy = security_config.get('security_headers', {}).get('headers', {}).get('Content-Security-Policy') except Exception as e: logger.warning(f"Could not load CSP from config: {e}") - + # Register middleware app.before_request(self._before_request) app.after_request(self._after_request) - + # Generate nonce for CSP self._csp_nonce = secrets.token_hex(16) - + def _before_request(self): """Process request before handling.""" # Generate request ID for correlation @@ -83,75 +83,75 @@ def _before_request(self): g.request_id = hashlib.sha256( f"{time.time()}:{request.remote_addr}:{secrets.token_hex(8)}".encode() ).hexdigest() - + # Generate correlation ID if self.config.enable_correlation_id: g.correlation_id = request.headers.get('X-Correlation-ID', g.request_id) - + # Log security-relevant request information self._log_security_info() - + def _after_request(self, response: Response) -> Response: """Process response after handling.""" # Add security headers self._add_security_headers(response) - + # Add request correlation headers self._add_correlation_headers(response) - + # Log security-relevant response information self._log_response_security(response) - + return response - + def _add_security_headers(self, response: Response): """Add security headers to response.""" # Content Security Policy if self.config.enable_content_security_policy: csp_policy = self._build_csp_policy() response.headers['Content-Security-Policy'] = csp_policy - + # HTTP Strict Transport Security if self.config.enable_strict_transport_security: response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains; preload' - + # X-Frame-Options if self.config.enable_x_frame_options: response.headers['X-Frame-Options'] = 'DENY' - + # X-Content-Type-Options if self.config.enable_x_content_type_options: response.headers['X-Content-Type-Options'] = 'nosniff' - + # X-XSS-Protection if self.config.enable_x_xss_protection: response.headers['X-XSS-Protection'] = '1; mode=block' - + # Referrer Policy if self.config.enable_referrer_policy: response.headers['Referrer-Policy'] = 'strict-origin-when-cross-origin' - + # Permissions Policy if self.config.enable_permissions_policy: permissions_policy = self._build_permissions_policy() response.headers['Permissions-Policy'] = permissions_policy - + # Cross-Origin Embedder Policy if self.config.enable_cross_origin_embedder_policy: response.headers['Cross-Origin-Embedder-Policy'] = 'require-corp' - + # Cross-Origin Opener Policy if self.config.enable_cross_origin_opener_policy: response.headers['Cross-Origin-Opener-Policy'] = 'same-origin' - + # Cross-Origin Resource Policy if self.config.enable_cross_origin_resource_policy: response.headers['Cross-Origin-Resource-Policy'] = 'same-origin' - + # Origin-Agent-Cluster if self.config.enable_origin_agent_cluster: response.headers['Origin-Agent-Cluster'] = '?1' - + def _build_csp_policy(self) -> str: """Return CSP policy from config, or a secure default if not set.""" if self.csp_policy: @@ -165,7 +165,7 @@ def _build_csp_policy(self) -> str: "base-uri 'self'; " "form-action 'self'" ) - + def _build_permissions_policy(self) -> str: """Build Permissions Policy.""" policies = [ @@ -198,15 +198,15 @@ def _build_permissions_policy(self) -> str: "xr-spatial-tracking=()" ] return ", ".join(policies) - + def _add_correlation_headers(self, response: Response): """Add request correlation headers.""" if hasattr(g, 'request_id'): response.headers['X-Request-ID'] = g.request_id - + if hasattr(g, 'correlation_id'): response.headers['X-Correlation-ID'] = g.correlation_id - + def _log_security_info(self): """Log security-relevant request information.""" security_info = { @@ -224,24 +224,24 @@ def _log_security_info(self): 'x_forwarded_for': request.headers.get('X-Forwarded-For', ''), 'x_real_ip': request.headers.get('X-Real-IP', ''), } - + # Log suspicious patterns suspicious_patterns = self._detect_suspicious_patterns() if suspicious_patterns: security_info['suspicious_patterns'] = suspicious_patterns logger.warning(f"Security warning: {suspicious_patterns}") - + logger.info(f"Security audit: {security_info}") - + def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: """Enhanced user agent analysis with scoring and detailed categorization.""" if not user_agent: return {"score": 0, "category": "empty", "patterns": [], "risk_level": "low"} - + score = 0 patterns = [] ua_lower = user_agent.lower() - + # Legitimate bot whitelist (negative scoring) legitimate_bots = [ 'googlebot', 'bingbot', 'slurp', 'duckduckbot', 'facebookexternalhit', @@ -249,66 +249,66 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: 'slackbot', 'github-camo', 'github-actions', 'vercel', 'netlify', 'uptimerobot', 'pingdom', 'statuscake', 'monitor', 'healthcheck' ] - + # High-risk patterns (score +3 each) high_risk_patterns = [ 'sqlmap', 'nikto', 'nmap', 'scanner', 'grabber', 'harvester', 'exploit', 'vulnerability', 'penetration', 'security', 'audit' ] - + # Medium-risk patterns (score +2 each) medium_risk_patterns = [ 'headless', 'phantom', 'selenium', 'webdriver', 'automated', 'testing', 'script', 'python-requests', 'curl', 'wget', 'httrack', 'scraper', 'crawler', 'spider', 'bot' ] - + # Low-risk patterns (score +1 each) low_risk_patterns = [ 'indexer', 'feed', 'rss', 'aggregator', 'monitor', 'checker', 'validator', 'linter', 'checker', 'analyzer' ] - + # Check legitimate bots first (negative scoring) for bot in legitimate_bots: if bot in ua_lower: score -= 2 patterns.append(f"legitimate_bot:{bot}") logger.debug(f"Legitimate bot detected: {bot}") - + # Check high-risk patterns for pattern in high_risk_patterns: if pattern in ua_lower: score += 3 patterns.append(f"high_risk:{pattern}") logger.debug(f"High-risk UA pattern detected: {pattern}") - + # Check medium-risk patterns for pattern in medium_risk_patterns: if pattern in ua_lower: score += 2 patterns.append(f"medium_risk:{pattern}") logger.debug(f"Medium-risk UA pattern detected: {pattern}") - + # Check low-risk patterns for pattern in low_risk_patterns: if pattern in ua_lower: score += 1 patterns.append(f"low_risk:{pattern}") logger.debug(f"Low-risk UA pattern detected: {pattern}") - + # Bonus for suspicious combinations if any(pattern in ua_lower for pattern in ['bot', 'crawler', 'spider']) and any(pattern in ua_lower for pattern in ['python', 'curl', 'wget', 'script']): score += 2 patterns.append("suspicious_combination") logger.debug("Suspicious UA combination detected") - + # Check for missing or generic user agents if user_agent in ['', 'null', 'undefined', 'unknown', 'anonymous']: score += 2 patterns.append("missing_generic_ua") logger.debug("Missing or generic user agent detected") - + # Determine category and risk level if score <= -1: category = "legitimate_bot" @@ -325,7 +325,7 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: else: category = "malicious" risk_level = "very_high" - + return { "score": max(0, score), # Don't return negative scores "category": category, @@ -333,11 +333,11 @@ def _analyze_user_agent_enhanced(self, user_agent: str) -> dict: "risk_level": risk_level, "user_agent": user_agent[:100] # Truncate for logging } - + def _detect_suspicious_patterns(self) -> List[str]: """Enhanced suspicious pattern detection with user agent analysis.""" patterns = [] - + # Check for suspicious headers suspicious_headers = [ 'X-Forwarded-Host', @@ -345,38 +345,38 @@ def _detect_suspicious_patterns(self) -> List[str]: 'X-Rewrite-URL', 'X-Custom-IP-Authorization' ] - + for header in suspicious_headers: if header in request.headers: patterns.append(f"Suspicious header: {header}") - + # Check for suspicious query parameters suspicious_params = [ 'cmd', 'exec', 'system', 'eval', 'script', 'union', 'select', 'insert', 'update', 'delete' ] - + for param in suspicious_params: if param in request.args: patterns.append(f"Suspicious query param: {param}") - + # Enhanced user agent analysis if self.config.enable_enhanced_ua_analysis: user_agent = request.headers.get('User-Agent', '') ua_analysis = self._analyze_user_agent_enhanced(user_agent) - + if ua_analysis["score"] >= self.config.ua_suspicious_score_threshold: patterns.append(f"Suspicious user agent: {ua_analysis['category']} (score: {ua_analysis['score']})") - + # Log detailed analysis logger.warning(f"User agent analysis: {ua_analysis}") - + # Optionally block based on configuration if self.config.ua_blocking_enabled and ua_analysis["risk_level"] in ["high", "very_high"]: patterns.append("BLOCKED: High-risk user agent") - + return patterns - + def _log_response_security(self, response: Response): """Log security-relevant response information.""" security_info = { @@ -396,9 +396,9 @@ def _log_response_security(self, response: Response): 'permissions_policy': response.headers.get('Permissions-Policy', ''), } } - + logger.info(f"Response security: {security_info}") - + def get_security_stats(self) -> Dict: """Get security headers statistics.""" return { diff --git a/src/unified_ai_api.py b/src/unified_ai_api.py index 3b8d57e2a..a50b2b1f8 100644 --- a/src/unified_ai_api.py +++ b/src/unified_ai_api.py @@ -49,24 +49,24 @@ # Enhanced WebSocket Connection Management class WebSocketConnectionManager: """Enhanced WebSocket connection manager with pooling and heartbeat.""" - + def __init__(self): self.active_connections: Dict[str, Set[WebSocket]] = defaultdict(set) self.connection_metadata: Dict[WebSocket, Dict[str, Any]] = {} self.heartbeat_interval = 30 # seconds self.max_connections_per_user = 5 self.connection_timeout = 300 # 5 minutes - + async def connect(self, websocket: WebSocket, user_id: str, token: str): """Connect a new WebSocket with enhanced management.""" # Check connection limits if len(self.active_connections[user_id]) >= self.max_connections_per_user: await websocket.close(code=4008, reason="Maximum connections reached") return False - + await websocket.accept() self.active_connections[user_id].add(websocket) - + # Store connection metadata self.connection_metadata[websocket] = { "user_id": user_id, @@ -76,24 +76,24 @@ async def connect(self, websocket: WebSocket, user_id: str, token: str): "message_count": 0, "bytes_processed": 0 } - + logger.info(f"WebSocket connected for user {user_id}. Total connections: {len(self.active_connections[user_id])}") return True - + async def disconnect(self, websocket: WebSocket): """Disconnect WebSocket and cleanup.""" user_id = None if websocket in self.connection_metadata: user_id = self.connection_metadata[websocket]["user_id"] del self.connection_metadata[websocket] - + if user_id and websocket in self.active_connections[user_id]: self.active_connections[user_id].remove(websocket) if not self.active_connections[user_id]: del self.active_connections[user_id] - + logger.info(f"WebSocket disconnected for user {user_id}") - + async def send_personal_message(self, message: Dict[str, Any], websocket: WebSocket): """Send message to specific WebSocket with error handling.""" try: @@ -103,7 +103,7 @@ async def send_personal_message(self, message: Dict[str, Any], websocket: WebSoc except Exception as e: logger.error(f"Failed to send message to WebSocket: {e}") await self.disconnect(websocket) - + async def broadcast_to_user(self, message: Dict[str, Any], user_id: str): """Broadcast message to all connections of a specific user.""" disconnected = set() @@ -115,34 +115,34 @@ async def broadcast_to_user(self, message: Dict[str, Any], user_id: str): except Exception as e: logger.error(f"Failed to broadcast to WebSocket: {e}") disconnected.add(websocket) - + # Cleanup disconnected connections for websocket in disconnected: await self.disconnect(websocket) - + async def update_heartbeat(self, websocket: WebSocket): """Update heartbeat timestamp for connection.""" if websocket in self.connection_metadata: self.connection_metadata[websocket]["last_heartbeat"] = time.time() - + async def cleanup_stale_connections(self): """Cleanup stale connections based on timeout.""" current_time = time.time() stale_connections = [] - + for websocket, metadata in self.connection_metadata.items(): if current_time - metadata["last_heartbeat"] > self.connection_timeout: stale_connections.append(websocket) - + for websocket in stale_connections: logger.warning(f"Cleaning up stale WebSocket connection for user {self.connection_metadata[websocket]['user_id']}") await self.disconnect(websocket) - + def get_connection_stats(self) -> Dict[str, Any]: """Get connection statistics.""" total_connections = sum(len(connections) for connections in self.active_connections.values()) total_users = len(self.active_connections) - + return { "total_connections": total_connections, "total_users": total_users, @@ -446,10 +446,10 @@ async def register_user(user_data: UserRegister) -> TokenResponse: # 2. Hash the password # 3. Store user in database # 4. Generate user ID - + # For demo purposes, we'll create a simple user user_id = f"user_{int(time.time())}" - + # Create user data for token token_user_data = { "user_id": user_id, @@ -457,13 +457,13 @@ async def register_user(user_data: UserRegister) -> TokenResponse: "email": user_data.email, "permissions": ["read", "write"] # Default permissions } - + # Generate tokens token_response = jwt_manager.create_token_pair(token_user_data) - + logger.info(f"New user registered: {user_data.username}") return token_response - + except Exception as exc: logger.error(f"Registration failed: {exc}") raise HTTPException( @@ -485,14 +485,14 @@ async def login_user(login_data: UserLogin) -> TokenResponse: # 1. Verify username/password against database # 2. Check if account is active # 3. Retrieve user permissions - + # For demo purposes, we'll accept any valid email/password if not login_data.username or not login_data.password: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Username and password required" ) - + # Create user data for token user_id = f"user_{hash(login_data.username) % 10000}" token_user_data = { @@ -501,13 +501,13 @@ async def login_user(login_data: UserLogin) -> TokenResponse: "email": login_data.username if "@" in login_data.username else f"{login_data.username}@example.com", "permissions": ["read", "write", "admin"] # Demo permissions } - + # Generate tokens token_response = jwt_manager.create_token_pair(token_user_data) - + logger.info(f"User logged in: {login_data.username}") return token_response - + except HTTPException: raise except Exception as exc: @@ -538,7 +538,7 @@ async def refresh_token(request: RefreshTokenRequest) -> TokenResponse: status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid refresh token" ) - + # Create new user data user_data = { "user_id": payload.user_id, @@ -546,13 +546,13 @@ async def refresh_token(request: RefreshTokenRequest) -> TokenResponse: "email": payload.email, "permissions": payload.permissions } - + # Generate new token pair token_response = jwt_manager.create_token_pair(user_data) - + logger.info(f"Token refreshed for user: {payload.username}") return token_response - + except HTTPException: raise except Exception as exc: @@ -583,9 +583,9 @@ async def logout_user( logger.info(f"User logged out and token blacklisted: {current_user.username}") else: logger.warning("No valid Authorization header found during logout") - + return {"message": "Successfully logged out"} - + except Exception as exc: logger.error(f"Logout failed: {exc}") raise HTTPException( @@ -814,30 +814,30 @@ async def transcribe_voice( ) -> VoiceTranscription: """Enhanced voice transcription with detailed analysis.""" start_time = time.time() - + try: # Validate file if not audio_file.filename: raise HTTPException(status_code=400, detail="Audio file required") - + # Check file size (max 50MB) content = await audio_file.read() if len(content) > 50 * 1024 * 1024: raise HTTPException(status_code=400, detail="File too large (max 50MB)") # Reset file position for later processing await audio_file.seek(0) - + # Save uploaded file temporarily with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file: temp_file.write(content) temp_file.flush() # Ensure data is written to disk temp_file_path = temp_file.name - + try: # Transcribe audio if voice_transcriber is None: raise HTTPException(status_code=503, detail="Voice transcription service unavailable") - + # Enhanced transcription with additional parameters transcription_result = voice_transcriber.transcribe( temp_file_path, @@ -845,12 +845,12 @@ async def transcribe_voice( model_size=model_size, timestamp=timestamp ) - + # Calculate additional metrics duration = transcription_result.get("duration", 0) word_count = len(transcription_result.get("text", "").split()) speaking_rate = (word_count / duration * 60) if duration > 0 else 0 - + # Audio quality assessment audio_quality = "excellent" if duration < 1: @@ -859,9 +859,9 @@ async def transcribe_voice( audio_quality = "fair" elif duration < 15: audio_quality = "good" - + processing_time = (time.time() - start_time) * 1000 - + return VoiceTranscription( text=transcription_result.get("text", ""), language=transcription_result.get("language", "unknown"), @@ -871,12 +871,12 @@ async def transcribe_voice( speaking_rate=speaking_rate, audio_quality=audio_quality ) - + finally: # Cleanup temporary file if os.path.exists(temp_file_path): os.unlink(temp_file_path) - + except HTTPException: raise except Exception as exc: @@ -900,7 +900,7 @@ async def batch_transcribe_voice( """Batch process multiple audio files for transcription.""" start_time = time.time() results = [] - + try: for i, audio_file in enumerate(audio_files): try: @@ -910,13 +910,13 @@ async def batch_transcribe_voice( temp_file.write(content) temp_file.flush() # Ensure data is written to disk temp_file_path = temp_file.name - + try: if voice_transcriber is None: raise HTTPException(status_code=503, detail="Voice transcription service unavailable") - + transcription_result = voice_transcriber.transcribe(temp_file_path, language=language) - + results.append({ "file_index": i, "filename": audio_file.filename, @@ -926,11 +926,11 @@ async def batch_transcribe_voice( "confidence": transcription_result.get("confidence", 0.0), "duration": transcription_result.get("duration", 0) }) - + finally: if os.path.exists(temp_file_path): os.unlink(temp_file_path) - + except Exception as exc: results.append({ "file_index": i, @@ -938,9 +938,9 @@ async def batch_transcribe_voice( "success": False, "error": str(exc) }) - + processing_time = (time.time() - start_time) * 1000 - + return { "total_files": len(audio_files), "successful_transcriptions": len([r for r in results if r["success"]]), @@ -948,7 +948,7 @@ async def batch_transcribe_voice( "processing_time_ms": processing_time, "results": results } - + except Exception as exc: logger.error(f"Batch transcription failed: {exc}") raise HTTPException( @@ -974,14 +974,14 @@ async def summarize_text( ) -> TextSummary: """Enhanced text summarization with multiple model options.""" start_time = time.time() - + try: if not text.strip(): raise HTTPException(status_code=400, detail="Text cannot be empty") - + if text_summarizer is None: raise HTTPException(status_code=503, detail="Text summarization service unavailable") - + # Enhanced summarization with parameters summary_result = text_summarizer.summarize( text, @@ -990,12 +990,12 @@ async def summarize_text( min_length=min_length, do_sample=do_sample ) - + # Calculate metrics original_length = len(text.split()) summary_length = len(summary_result.get("summary", "").split()) compression_ratio = 1 - (summary_length / original_length) if original_length > 0 else 0 - + # Determine emotional tone from summary emotional_tone = "neutral" if emotion_detector and summary_result.get("summary"): @@ -1008,16 +1008,16 @@ async def summarize_text( emotional_tone = "negative" except Exception as exc: logger.warning(f"Could not determine emotional tone from summary: {exc}") - + processing_time = (time.time() - start_time) * 1000 - + return TextSummary( summary=summary_result.get("summary", ""), key_emotions=summary_result.get("key_emotions", []), compression_ratio=compression_ratio, emotional_tone=emotional_tone ) - + except HTTPException: raise except Exception as exc: @@ -1035,25 +1035,25 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query if not token: await websocket.close(code=4001, reason="Authentication token required") return - + try: # Verify JWT token using the global jwt_manager instance payload = jwt_manager.verify_token(token) if not payload: await websocket.close(code=4001, reason="Invalid authentication token") return - + # Check if user has real-time processing permission if "realtime_processing" not in payload.permissions: await websocket.close(code=4003, reason="Insufficient permissions") return - + except Exception as e: await websocket.close(code=4001, reason=f"Authentication failed: {str(e)}") return - + await websocket.accept() - + # Authenticate WebSocket connection try: # Get token from query parameters or initial message @@ -1071,7 +1071,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query }) await websocket.close() return - + # Verify token using the global jwt_manager instance payload = jwt_manager.verify_token(token) if not payload: @@ -1081,9 +1081,9 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query }) await websocket.close() return - + logger.info(f"WebSocket authenticated for user: {payload.username}") - + except Exception as exc: await websocket.send_json({ "type": "error", @@ -1091,7 +1091,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query }) await websocket.close() return - + try: while True: # Receive audio data or control messages @@ -1099,7 +1099,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query data = await websocket.receive_bytes() except WebSocketDisconnect: break - + # Process audio in real-time if voice_transcriber: try: @@ -1108,11 +1108,11 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query temp_file.write(data) temp_file.flush() # Ensure data is written to disk temp_file_path = temp_file.name - + try: # Transcribe result = voice_transcriber.transcribe(temp_file_path) - + # Send result back await websocket.send_json({ "type": "transcription", @@ -1120,11 +1120,11 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query "confidence": result.get("confidence", 0.0), "language": result.get("language", "unknown") }) - + finally: if os.path.exists(temp_file_path): os.unlink(temp_file_path) - + except Exception as exc: await websocket.send_json({ "type": "error", @@ -1135,7 +1135,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query "type": "error", "message": "Voice transcription service unavailable" }) - + except WebSocketDisconnect: logger.info("WebSocket client disconnected") except Exception as exc: @@ -1162,11 +1162,11 @@ async def get_performance_metrics( try: # Get system metrics import psutil - + cpu_percent = await asyncio.to_thread(psutil.cpu_percent, interval=1) memory = await asyncio.to_thread(psutil.virtual_memory) disk = await asyncio.to_thread(psutil.disk_usage, '/') - + # Model performance metrics model_metrics = { "emotion_detection": { @@ -1185,7 +1185,7 @@ async def get_performance_metrics( "total_requests": 0 } } - + return { "timestamp": time.time(), "system": { @@ -1202,7 +1202,7 @@ async def get_performance_metrics( "total_requests": 0 # In real app, track from database } } - + except Exception as exc: logger.error(f"Failed to get performance metrics: {exc}") raise HTTPException( @@ -1222,10 +1222,10 @@ async def detailed_health_check( """Comprehensive health check with detailed diagnostics.""" health_status = "healthy" issues = [] - + # Check models model_checks = {} - + if emotion_detector is None: health_status = "degraded" issues.append("Emotion detection model not loaded") @@ -1239,7 +1239,7 @@ async def detailed_health_check( health_status = "degraded" issues.append(f"Emotion detection model error: {exc}") model_checks["emotion_detection"] = {"status": "error", "error": str(exc)} - + if text_summarizer is None: health_status = "degraded" issues.append("Text summarization model not loaded") @@ -1253,28 +1253,28 @@ async def detailed_health_check( health_status = "degraded" issues.append(f"Text summarization model error: {exc}") model_checks["text_summarization"] = {"status": "error", "error": str(exc)} - + if voice_transcriber is None: health_status = "degraded" issues.append("Voice processing model not loaded") model_checks["voice_processing"] = {"status": "unavailable", "error": "Model not loaded"} else: model_checks["voice_processing"] = {"status": "healthy", "test_passed": True} - + # Check system resources try: import psutil cpu_percent = await asyncio.to_thread(psutil.cpu_percent, interval=1) memory = await asyncio.to_thread(psutil.virtual_memory) - + if cpu_percent > 90: health_status = "degraded" issues.append(f"High CPU usage: {cpu_percent}%") - + if memory.percent > 90: health_status = "degraded" issues.append(f"High memory usage: {memory.percent}%") - + system_checks = { "cpu_percent": cpu_percent, "memory_percent": memory.percent, @@ -1284,7 +1284,7 @@ async def detailed_health_check( system_checks = {"status": "error", "error": str(exc)} health_status = "degraded" issues.append(f"System check failed: {exc}") - + return { "status": health_status, "timestamp": time.time(), diff --git a/tests/conftest.py b/tests/conftest.py index 34621f56b..b14a8a990 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -103,11 +103,11 @@ def cpu_device(): def api_client(): """Provide FastAPI test client.""" client = TestClient(app) - + # Reset rate limiter state before each test if hasattr(app.state, 'rate_limiter'): app.state.rate_limiter.reset_state() - + return client From a6628a730c26b0553865f08ee23be4efd86443d9 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 8 Aug 2025 23:30:08 +0000 Subject: [PATCH 2/2] refactor: remove unnecessary whitespace Resolved issues in the following files with DeepSource Autofix: 1. scripts/legacy/comprehensive_model_validation.py 2. scripts/legacy/deep_model_analysis.py --- .../legacy/comprehensive_model_validation.py | 20 ++++++++--------- scripts/legacy/deep_model_analysis.py | 22 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/legacy/comprehensive_model_validation.py b/scripts/legacy/comprehensive_model_validation.py index c25d8dead..5c9f597af 100644 --- a/scripts/legacy/comprehensive_model_validation.py +++ b/scripts/legacy/comprehensive_model_validation.py @@ -85,7 +85,7 @@ def comprehensive_validation(): return False # Test 1: Basic Functionality - print(f"\n๐Ÿงช TEST 1: BASIC FUNCTIONALITY") + print("\n๐Ÿงช TEST 1: BASIC FUNCTIONALITY") print("-" * 40) test_cases = [ @@ -135,7 +135,7 @@ def comprehensive_validation(): return False accuracy = correct_predictions / total_predictions - print(f"\n๐Ÿ“Š Basic Functionality Results:") + print("\n๐Ÿ“Š Basic Functionality Results:") print(f" Correct: {correct_predictions}/{total_predictions}") print(f" Accuracy: {accuracy:.1%}") @@ -144,7 +144,7 @@ def comprehensive_validation(): return False # Test 2: Confidence Distribution - print(f"\n๐Ÿงช TEST 2: CONFIDENCE DISTRIBUTION") + print("\n๐Ÿงช TEST 2: CONFIDENCE DISTRIBUTION") print("-" * 40) confidence_scores = [] @@ -170,7 +170,7 @@ def comprehensive_validation(): print(f"โš ๏ธ WARNING: Low average confidence ({avg_confidence:.3f})") # Test 3: Edge Cases - print(f"\n๐Ÿงช TEST 3: EDGE CASES") + print("\n๐Ÿงช TEST 3: EDGE CASES") print("-" * 40) edge_cases = [ @@ -206,7 +206,7 @@ def comprehensive_validation(): print(f"\n๐Ÿ“Š Edge Case Results: {edge_case_success}/{len(edge_cases)} successful") # Test 4: Performance Benchmark - print(f"\n๐Ÿงช TEST 4: PERFORMANCE BENCHMARK") + print("\n๐Ÿงช TEST 4: PERFORMANCE BENCHMARK") print("-" * 40) benchmark_text = "I'm feeling really happy today!" @@ -233,7 +233,7 @@ def comprehensive_validation(): print(f"โš ๏ธ WARNING: Slow inference time ({avg_time:.4f}s)") # Test 5: Consistency Check - print(f"\n๐Ÿงช TEST 5: CONSISTENCY CHECK") + print("\n๐Ÿงช TEST 5: CONSISTENCY CHECK") print("-" * 40) consistency_text = "I'm feeling happy today!" @@ -263,7 +263,7 @@ def comprehensive_validation(): return False # Final Validation Summary - print(f"\n๐ŸŽฏ FINAL VALIDATION SUMMARY") + print("\n๐ŸŽฏ FINAL VALIDATION SUMMARY") print("=" * 60) validation_results = { @@ -284,10 +284,10 @@ def comprehensive_validation(): print(f"\n{'๐ŸŽ‰ ALL TESTS PASSED!' if all_passed else 'โŒ SOME TESTS FAILED'}") if all_passed: - print(f"โœ… Your 99.54% F1 score model is 100% RELIABLE!") - print(f"๐Ÿš€ Ready for production deployment!") + print("โœ… Your 99.54% F1 score model is 100% RELIABLE!") + print("๐Ÿš€ Ready for production deployment!") else: - print(f"โš ๏ธ Model needs further validation before deployment") + print("โš ๏ธ Model needs further validation before deployment") return all_passed diff --git a/scripts/legacy/deep_model_analysis.py b/scripts/legacy/deep_model_analysis.py index db1f0e950..054c162ef 100644 --- a/scripts/legacy/deep_model_analysis.py +++ b/scripts/legacy/deep_model_analysis.py @@ -28,14 +28,14 @@ def deep_model_analysis(): # Define emotion mapping emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - print(f"\n๐Ÿ“Š EMOTION MAPPING ANALYSIS") + print("\n๐Ÿ“Š EMOTION MAPPING ANALYSIS") print("-" * 40) print("Current mapping (LABEL_0 to LABEL_11):") for i, emotion in enumerate(emotion_mapping): print(f" LABEL_{i} โ†’ {emotion}") # Test with different variations - print(f"\n๐Ÿงช DETAILED PREDICTION ANALYSIS") + print("\n๐Ÿงช DETAILED PREDICTION ANALYSIS") print("-" * 40) test_cases = [ @@ -62,7 +62,7 @@ def deep_model_analysis(): # Get top 3 predictions top_probs, top_indices = torch.topk(probabilities[0], 3) - print(f"๐Ÿ” Top 3 predictions:") + print("๐Ÿ” Top 3 predictions:") for i, (prob, idx) in enumerate(zip(top_probs, top_indices)): emotion = emotion_mapping[idx.item()] print(f" {i+1}. {emotion}: {prob.item():.3f}") @@ -73,7 +73,7 @@ def deep_model_analysis(): print(f"๐Ÿ“Š Expected emotion '{expected_emotion}' probability: {expected_prob:.3f}") # Analyze model confidence patterns - print(f"\n๐Ÿ“ˆ CONFIDENCE PATTERN ANALYSIS") + print("\n๐Ÿ“ˆ CONFIDENCE PATTERN ANALYSIS") print("-" * 40) confidence_by_emotion = {emotion: [] for emotion in emotion_mapping} @@ -99,7 +99,7 @@ def deep_model_analysis(): print(f"'{word}' โ†’ {predicted_emotion} (confidence: {confidence:.3f})") # Check for bias towards certain emotions - print(f"\n๐ŸŽฏ EMOTION BIAS ANALYSIS") + print("\n๐ŸŽฏ EMOTION BIAS ANALYSIS") print("-" * 40) emotion_counts = {} @@ -118,7 +118,7 @@ def deep_model_analysis(): print(f"โŒ WARNING: Model shows bias towards '{most_common[0]}'") # Test with training-like data - print(f"\n๐ŸŽ“ TRAINING-LIKE DATA TEST") + print("\n๐ŸŽ“ TRAINING-LIKE DATA TEST") print("-" * 40) # These should be more similar to what the model was trained on @@ -171,17 +171,17 @@ def deep_model_analysis(): print(f"\n๐Ÿ“Š Training-like accuracy: {training_like_accuracy:.1%}") # Final analysis - print(f"\n๐Ÿ” ANALYSIS SUMMARY") + print("\n๐Ÿ” ANALYSIS SUMMARY") print("=" * 50) if training_like_accuracy > 0.8: print(f"โœ… Model performs well on training-like data ({training_like_accuracy:.1%})") - print(f"โš ๏ธ Issue: Model may be overfitting to specific training patterns") - print(f"๐Ÿ’ก Solution: Model needs more diverse training data or regularization") + print("โš ๏ธ Issue: Model may be overfitting to specific training patterns") + print("๐Ÿ’ก Solution: Model needs more diverse training data or regularization") else: print(f"โŒ Model performs poorly even on training-like data ({training_like_accuracy:.1%})") - print(f"โš ๏ธ Issue: Fundamental problem with model training or label mapping") - print(f"๐Ÿ’ก Solution: Retrain model with better data or check label mapping") + print("โš ๏ธ Issue: Fundamental problem with model training or label mapping") + print("๐Ÿ’ก Solution: Retrain model with better data or check label mapping") return training_like_accuracy > 0.8