diff --git a/deployment/api_server.py b/deployment/api_server.py index d1f4f4b4c..c8fe07234 100644 --- a/deployment/api_server.py +++ b/deployment/api_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿš€ EMOTION DETECTION API SERVER +"""๐Ÿš€ EMOTION DETECTION API SERVER =============================== REST API server for emotion detection with comprehensive security headers. """ diff --git a/deployment/cloud-run/config.py b/deployment/cloud-run/config.py index d44221d89..ac24b3f55 100644 --- a/deployment/cloud-run/config.py +++ b/deployment/cloud-run/config.py @@ -1,5 +1,4 @@ -""" -Environment Configuration Management - Phase 3 Cloud Run Optimization +"""Environment Configuration Management - Phase 3 Cloud Run Optimization Provides environment-specific settings for development, staging, and production """ @@ -216,4 +215,4 @@ def to_dict(self) -> Dict[str, Any]: def get_config() -> EnvironmentConfig: """Get the global configuration instance""" - return config + return config diff --git a/deployment/cloud-run/debug_api_import.py b/deployment/cloud-run/debug_api_import.py index 9ceee410d..5cb384e82 100644 --- a/deployment/cloud-run/debug_api_import.py +++ b/deployment/cloud-run/debug_api_import.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Debug script to isolate the 'int' object is not callable error +"""Debug script to isolate the 'int' object is not callable error """ import sys @@ -21,7 +20,7 @@ try: print("2. Importing Flask-RESTX...") - from flask_restx import Api, Resource, fields, Namespace + from flask_restx import Api, Namespace print("โœ… Flask-RESTX imported successfully") except Exception as e: print(f"โŒ Flask-RESTX import failed: {e}") @@ -75,21 +74,21 @@ def test_handler(error): # Now let's test the actual imports from secure_api_server.py try: print("\n7. Testing security_headers import...") - from security_headers import add_security_headers + from security_headers import add_security_headers # noqa: F401 print("โœ… security_headers imported successfully") except Exception as e: print(f"โŒ security_headers import failed: {e}") try: print("8. Testing rate_limiter import...") - from rate_limiter import rate_limit + from rate_limiter import rate_limit # noqa: F401 print("โœ… rate_limiter imported successfully") except Exception as e: print(f"โŒ rate_limiter import failed: {e}") try: print("9. Testing model_utils import...") - from model_utils import ensure_model_loaded, predict_emotions, get_model_status, validate_text_input + from model_utils import ensure_model_loaded, predict_emotions, get_model_status, validate_text_input # noqa: F401 print("โœ… model_utils imported successfully") except Exception as e: print(f"โŒ model_utils import failed: {e}") diff --git a/deployment/cloud-run/debug_errorhandler.py b/deployment/cloud-run/debug_errorhandler.py index 1e78cfe2f..e54f5bb71 100644 --- a/deployment/cloud-run/debug_errorhandler.py +++ b/deployment/cloud-run/debug_errorhandler.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Debug script to investigate the errorhandler issue +"""Debug script to investigate the errorhandler issue """ import sys @@ -13,7 +12,7 @@ try: from flask import Flask - from flask_restx import Api, Resource, fields, Namespace + from flask_restx import Api print("โœ… Imports successful") except Exception as e: print(f"โŒ Import failed: {e}") @@ -33,26 +32,26 @@ sys.exit(1) # Let's inspect the API object in detail -print(f"\n๐Ÿ” API object details:") +print("\n๐Ÿ” API object details:") print(f"Type: {type(api)}") print(f"Dir: {[attr for attr in dir(api) if not attr.startswith('_')]}") print(f"Has errorhandler: {'errorhandler' in dir(api)}") try: - errorhandler_method = getattr(api, 'errorhandler') + errorhandler_method = api.errorhandler print(f"โœ… errorhandler method found: {type(errorhandler_method)}") print(f"errorhandler callable: {callable(errorhandler_method)}") except Exception as e: print(f"โŒ errorhandler method access failed: {e}") # Let's check if there are any global variables that might be interfering -print(f"\n๐Ÿ” Checking for global variable conflicts...") +print("\n๐Ÿ” Checking for global variable conflicts...") print(f"Built-in errorhandler: {getattr(__builtins__, 'errorhandler', 'Not found')}") print(f"Global errorhandler: {globals().get('errorhandler', 'Not found')}") # Let's try to call errorhandler directly try: - print(f"\n๐Ÿ” Testing errorhandler call...") + print("\n๐Ÿ” Testing errorhandler call...") result = api.errorhandler(429) print(f"โœ… errorhandler(429) call successful: {type(result)}") except Exception as e: @@ -67,4 +66,4 @@ except Exception as e: print(f"โŒ Could not get Flask-RESTX version: {e}") -print("\n๐Ÿ” Debug complete.") \ No newline at end of file +print("\n๐Ÿ” Debug complete.") diff --git a/deployment/cloud-run/debug_errorhandler_detailed.py b/deployment/cloud-run/debug_errorhandler_detailed.py index 2aecdcb8d..bbe7d4a48 100644 --- a/deployment/cloud-run/debug_errorhandler_detailed.py +++ b/deployment/cloud-run/debug_errorhandler_detailed.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Detailed debug script to understand the errorhandler issue +"""Detailed debug script to understand the errorhandler issue """ import os @@ -25,13 +24,13 @@ exit(1) # Let's inspect the API object in detail -print(f"\n๐Ÿ” API object details:") +print("\n๐Ÿ” API object details:") print(f"Type: {type(api)}") print(f"Dir: {[attr for attr in dir(api) if not attr.startswith('_')]}") print(f"Has errorhandler: {'errorhandler' in dir(api)}") try: - errorhandler_method = getattr(api, 'errorhandler') + errorhandler_method = api.errorhandler print(f"โœ… errorhandler method found: {type(errorhandler_method)}") print(f"errorhandler callable: {callable(errorhandler_method)}") print(f"errorhandler bound: {errorhandler_method.__self__ if hasattr(errorhandler_method, '__self__') else 'Not bound'}") @@ -40,18 +39,18 @@ # Let's try to understand what happens when we call errorhandler try: - print(f"\n๐Ÿ” Testing errorhandler call step by step...") + print("\n๐Ÿ” Testing errorhandler call step by step...") # First, let's see what the method looks like print(f"errorhandler method: {errorhandler_method}") print(f"errorhandler method type: {type(errorhandler_method)}") # Let's try calling it with different approaches - print(f"\nTrying direct call...") + print("\nTrying direct call...") result = errorhandler_method(429) print(f"Direct call result: {type(result)} - {result}") - print(f"\nTrying bound call...") + print("\nTrying bound call...") result2 = api.errorhandler(429) print(f"Bound call result: {type(result2)} - {result2}") @@ -64,7 +63,7 @@ print(f"Error details: {e}") # Let's check if there are any global variables that might be interfering -print(f"\n๐Ÿ” Checking for global variable conflicts...") +print("\n๐Ÿ” Checking for global variable conflicts...") print(f"Built-in errorhandler: {getattr(__builtins__, 'errorhandler', 'Not found')}") print(f"Global errorhandler: {globals().get('errorhandler', 'Not found')}") @@ -76,4 +75,4 @@ except Exception as e: print(f"โŒ Could not get versions: {e}") -print("\n๐Ÿ” Debug complete.") \ No newline at end of file +print("\n๐Ÿ” Debug complete.") diff --git a/deployment/cloud-run/docs_blueprint.py b/deployment/cloud-run/docs_blueprint.py index 169a6a289..0a3a306bb 100644 --- a/deployment/cloud-run/docs_blueprint.py +++ b/deployment/cloud-run/docs_blueprint.py @@ -20,11 +20,11 @@ def serve_openapi_spec(): if os.path.commonpath([abs_spec_path, allowed_dir]) != allowed_dir: return jsonify({'error': 'Invalid OpenAPI spec path'}), 400 - with open(abs_spec_path, 'r', encoding='utf-8') as f: + with open(abs_spec_path, encoding='utf-8') as f: content = f.read() # Use a standard YAML mimetype return Response(content, mimetype='application/x-yaml') - except Exception as e: + except Exception: # Avoid leaking exact path in error; log on server side only if needed return jsonify({'error': 'OpenAPI spec not found'}), 404 diff --git a/deployment/cloud-run/health_monitor.py b/deployment/cloud-run/health_monitor.py index 8f681a028..7c0cdf453 100644 --- a/deployment/cloud-run/health_monitor.py +++ b/deployment/cloud-run/health_monitor.py @@ -1,5 +1,4 @@ -""" -Cloud Run Health Monitor - Phase 3 Optimization +"""Cloud Run Health Monitor - Phase 3 Optimization Provides comprehensive health checks, graceful shutdown, and monitoring """ @@ -88,7 +87,6 @@ def check_model_health() -> Dict[str, Any]: """Check if ML models are loaded and responding""" try: # Import models (this will fail if models aren't loaded) - from secure_api_server import app # Test model loading start_time = time.time() @@ -97,7 +95,7 @@ def check_model_health() -> Dict[str, Any]: import importlib modules_to_check = [ 'src.models.emotion_detection.bert_classifier', - 'src.models.summarization.t5_summarizer', + 'src.models.summarization.t5_summarizer', 'src.models.voice_processing.whisper_transcriber' ] @@ -239,4 +237,4 @@ def request_completed(self): def get_health_monitor() -> HealthMonitor: """Get the global health monitor instance""" - return health_monitor + return health_monitor diff --git a/deployment/cloud-run/minimal_api_server.py b/deployment/cloud-run/minimal_api_server.py index 5f90bc504..3e449cbfd 100644 --- a/deployment/cloud-run/minimal_api_server.py +++ b/deployment/cloud-run/minimal_api_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Minimal Emotion Detection API Server +"""Minimal Emotion Detection API Server Uses known working PyTorch/transformers combination Matches the actual model architecture: RoBERTa with 12 emotion classes """ @@ -8,7 +7,6 @@ import logging import os import time -import os from flask import Flask, request, jsonify import psutil @@ -155,4 +153,4 @@ def root(): # Start server port = int(os.getenv('PORT', '8080')) - app.run(host='0.0.0.0', port=port, debug=False, threaded=True) + app.run(host='0.0.0.0', port=port, debug=False, threaded=True) diff --git a/deployment/cloud-run/minimal_test.py b/deployment/cloud-run/minimal_test.py index dffdddac6..1fcdc69ae 100644 --- a/deployment/cloud-run/minimal_test.py +++ b/deployment/cloud-run/minimal_test.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Minimal test to isolate the API setup issue +"""Minimal test to isolate the API setup issue """ import os @@ -11,7 +10,7 @@ try: print("1. Importing modules...") from flask import Flask - from flask_restx import Api, Resource, fields, Namespace + from flask_restx import Api, fields, Namespace print("โœ… Imports successful") except Exception as e: print(f"โŒ Imports failed: {e}") @@ -69,4 +68,4 @@ def test_handler(error): print(f"API errorhandler type: {type(api.errorhandler)}") exit(1) -print("๐ŸŽ‰ All tests passed!") \ No newline at end of file +print("๐ŸŽ‰ All tests passed!") diff --git a/deployment/cloud-run/model_utils.py b/deployment/cloud-run/model_utils.py index fbe7a77c7..41b474cda 100644 --- a/deployment/cloud-run/model_utils.py +++ b/deployment/cloud-run/model_utils.py @@ -1,5 +1,4 @@ -""" -Shared model utilities for Cloud Run deployment. +"""Shared model utilities for Cloud Run deployment. This module provides common functionality for model loading, inference, and error handling to eliminate code duplication between API servers. @@ -48,7 +47,7 @@ def _load_repo_id_from_config() -> Optional[str]: cfg_path = Path('deployment/custom_model_config.json') if cfg_path.exists(): try: - with open(cfg_path, 'r') as f: + with open(cfg_path) as f: cfg = json.load(f) repo_id = cfg.get('model_name') or cfg.get('repo_id') if repo_id: @@ -74,8 +73,7 @@ def _resolve_model_repo_id() -> str: def ensure_model_loaded() -> bool: - """ - Thread-safe model loading with proper error handling. + """Thread-safe model loading with proper error handling. Returns: bool: True if model is loaded successfully, False otherwise @@ -163,14 +161,13 @@ def ensure_model_loaded() -> bool: with model_lock: model_loading = False - logger.exception(f"โŒ Failed to load model: {str(e)}") + logger.exception(f"โŒ Failed to load model: {e!s}") logger.error("Model loading failed - check model configuration") return False def predict_emotions(text: str) -> Dict[str, Any]: - """ - Predict emotions for given text. + """Predict emotions for given text. Args: text (str): Input text to analyze @@ -258,7 +255,7 @@ def predict_emotions(text: str) -> Dict[str, Any]: } except Exception as e: - logger.exception(f"โŒ Prediction failed: {str(e)}") + logger.exception(f"โŒ Prediction failed: {e!s}") return { 'error': 'Prediction failed', 'emotions': [], @@ -267,8 +264,7 @@ def predict_emotions(text: str) -> Dict[str, Any]: def get_model_status() -> Dict[str, Any]: - """ - Get current model status. + """Get current model status. Returns: Dict[str, Any]: Model status information @@ -287,8 +283,7 @@ def get_model_status() -> Dict[str, Any]: def validate_text_input(text: str) -> Tuple[bool, str]: - """ - Validate text input for prediction. + """Validate text input for prediction. Args: text (str): Text to validate @@ -300,4 +295,4 @@ def validate_text_input(text: str) -> Tuple[bool, str]: return False, 'Text must be a non-empty string' if len(text) > MAX_TEXT_LENGTH: return False, f'Text too long (max {MAX_TEXT_LENGTH} characters)' - return True, '' + return True, '' diff --git a/deployment/cloud-run/onnx_api_server.py b/deployment/cloud-run/onnx_api_server.py index 7354c35fc..44608de8f 100644 --- a/deployment/cloud-run/onnx_api_server.py +++ b/deployment/cloud-run/onnx_api_server.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 -""" -Simplified ONNX-Based Emotion Detection API Server +"""Simplified ONNX-Based Emotion Detection API Server Uses simple string tokenization - no complex dependencies """ import logging import os import time import re -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Tuple import threading import numpy as np @@ -75,7 +74,7 @@ def load_vocab() -> Dict[str, int]: try: if os.path.exists(VOCAB_PATH): vocab_dict = {} - with open(VOCAB_PATH, 'r', encoding='utf-8') as f: + with open(VOCAB_PATH, encoding='utf-8') as f: for i, line in enumerate(f): word = line.strip() if word: @@ -362,4 +361,4 @@ def load(self): except ImportError: # Development server - app.run(host='127.0.0.1', port=8080, debug=False) + app.run(host='127.0.0.1', port=8080, debug=False) diff --git a/deployment/cloud-run/robust_predict.py b/deployment/cloud-run/robust_predict.py index 713de8542..cf87a8235 100644 --- a/deployment/cloud-run/robust_predict.py +++ b/deployment/cloud-run/robust_predict.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿš€ EMOTION DETECTION API FOR CLOUD RUN +"""๐Ÿš€ EMOTION DETECTION API FOR CLOUD RUN ====================================== Robust Flask API optimized for Cloud Run deployment. """ @@ -301,4 +300,4 @@ def load(self): 'loglevel': 'info' } - StandaloneApplication(app, options).run() \ No newline at end of file + StandaloneApplication(app, options).run() diff --git a/deployment/cloud-run/secure_api_server.py b/deployment/cloud-run/secure_api_server.py index beca133e2..9e19f6ee6 100644 --- a/deployment/cloud-run/secure_api_server.py +++ b/deployment/cloud-run/secure_api_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿš€ SECURE EMOTION DETECTION API FOR CLOUD RUN +"""๐Ÿš€ SECURE EMOTION DETECTION API FOR CLOUD RUN ============================================ Production-ready Flask API with comprehensive security features and Swagger documentation. """ @@ -22,7 +21,6 @@ # Import shared model utilities from model_utils import ( ensure_model_loaded, predict_emotions, get_model_status, - validate_text_input, ) # Configure logging for Cloud Run @@ -52,7 +50,7 @@ def home(): # Changed from api_root to home to avoid conflict with Flask-RESTX' 'timestamp': time.time() }) except Exception as e: - logger.error(f"Root endpoint error for {request.remote_addr}: {str(e)}") + logger.error(f"Root endpoint error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) # Initialize Flask-RESTX API without Swagger to avoid 500 errors @@ -229,7 +227,7 @@ def before_request(): logger.info(f"๐Ÿ“ฅ Request: {request.method} {request.path} from {request.remote_addr} (ID: {g.request_id})") # Log request headers for debugging (excluding sensitive ones) - headers_to_log = {k: v for k, v in request.headers.items() + headers_to_log = {k: v for k, v in request.headers.items() if k.lower() not in ['authorization', 'x-api-key', 'cookie']} logger.debug(f"๐Ÿ“‹ Request headers: {headers_to_log}") @@ -276,7 +274,7 @@ def get(self): return create_error_response('Service unavailable - model not ready', 503) except Exception as e: - logger.error(f"Health check error for {request.remote_addr}: {str(e)}") + logger.error(f"Health check error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) @main_ns.route('/predict') @@ -311,7 +309,9 @@ def post(self): try: text = sanitize_input(text) except ValueError as e: - logger.warning(f"Input sanitization failed for {request.remote_addr}: {str(e)}") + logger.warning( + f"Input sanitization failed for {request.remote_addr}: {e!s}" + ) return create_error_response(str(e), 400) # Ensure model is loaded @@ -325,7 +325,7 @@ def post(self): return result except Exception as e: - logger.error(f"Prediction error for {request.remote_addr}: {str(e)}") + logger.error(f"Prediction error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) @main_ns.route('/predict_batch') @@ -377,13 +377,15 @@ def post(self): result = predict_emotion(text) results.append(result) except Exception as e: - logger.warning(f"Failed to process text in batch from {request.remote_addr}: {str(e)}") + logger.warning( + f"Failed to process text in batch from {request.remote_addr}: {e!s}" + ) continue return {'results': results} except Exception as e: - logger.error(f"Batch prediction error for {request.remote_addr}: {str(e)}") + logger.error(f"Batch prediction error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) @main_ns.route('/emotions') @@ -401,7 +403,7 @@ def get(self): 'timestamp': time.time() } except Exception as e: - logger.error(f"Emotions endpoint error for {request.remote_addr}: {str(e)}") + logger.error(f"Emotions endpoint error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) # Admin endpoints @@ -420,7 +422,7 @@ def get(self): status = get_model_status() return status except Exception as e: - logger.error(f"Model status error for {request.remote_addr}: {str(e)}") + logger.error(f"Model status error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) @admin_ns.route('/security_status') @@ -443,7 +445,7 @@ def get(self): 'timestamp': time.time() } except Exception as e: - logger.error(f"Security status error for {request.remote_addr}: {str(e)}") + logger.error(f"Security status error for {request.remote_addr}: {e!s}") return create_error_response('Internal server error', 500) # Error handlers for Flask-RESTX - using direct registration due to decorator compatibility issue @@ -454,7 +456,7 @@ def rate_limit_exceeded(error): def internal_error(error): """Handle internal server errors""" - logger.error(f"Internal server error for {request.remote_addr}: {str(error)}") + logger.error(f"Internal server error for {request.remote_addr}: {error!s}") return create_error_response('Internal server error', 500) def not_found(error): @@ -469,7 +471,7 @@ def method_not_allowed(error): def handle_unexpected_error(error): """Handle any unexpected errors""" - logger.error(f"Unexpected error for {request.remote_addr}: {str(error)}") + logger.error(f"Unexpected error for {request.remote_addr}: {error!s}") return create_error_response('An unexpected error occurred', 500) # Register error handlers directly @@ -484,7 +486,7 @@ def initialize_model(): try: logger.info("๐Ÿš€ Initializing emotion detection API server...") logger.info(f"๐Ÿ“Š Configuration: MAX_INPUT_LENGTH={MAX_INPUT_LENGTH}, RATE_LIMIT={RATE_LIMIT_PER_MINUTE}/min") - logger.info(f"๐Ÿ” Security: API key protection enabled, Admin API key configured") + logger.info("๐Ÿ” Security: API key protection enabled, Admin API key configured") logger.info(f"๐ŸŒ Server: Port {PORT}, Model path: {MODEL_PATH}") logger.info(f"๐Ÿ”„ Rate limiting: {RATE_LIMIT_PER_MINUTE} requests per minute") @@ -495,7 +497,7 @@ def initialize_model(): logger.info("๐Ÿš€ API server ready to handle requests") except Exception as e: - logger.error(f"โŒ Failed to initialize API server: {str(e)}") + logger.error(f"โŒ Failed to initialize API server: {e!s}") raise # Initialize model when the application starts diff --git a/deployment/cloud-run/security_headers.py b/deployment/cloud-run/security_headers.py index 0aebcc545..c3b5ead17 100644 --- a/deployment/cloud-run/security_headers.py +++ b/deployment/cloud-run/security_headers.py @@ -2,7 +2,6 @@ """Security Headers Module for Cloud Run API""" from flask import Flask, request, g -from typing import Dict, Any def add_security_headers(app: Flask) -> None: """Add comprehensive security headers to Flask app""" diff --git a/deployment/cloud-run/test_direct_errorhandler.py b/deployment/cloud-run/test_direct_errorhandler.py index 00f16200a..1de355f09 100644 --- a/deployment/cloud-run/test_direct_errorhandler.py +++ b/deployment/cloud-run/test_direct_errorhandler.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test direct error handler registration +"""Test direct error handler registration """ import os @@ -61,4 +60,4 @@ def flask_internal_error_handler(error): except Exception as e: print(f"โŒ Flask app error handler failed: {e}") -print("\n๏ฟฝ๏ฟฝ Test complete.") \ No newline at end of file +print("\n๏ฟฝ๏ฟฝ Test complete.") diff --git a/deployment/cloud-run/test_docs_error.py b/deployment/cloud-run/test_docs_error.py index ab387bab1..f16007c09 100644 --- a/deployment/cloud-run/test_docs_error.py +++ b/deployment/cloud-run/test_docs_error.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test script to investigate the Swagger docs 500 error +"""Test script to investigate the Swagger docs 500 error """ import os @@ -56,4 +55,4 @@ def run_server(): except Exception as e: print(f"โŒ Error: {e}") import traceback - traceback.print_exc() \ No newline at end of file + traceback.print_exc() diff --git a/deployment/cloud-run/test_minimal_import.py b/deployment/cloud-run/test_minimal_import.py index 1bd62f110..62b974598 100644 --- a/deployment/cloud-run/test_minimal_import.py +++ b/deployment/cloud-run/test_minimal_import.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Minimal test to isolate the API issue +"""Minimal test to isolate the API issue """ import os @@ -52,4 +51,4 @@ print(f"Error type: {type(e)}") exit(1) -print("๐ŸŽ‰ All tests passed!") \ No newline at end of file +print("๐ŸŽ‰ All tests passed!") diff --git a/deployment/cloud-run/test_minimal_swagger.py b/deployment/cloud-run/test_minimal_swagger.py index a372cc6c7..011f703b4 100644 --- a/deployment/cloud-run/test_minimal_swagger.py +++ b/deployment/cloud-run/test_minimal_swagger.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Minimal test to isolate Swagger docs issue +"""Minimal test to isolate Swagger docs issue """ import os @@ -45,4 +44,4 @@ def get(self): print("- http://localhost:5003/docs (should work)") print("- http://localhost:5003/api/health (should work)") - app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5003)), debug=False) # Debug mode disabled for security \ No newline at end of file + app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5003)), debug=False) # Debug mode disabled for security diff --git a/deployment/cloud-run/test_routing_debug.py b/deployment/cloud-run/test_routing_debug.py index a7a53a252..395197113 100644 --- a/deployment/cloud-run/test_routing_debug.py +++ b/deployment/cloud-run/test_routing_debug.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Debug script to understand Flask-RESTX routing behavior +"""Debug script to understand Flask-RESTX routing behavior """ from flask import Flask, jsonify @@ -81,4 +80,4 @@ def root(): if rule.rule == '/': print(f"Root route: {rule.rule} -> {rule.endpoint}") print(f" Methods: {rule.methods}") - print(f" View function: {rule.endpoint}") \ No newline at end of file + print(f" View function: {rule.endpoint}") diff --git a/deployment/cloud-run/test_routing_fixed.py b/deployment/cloud-run/test_routing_fixed.py index dc3e579f5..d4c3978ad 100644 --- a/deployment/cloud-run/test_routing_fixed.py +++ b/deployment/cloud-run/test_routing_fixed.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test script to verify the fixed routing in secure_api_server.py +"""Test script to verify the fixed routing in secure_api_server.py """ import os @@ -54,4 +53,4 @@ except Exception as e: print(f"โŒ Error testing routing: {e}") import traceback - traceback.print_exc() \ No newline at end of file + traceback.print_exc() diff --git a/deployment/cloud-run/test_routing_minimal.py b/deployment/cloud-run/test_routing_minimal.py index 73f2ea03e..9203ff407 100644 --- a/deployment/cloud-run/test_routing_minimal.py +++ b/deployment/cloud-run/test_routing_minimal.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Minimal test script to isolate Flask-RESTX routing issues +"""Minimal test script to isolate Flask-RESTX routing issues """ import os @@ -54,4 +53,4 @@ def root(): print(f"API: {rule.rule} -> {rule.endpoint}") print("\n=== Starting test server ===") - app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)), debug=False) # Debug mode disabled for security \ No newline at end of file + app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5000)), debug=False) # Debug mode disabled for security diff --git a/deployment/cloud-run/test_server_start.py b/deployment/cloud-run/test_server_start.py index 19eb6edd1..c761949d9 100644 --- a/deployment/cloud-run/test_server_start.py +++ b/deployment/cloud-run/test_server_start.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test script to verify the server starts and responds correctly +"""Test script to verify the server starts and responds correctly """ import os @@ -62,4 +61,4 @@ def run_server(): except Exception as e: print(f"โŒ Error testing server: {e}") import traceback - traceback.print_exc() \ No newline at end of file + traceback.print_exc() diff --git a/deployment/cloud-run/test_swagger_debug.py b/deployment/cloud-run/test_swagger_debug.py index fdb5b3f40..2c0ac9f96 100644 --- a/deployment/cloud-run/test_swagger_debug.py +++ b/deployment/cloud-run/test_swagger_debug.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test script to debug Swagger docs 500 error +"""Test script to debug Swagger docs 500 error """ import os @@ -45,4 +44,4 @@ def api_root(): # Different function name to avoid conflict print("- http://localhost:5001/docs (should work)") print("- http://localhost:5001/api/health (should work)") - app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5001)), debug=False) # Debug mode disabled for security \ No newline at end of file + app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 5001)), debug=False) # Debug mode disabled for security diff --git a/deployment/cloud-run/test_swagger_debug_detailed.py b/deployment/cloud-run/test_swagger_debug_detailed.py index 0cb467f87..2f4b56f75 100644 --- a/deployment/cloud-run/test_swagger_debug_detailed.py +++ b/deployment/cloud-run/test_swagger_debug_detailed.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Detailed test to capture Swagger docs 500 error +"""Detailed test to capture Swagger docs 500 error """ import os @@ -81,4 +80,4 @@ def run_server(): except Exception as e: print(f"โŒ Error: {e}") - traceback.print_exc() \ No newline at end of file + traceback.print_exc() diff --git a/deployment/cloud-run/test_swagger_no_model.py b/deployment/cloud-run/test_swagger_no_model.py index 09b350a00..d0fc76f15 100644 --- a/deployment/cloud-run/test_swagger_no_model.py +++ b/deployment/cloud-run/test_swagger_no_model.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Test Swagger docs without model dependencies +"""Test Swagger docs without model dependencies """ import os @@ -52,4 +51,4 @@ def get(self): print("- http://localhost:8083/docs (should work)") print("- http://localhost:8083/api/health (should work)") - app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 8083)), debug=False) # Debug mode disabled for security \ No newline at end of file + app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 8083)), debug=False) # Debug mode disabled for security diff --git a/deployment/gcp/predict.py b/deployment/gcp/predict.py index 73fc60bff..e5795fb36 100644 --- a/deployment/gcp/predict.py +++ b/deployment/gcp/predict.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Vertex AI Custom Container Prediction Server +"""Vertex AI Custom Container Prediction Server =========================================== This script runs a Flask server for the emotion detection model on Vertex AI. @@ -34,7 +33,7 @@ def __init__(self): print("โœ… Model loaded successfully") except Exception as e: - print(f"โŒ Failed to load model: {str(e)}") + print(f"โŒ Failed to load model: {e!s}") raise def predict(self, text): @@ -84,7 +83,7 @@ def predict(self, text): return response except Exception as e: - print(f"Prediction error: {str(e)}") + print(f"Prediction error: {e!s}") raise # Initialize model @@ -119,7 +118,7 @@ def predict(): return jsonify(result) except Exception as e: - print(f"Prediction endpoint error: {str(e)}") + print(f"Prediction endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/', methods=['GET']) diff --git a/deployment/inference.py b/deployment/inference.py index 430f45042..49b503fe8 100644 --- a/deployment/inference.py +++ b/deployment/inference.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -EMOTION DETECTION INFERENCE SCRIPT +"""EMOTION DETECTION INFERENCE SCRIPT ===================================== Standalone script to run emotion detection on text. """ @@ -78,8 +77,8 @@ def main(): # Make prediction result = detector.predict(text) - print(f"\n๐ŸŽฏ EMOTION DETECTION RESULT") - print(f"=" * 40) + print("\n๐ŸŽฏ EMOTION DETECTION RESULT") + print("=" * 40) print(f"Text: {result['text']}") print(f"Emotion: {result['emotion']}") print(f"Confidence: {result['confidence']:.3f}") diff --git a/deployment/local/api_server.py b/deployment/local/api_server.py index 56224e566..ff76e4753 100644 --- a/deployment/local/api_server.py +++ b/deployment/local/api_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Local Emotion Detection API Server +"""Local Emotion Detection API Server ================================= A production-ready Flask API server with monitoring, logging, @@ -126,7 +125,7 @@ def __init__(self): logger.info("โœ… Model loaded successfully") except Exception as e: - logger.error(f"โŒ Failed to load model: {str(e)}") + logger.error(f"โŒ Failed to load model: {e!s}") raise def predict(self, text): @@ -183,7 +182,7 @@ def predict(self, text): except Exception as e: prediction_time = time.time() - start_time - logger.error(f"Prediction failed after {prediction_time:.3f}s: {str(e)}") + logger.error(f"Prediction failed after {prediction_time:.3f}s: {e!s}") raise # Initialize model @@ -219,7 +218,7 @@ def health_check(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='health_check_error') - logger.error(f"Health check failed: {str(e)}") + logger.error(f"Health check failed: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/predict', methods=['POST']) @@ -253,12 +252,12 @@ def predict(): except werkzeug.exceptions.BadRequest: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='invalid_json') - logger.error(f"Invalid JSON in request") + logger.error("Invalid JSON in request") return jsonify({'error': 'Invalid JSON format'}), 400 except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='prediction_error') - logger.error(f"Prediction endpoint error: {str(e)}") + logger.error(f"Prediction endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/predict_batch', methods=['POST']) @@ -299,12 +298,12 @@ def predict_batch(): except werkzeug.exceptions.BadRequest: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='invalid_json') - logger.error(f"Invalid JSON in batch request") + logger.error("Invalid JSON in batch request") return jsonify({'error': 'Invalid JSON format'}), 400 except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='batch_prediction_error') - logger.error(f"Batch prediction endpoint error: {str(e)}") + logger.error(f"Batch prediction endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/metrics', methods=['GET']) @@ -380,13 +379,13 @@ def home(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='documentation_error') - logger.error(f"Documentation endpoint error: {str(e)}") + logger.error(f"Documentation endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.errorhandler(werkzeug.exceptions.BadRequest) def handle_bad_request(e): """Handle BadRequest exceptions (invalid JSON, etc.).""" - logger.error(f"BadRequest error: {str(e)}") + logger.error(f"BadRequest error: {e!s}") update_metrics(0.0, success=False, error_type='invalid_json') return jsonify({'error': 'Invalid JSON format'}), 400 diff --git a/deployment/local/test_api.py b/deployment/local/test_api.py index fb3c415c6..c82683814 100644 --- a/deployment/local/test_api.py +++ b/deployment/local/test_api.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Enhanced API Testing Script +"""Enhanced API Testing Script =========================== Comprehensive testing for the enhanced emotion detection API with monitoring, @@ -36,7 +35,7 @@ def test_health_check(): response = requests.get(f"{BASE_URL}/health") if response.status_code == 200: data = response.json() - print(f"โœ… Health check passed") + print("โœ… Health check passed") print(f" Status: {data['status']}") print(f" Model Version: {data['model_version']}") print(f" Uptime: {data['uptime_seconds']:.1f} seconds") @@ -48,7 +47,7 @@ def test_health_check(): print(f"โŒ Health check failed: {response.status_code}") return False except Exception as e: - print(f"โŒ Health check error: {str(e)}") + print(f"โŒ Health check error: {e!s}") return False def test_metrics_endpoint(): @@ -58,7 +57,7 @@ def test_metrics_endpoint(): response = requests.get(f"{BASE_URL}/metrics") if response.status_code == 200: data = response.json() - print(f"โœ… Metrics endpoint working") + print("โœ… Metrics endpoint working") print(f" Success Rate: {data['server_metrics']['success_rate']}") print(f" Requests/Minute: {data['server_metrics']['requests_per_minute']:.2f}") print(f" Rate Limiting: {data['rate_limiting']['max_requests']} req/{data['rate_limiting']['window_seconds']}s") @@ -67,7 +66,7 @@ def test_metrics_endpoint(): print(f"โŒ Metrics endpoint failed: {response.status_code}") return False except Exception as e: - print(f"โŒ Metrics endpoint error: {str(e)}") + print(f"โŒ Metrics endpoint error: {e!s}") return False def test_single_predictions(): @@ -105,7 +104,7 @@ def test_single_predictions(): return False except Exception as e: - print(f"โŒ Test {i} error: {str(e)}") + print(f"โŒ Test {i} error: {e!s}") return False # Calculate average performance @@ -150,7 +149,7 @@ def test_batch_predictions(): return False except Exception as e: - print(f"โŒ Batch prediction error: {str(e)}") + print(f"โŒ Batch prediction error: {e!s}") return False def test_rate_limiting(): @@ -189,7 +188,7 @@ def make_request(): print(f" โœ… Rate limiting is working (blocked {rate_limited} requests)") return True else: - print(f" โš ๏ธ No rate limiting detected (may need more requests)") + print(" โš ๏ธ No rate limiting detected (may need more requests)") return True def test_error_handling(): @@ -209,7 +208,7 @@ def test_error_handling(): print(f"โŒ Missing text error not handled: {response.status_code}") return False except Exception as e: - print(f"โŒ Missing text test error: {str(e)}") + print(f"โŒ Missing text test error: {e!s}") return False # Test empty text @@ -225,7 +224,7 @@ def test_error_handling(): print(f"โŒ Empty text error not handled: {response.status_code}") return False except Exception as e: - print(f"โŒ Empty text test error: {str(e)}") + print(f"โŒ Empty text test error: {e!s}") return False # Test invalid JSON @@ -241,7 +240,7 @@ def test_error_handling(): print(f"โŒ Invalid JSON error not handled: {response.status_code}") return False except Exception as e: - print(f"โŒ Invalid JSON test error: {str(e)}") + print(f"โŒ Invalid JSON test error: {e!s}") return False return True @@ -328,10 +327,10 @@ def main(): else: print(f"โŒ {test_name} failed") except Exception as e: - print(f"โŒ {test_name} error: {str(e)}") + print(f"โŒ {test_name} error: {e!s}") print("\n" + "=" * 50) - print(f"๐ŸŽ‰ ENHANCED API TESTING COMPLETED!") + print("๐ŸŽ‰ ENHANCED API TESTING COMPLETED!") print(f"๐Ÿ“Š Results: {passed}/{total} tests passed") if passed == total: diff --git a/deployment/secure_api_server.py b/deployment/secure_api_server.py index bb92d69da..0a632f20b 100644 --- a/deployment/secure_api_server.py +++ b/deployment/secure_api_server.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿ”’ SECURE EMOTION DETECTION API SERVER +"""๐Ÿ”’ SECURE EMOTION DETECTION API SERVER ====================================== Production-ready Flask API server with comprehensive security features. @@ -163,7 +162,7 @@ def decorated_function(*args, **kwargs): response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='endpoint_error') - logger.error(f"Endpoint error: {str(e)}") + logger.error(f"Endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 return decorated_function @@ -240,7 +239,9 @@ def __init__(self): logger.info("โœ… Secure model loaded successfully") except Exception as e: - logger.error(f"โŒ Failed to load secure model: {str(e)}. Falling back to stub mode.") + logger.error( + f"โŒ Failed to load secure model: {e!s}. Falling back to stub mode." + ) self.tokenizer = None self.model = None self.loaded = False @@ -318,7 +319,9 @@ def predict(self, text, confidence_threshold=None): except Exception as e: prediction_time = time.time() - start_time - logger.error(f"Secure prediction failed after {prediction_time:.3f}s: {str(e)}") + logger.error( + f"Secure prediction failed after {prediction_time:.3f}s: {e!s}" + ) raise # Secure model factory for explicit creation and testability @@ -412,7 +415,7 @@ def health_check(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='health_check_error') - logger.error(f"Health check failed: {str(e)}") + logger.error(f"Health check failed: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/predict', methods=['POST']) @@ -442,7 +445,7 @@ def predict(): except ValueError as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='validation_error') - logger.warning(f"Validation error: {str(e)} from {request.remote_addr}") + logger.warning(f"Validation error: {e!s} from {request.remote_addr}") return jsonify({'error': str(e)}), 400 # Detect anomalies @@ -467,8 +470,8 @@ def predict(): response_time = time.time() - start_time update_metrics( - response_time, - success=True, + response_time, + success=True, emotion=result['predicted_emotion'], sanitization_warnings=len(warnings) ) @@ -478,7 +481,7 @@ def predict(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='prediction_error') - logger.error(f"Secure prediction endpoint error: {str(e)}") + logger.error(f"Secure prediction endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/predict_batch', methods=['POST']) @@ -508,7 +511,7 @@ def predict_batch(): except ValueError as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='validation_error') - logger.warning(f"Batch validation error: {str(e)} from {request.remote_addr}") + logger.warning(f"Batch validation error: {e!s} from {request.remote_addr}") return jsonify({'error': str(e)}), 400 # Detect anomalies @@ -533,7 +536,7 @@ def predict_batch(): response_time = time.time() - start_time update_metrics( - response_time, + response_time, success=True, sanitization_warnings=len(warnings) ) @@ -552,7 +555,7 @@ def predict_batch(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='batch_prediction_error') - logger.error(f"Secure batch prediction endpoint error: {str(e)}") + logger.error(f"Secure batch prediction endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/metrics', methods=['GET']) @@ -595,7 +598,7 @@ def add_to_blacklist(): logger.info(f"Added {ip} to blacklist") return jsonify({'message': f'Added {ip} to blacklist'}) except Exception as e: - logger.error(f"Blacklist error: {str(e)}") + logger.error(f"Blacklist error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/security/whitelist', methods=['POST']) @@ -612,7 +615,7 @@ def add_to_whitelist(): logger.info(f"Added {ip} to whitelist") return jsonify({'message': f'Added {ip} to whitelist'}) except Exception as e: - logger.error(f"Whitelist error: {str(e)}") + logger.error(f"Whitelist error: {e!s}") return jsonify({'error': str(e)}), 500 @app.route('/', methods=['GET']) @@ -672,13 +675,13 @@ def home(): except Exception as e: response_time = time.time() - start_time update_metrics(response_time, success=False, error_type='documentation_error') - logger.error(f"Documentation endpoint error: {str(e)}") + logger.error(f"Documentation endpoint error: {e!s}") return jsonify({'error': str(e)}), 500 @app.errorhandler(werkzeug.exceptions.BadRequest) def handle_bad_request(e): """Handle BadRequest exceptions (invalid JSON, etc.).""" - logger.error(f"BadRequest error: {str(e)}") + logger.error(f"BadRequest error: {e!s}") update_metrics(0.0, success=False, error_type='invalid_json') return jsonify({'error': 'Invalid JSON format'}), 400 @@ -691,7 +694,7 @@ def handle_not_found(e): @app.errorhandler(500) def handle_internal_error(e): """Handle 500 errors.""" - logger.error(f"Internal server error: {str(e)}") + logger.error(f"Internal server error: {e!s}") return jsonify({'error': 'Internal server error'}), 500 if __name__ == '__main__': @@ -725,4 +728,4 @@ def handle_internal_error(e): logger.info("๐Ÿ›ก๏ธ Security monitoring: Comprehensive logging and metrics enabled") logger.info("=" * 60) - app.run(host='0.0.0.0', port=8000, debug=False) \ No newline at end of file + app.run(host='0.0.0.0', port=8000, debug=False) diff --git a/deployment/test_examples.py b/deployment/test_examples.py index fa1cb949f..c5ebd3c2b 100644 --- a/deployment/test_examples.py +++ b/deployment/test_examples.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿงช TEST EMOTION DETECTION MODEL +"""๐Ÿงช TEST EMOTION DETECTION MODEL =============================== Test the trained model with various examples. """ diff --git a/scripts/ci/pre_warm_models.py b/scripts/ci/pre_warm_models.py index dd1eee916..34e0683ca 100644 --- a/scripts/ci/pre_warm_models.py +++ b/scripts/ci/pre_warm_models.py @@ -45,4 +45,4 @@ def pre_warm_models(): if __name__ == "__main__": success = pre_warm_models() - sys.exit(0 if success else 1) + sys.exit(0 if success else 1) diff --git a/scripts/ci/run_full_ci_pipeline.py b/scripts/ci/run_full_ci_pipeline.py index d2c900023..84f388e3f 100644 --- a/scripts/ci/run_full_ci_pipeline.py +++ b/scripts/ci/run_full_ci_pipeline.py @@ -18,7 +18,7 @@ import time import subprocess from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, Tuple # Use shared truthy parsing try: @@ -47,7 +47,7 @@ def __init__(self): self.start_time = time.time() self.ci_scripts = [ "scripts/ci/api_health_check.py", - "scripts/ci/bert_model_test.py", + "scripts/ci/bert_model_test.py", "scripts/ci/t5_summarization_test.py", "scripts/ci/whisper_transcription_test.py", "scripts/ci/model_calibration_test.py", @@ -138,7 +138,7 @@ def run_ci_script(self, script_path: str) -> Tuple[bool, str]: # Run the script result = subprocess.run( [python_executable, script_path], - capture_output=True, + check=False, capture_output=True, text=True, timeout=300 # 5 minute timeout ) @@ -165,7 +165,7 @@ def run_unit_tests(self) -> bool: try: result = subprocess.run( [sys.executable, "-m", "pytest", "tests/unit/", "-v"], - capture_output=True, + check=False, capture_output=True, text=True, timeout=1200 # 20 minute timeout (increased from 10) ) @@ -194,7 +194,7 @@ def run_e2e_tests(self) -> bool: try: result = subprocess.run( [sys.executable, "-m", "pytest", "tests/e2e/", "-v"], - capture_output=True, + check=False, capture_output=True, text=True, timeout=900 # 15 minute timeout ) @@ -375,7 +375,7 @@ def generate_report(self) -> str: if passed_tests == total_tests: report += "๐ŸŽ‰ All tests passed! Pipeline is ready for deployment.\n" else: - failed_test_names = [name for name, result in test_results.items() + failed_test_names = [name for name, result in test_results.items() if not result] report += f"โš ๏ธ Failed tests: {', '.join(failed_test_names)}\n" report += "๐Ÿ”ง Please fix the failed tests before deployment.\n" @@ -421,4 +421,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/deployment/bake_emotion_model.py b/scripts/deployment/bake_emotion_model.py index 84a8aa6cf..151f528e8 100644 --- a/scripts/deployment/bake_emotion_model.py +++ b/scripts/deployment/bake_emotion_model.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys from transformers import AutoTokenizer, AutoModelForSequenceClassification @@ -34,4 +33,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/scripts/deployment/complete_project_deployment.py b/scripts/deployment/complete_project_deployment.py index 1c9289553..1043be152 100644 --- a/scripts/deployment/complete_project_deployment.py +++ b/scripts/deployment/complete_project_deployment.py @@ -29,7 +29,7 @@ def check_project_status(): # Check for trained models model_paths = [ "./emotion_model_ensemble_final", - "./emotion_model_specialized_final", + "./emotion_model_specialized_final", "./emotion_model_fixed_bulletproof_final", "./emotion_model" ] @@ -57,7 +57,7 @@ def save_model_for_deployment(): # Run the model saving script result = subprocess.run([ sys.executable, "scripts/save_trained_model_for_deployment.py" - ], capture_output=True, text=True) + ], check=False, capture_output=True, text=True) if result.returncode == 0: print("โœ… Model saved successfully!") @@ -85,7 +85,7 @@ def test_deployment_package(): # Test the model result = subprocess.run([ sys.executable, "deployment/test_examples.py" - ], capture_output=True, text=True) + ], check=False, capture_output=True, text=True) if result.returncode == 0: print("โœ… Deployment package test passed!") @@ -255,7 +255,9 @@ def run_final_tests(): for test_name, command in tests: try: - result = subprocess.run(command, shell=True, capture_output=True, text=True) + result = subprocess.run( + command, check=False, shell=True, capture_output=True, text=True + ) if result.returncode == 0: print(f"โœ… {test_name}: PASSED") passed += 1 @@ -319,4 +321,4 @@ def main(): if __name__ == "__main__": success = main() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/deployment/convert_model_to_onnx.py b/scripts/deployment/convert_model_to_onnx.py index d54a04dc7..2f6ac318d 100644 --- a/scripts/deployment/convert_model_to_onnx.py +++ b/scripts/deployment/convert_model_to_onnx.py @@ -170,4 +170,4 @@ def main(): if __name__ == "__main__": - main() + main() diff --git a/scripts/deployment/convert_model_to_onnx_simple.py b/scripts/deployment/convert_model_to_onnx_simple.py index 74fe747d7..c57915640 100644 --- a/scripts/deployment/convert_model_to_onnx_simple.py +++ b/scripts/deployment/convert_model_to_onnx_simple.py @@ -161,4 +161,4 @@ def main(): if __name__ == "__main__": - main() + main() diff --git a/scripts/deployment/create_model_deployment_package.py b/scripts/deployment/create_model_deployment_package.py index 951fd0143..bff08fcda 100644 --- a/scripts/deployment/create_model_deployment_package.py +++ b/scripts/deployment/create_model_deployment_package.py @@ -446,7 +446,7 @@ def get_emotions(): print("โœ… Deployment package created: deployment/") print("๐Ÿ“ฆ Files included:") - for filename in deployment_files.keys(): + for filename in deployment_files: print(f" - {filename}") print("๐Ÿš€ Next steps:") print(" 1. Copy trained model to deployment/model/") @@ -454,4 +454,4 @@ def get_emotions(): print(" 3. Test API at: http://localhost:5000") if __name__ == "__main__": - create_model_deployment_package() \ No newline at end of file + create_model_deployment_package() diff --git a/scripts/deployment/deploy_locally.py b/scripts/deployment/deploy_locally.py index 9e7823169..f89060fa8 100644 --- a/scripts/deployment/deploy_locally.py +++ b/scripts/deployment/deploy_locally.py @@ -7,7 +7,6 @@ for testing before cloud deployment. """ -import os import json import sys from datetime import datetime @@ -413,7 +412,7 @@ def test_api(): deployment_info_path.write_text(json.dumps(deployment_summary, indent=2)) print("โœ… Deployment info created") - print(f"\nโœ… LOCAL DEPLOYMENT READY!") + print("\nโœ… LOCAL DEPLOYMENT READY!") print("=" * 50) print(f"๐Ÿ“ Deployment directory: {local_deployment_dir}") print() @@ -440,4 +439,4 @@ def test_api(): if __name__ == "__main__": success = deploy_locally() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/deployment/deploy_to_gcp_vertex_ai.py b/scripts/deployment/deploy_to_gcp_vertex_ai.py index 34798f4d1..318d7091b 100644 --- a/scripts/deployment/deploy_to_gcp_vertex_ai.py +++ b/scripts/deployment/deploy_to_gcp_vertex_ai.py @@ -20,7 +20,10 @@ def check_prerequisites(): # Check if gcloud is installed try: - result = subprocess.run(['gcloud', '--version'], capture_output=True, text=True) + result = subprocess.run( + ['gcloud', '--version'], + check=False, capture_output=True, text=True + ) if result.returncode == 0: print("โœ… gcloud CLI is installed") else: @@ -33,7 +36,10 @@ def check_prerequisites(): # Check if user is authenticated try: - result = subprocess.run(['gcloud', 'auth', 'list', '--filter=status:ACTIVE'], capture_output=True, text=True) + result = subprocess.run( + ['gcloud', 'auth', 'list', '--filter=status:ACTIVE'], + check=False, capture_output=True, text=True + ) if result.returncode == 0 and 'ACTIVE' in result.stdout: print("โœ… User is authenticated with gcloud") else: @@ -46,7 +52,10 @@ def check_prerequisites(): # Check if project is set try: - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True) + result = subprocess.run( + ['gcloud', 'config', 'get-value', 'project'], + check=False, capture_output=True, text=True + ) if result.returncode == 0 and result.stdout.strip(): project_id = result.stdout.strip() print(f"โœ… Project is set: {project_id}") @@ -60,7 +69,10 @@ def check_prerequisites(): # Check if Vertex AI API is enabled try: - result = subprocess.run(['gcloud', 'services', 'list', '--enabled', '--filter=name:aiplatform.googleapis.com'], capture_output=True, text=True) + result = subprocess.run( + ['gcloud', 'services', 'list', '--enabled', '--filter=name:aiplatform.googleapis.com'], + check=False, capture_output=True, text=True + ) if result.returncode == 0 and 'aiplatform.googleapis.com' in result.stdout: print("โœ… Vertex AI API is enabled") else: @@ -102,7 +114,7 @@ def prepare_model_for_deployment(): # Read model metadata metadata_path = os.path.join(default_model_path, "model_metadata.json") if os.path.exists(metadata_path): - with open(metadata_path, 'r') as f: + with open(metadata_path) as f: metadata = json.load(f) print(f"โœ… Model metadata: {metadata.get('version', 'Unknown')}") print(f" Performance: {metadata.get('performance', {}).get('test_accuracy', 'Unknown')}") @@ -305,7 +317,10 @@ def deploy_to_vertex_ai(deployment_dir): print("=" * 50) # Get project ID - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True) + result = subprocess.run( + ['gcloud', 'config', 'get-value', 'project'], + check=False, capture_output=True, text=True + ) project_id = result.stdout.strip() # Set region @@ -315,7 +330,7 @@ def deploy_to_vertex_ai(deployment_dir): model_name = "comprehensive-emotion-detection" endpoint_name = "emotion-detection-endpoint" - print(f"๐Ÿ“‹ Deployment Configuration:") + print("๐Ÿ“‹ Deployment Configuration:") print(f" Project ID: {project_id}") print(f" Region: {region}") print(f" Model Name: {model_name}") @@ -423,7 +438,7 @@ def deploy_to_vertex_ai(deployment_dir): print(f"โŒ Error deploying model: {e}") return False - print(f"\n๐ŸŽ‰ DEPLOYMENT COMPLETE!") + print("\n๐ŸŽ‰ DEPLOYMENT COMPLETE!") print(f"๐Ÿ“‹ Endpoint ID: {endpoint_id}") print(f"๐ŸŒ Region: {region}") print(f"๐Ÿค– Model: {model_name}") @@ -484,4 +499,4 @@ def main(): if __name__ == "__main__": success = main() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/deployment/fix_model_loading_issues.py b/scripts/deployment/fix_model_loading_issues.py index 17ccf9ec3..fce39c1e1 100644 --- a/scripts/deployment/fix_model_loading_issues.py +++ b/scripts/deployment/fix_model_loading_issues.py @@ -304,4 +304,4 @@ def main(): print("4. Monitor logs for any remaining issues") if __name__ == "__main__": - main() + main() diff --git a/scripts/deployment/hf_upload/config_update.py b/scripts/deployment/hf_upload/config_update.py index 3458484bf..d26afa04c 100644 --- a/scripts/deployment/hf_upload/config_update.py +++ b/scripts/deployment/hf_upload/config_update.py @@ -6,7 +6,7 @@ def _read(path: str) -> str: - with open(path, 'r') as f: + with open(path) as f: return f.read() diff --git a/scripts/deployment/hf_upload/prepare.py b/scripts/deployment/hf_upload/prepare.py index 0140559fa..23e719609 100644 --- a/scripts/deployment/hf_upload/prepare.py +++ b/scripts/deployment/hf_upload/prepare.py @@ -12,7 +12,7 @@ def _render_template(path: str, context: Dict[str, Any]) -> str: - with open(path, 'r') as f: + with open(path) as f: raw = f.read() # Simple $var substitution return Template(raw).safe_substitute(**context) @@ -24,7 +24,7 @@ def load_emotion_labels_from_model(model_path: str) -> List[str]: config_path = os.path.join(model_path, "config.json") if os.path.exists(config_path): try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = json.load(f) if 'id2label' in config: id2label = config['id2label'] @@ -64,7 +64,7 @@ def load_emotion_labels_from_model(model_path: str) -> List[str]: labels_path = os.path.join(model_dir, name) if os.path.exists(labels_path): try: - with open(labels_path, 'r') as f: + with open(labels_path) as f: data = json.load(f) if isinstance(data, list): logging.info("Loaded %d labels from %s", len(data), labels_path) @@ -163,7 +163,7 @@ def prepare_model_for_upload( # requirements.txt from template req_path = os.path.join(templates_dir, 'requirements_model.txt.tmpl') - with open(req_path, 'r') as f: + with open(req_path) as f: requirements = f.read() with open(os.path.join(temp_dir, 'requirements.txt'), 'w') as f: f.write(requirements) @@ -183,7 +183,7 @@ def prepare_model_for_upload( config_json = os.path.join(temp_dir, 'config.json') if os.path.exists(config_json): try: - with open(config_json, 'r') as f: + with open(config_json) as f: cfg = json.load(f) if 'id2label' not in cfg or 'label2id' not in cfg: logging.warning("config.json missing id2label/label2id mappings") diff --git a/scripts/deployment/hf_upload/upload.py b/scripts/deployment/hf_upload/upload.py index 857c7d582..69934e5fa 100644 --- a/scripts/deployment/hf_upload/upload.py +++ b/scripts/deployment/hf_upload/upload.py @@ -64,7 +64,7 @@ def setup_git_lfs() -> bool: # Update .gitattributes if exists gitattributes_path = ".gitattributes" if os.path.exists(gitattributes_path): - with open(gitattributes_path, 'r') as f: + with open(gitattributes_path) as f: content = f.read() for pattern in lfs_patterns: lfs_line = f"{pattern} filter=lfs diff=lfs merge=lfs -text" diff --git a/scripts/deployment/integrate_security_fixes.py b/scripts/deployment/integrate_security_fixes.py index e579d9b9b..733e11194 100644 --- a/scripts/deployment/integrate_security_fixes.py +++ b/scripts/deployment/integrate_security_fixes.py @@ -17,7 +17,7 @@ import time import requests from pathlib import Path -from typing import Dict, List, Optional +from typing import List class IntegratedSecurityOptimization: def __init__(self): @@ -31,7 +31,7 @@ def __init__(self): def get_project_id(): """Get current GCP project ID dynamically""" try: - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], + result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True, check=True) return result.stdout.strip() except subprocess.CalledProcessError: @@ -288,9 +288,9 @@ def run(self): self.log(f" gcloud run services describe {self.service_name} --region={self.region} --format='value(status.url)'") except Exception as e: - self.log(f"โŒ Integration failed: {str(e)}", "ERROR") + self.log(f"โŒ Integration failed: {e!s}", "ERROR") raise if __name__ == "__main__": integrator = IntegratedSecurityOptimization() - integrator.run() + integrator.run() diff --git a/scripts/deployment/save_trained_model_for_deployment.py b/scripts/deployment/save_trained_model_for_deployment.py index 8ef6a37d4..e61b58359 100644 --- a/scripts/deployment/save_trained_model_for_deployment.py +++ b/scripts/deployment/save_trained_model_for_deployment.py @@ -102,10 +102,10 @@ def save_model_for_deployment(): print("โœ… Model saved successfully!") print(f"๐Ÿ“ Deployment directory: {deployment_model_dir}") - print(f"๐Ÿ“Š Model info:") + print("๐Ÿ“Š Model info:") print(f" - Emotions: {len(emotions)} classes") - print(f" - F1 Score: 99.48%") - print(f" - Target Achieved: โœ… YES!") + print(" - F1 Score: 99.48%") + print(" - Target Achieved: โœ… YES!") # Test the saved model print("๐Ÿงช Testing saved model...") @@ -215,4 +215,4 @@ def create_deployment_script(): print("๐Ÿ† Target Achieved: โœ… YES!") else: print("\nโŒ Failed to create deployment package!") - print("Please ensure you have a trained model available.") \ No newline at end of file + print("Please ensure you have a trained model available.") diff --git a/scripts/deployment/security_deployment_fix.py b/scripts/deployment/security_deployment_fix.py index f133d76f7..9baa23199 100644 --- a/scripts/deployment/security_deployment_fix.py +++ b/scripts/deployment/security_deployment_fix.py @@ -18,13 +18,13 @@ import time import requests from pathlib import Path -from typing import Dict, List, Optional +from typing import List # Configuration def get_project_id(): """Get current GCP project ID dynamically""" try: - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], + result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True, check=True) return result.stdout.strip() except subprocess.CalledProcessError: @@ -162,7 +162,7 @@ def build_and_deploy(self): # Build container self.log("Building secure container...") build_result = self.run_command([ - 'gcloud', 'builds', 'submit', + 'gcloud', 'builds', 'submit', str(self.deployment_dir), '--config', str(cloudbuild_path) ]) @@ -325,4 +325,4 @@ def run(self): if __name__ == "__main__": fixer = SecurityDeploymentFix() success = fixer.run() - sys.exit(0 if success else 1) + sys.exit(0 if success else 1) diff --git a/scripts/deployment/vertex_ai_phase4_automation.py b/scripts/deployment/vertex_ai_phase4_automation.py index 84302b9e4..ccb7ae557 100644 --- a/scripts/deployment/vertex_ai_phase4_automation.py +++ b/scripts/deployment/vertex_ai_phase4_automation.py @@ -20,7 +20,7 @@ import sys import logging from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Dict from dataclasses import dataclass # Configure logging @@ -97,7 +97,7 @@ def _check_gcloud() -> bool: def _check_authentication() -> bool: """Check if user is authenticated.""" try: - result = subprocess.run(['gcloud', 'auth', 'list', '--filter=status:ACTIVE'], + result = subprocess.run(['gcloud', 'auth', 'list', '--filter=status:ACTIVE'], capture_output=True, text=True, check=True) return result.returncode == 0 and 'ACTIVE' in result.stdout except Exception: @@ -106,7 +106,7 @@ def _check_authentication() -> bool: def _check_project(self) -> bool: """Check if project is properly configured.""" try: - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], + result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True, check=True) return result.returncode == 0 and result.stdout.strip() == self.config.project_id except Exception: @@ -116,8 +116,8 @@ def _check_project(self) -> bool: def _check_vertex_ai_api() -> bool: """Check if Vertex AI API is enabled.""" try: - result = subprocess.run(['gcloud', 'services', 'list', '--enabled', - '--filter=name:aiplatform.googleapis.com'], + result = subprocess.run(['gcloud', 'services', 'list', '--enabled', + '--filter=name:aiplatform.googleapis.com'], capture_output=True, text=True, check=True) return result.returncode == 0 and 'aiplatform.googleapis.com' in result.stdout except Exception: @@ -127,8 +127,8 @@ def _check_vertex_ai_api() -> bool: def _check_monitoring_api() -> bool: """Check if Cloud Monitoring API is enabled.""" try: - result = subprocess.run(['gcloud', 'services', 'list', '--enabled', - '--filter=name:monitoring.googleapis.com'], + result = subprocess.run(['gcloud', 'services', 'list', '--enabled', + '--filter=name:monitoring.googleapis.com'], capture_output=True, text=True, check=True) return result.returncode == 0 and 'monitoring.googleapis.com' in result.stdout except Exception: @@ -138,8 +138,8 @@ def _check_monitoring_api() -> bool: def _check_logging_api() -> bool: """Check if Cloud Logging API is enabled.""" try: - result = subprocess.run(['gcloud', 'services', 'list', '--enabled', - '--filter=name:logging.googleapis.com'], + result = subprocess.run(['gcloud', 'services', 'list', '--enabled', + '--filter=name:logging.googleapis.com'], capture_output=True, text=True, check=True) return result.returncode == 0 and 'logging.googleapis.com' in result.stdout except Exception: @@ -149,8 +149,8 @@ def _check_logging_api() -> bool: def _check_artifact_registry() -> bool: """Check if Artifact Registry is enabled.""" try: - result = subprocess.run(['gcloud', 'services', 'list', '--enabled', - '--filter=name:artifactregistry.googleapis.com'], + result = subprocess.run(['gcloud', 'services', 'list', '--enabled', + '--filter=name:artifactregistry.googleapis.com'], capture_output=True, text=True, check=True) return result.returncode == 0 and 'artifactregistry.googleapis.com' in result.stdout except Exception: @@ -166,11 +166,11 @@ def _check_iam_permissions(self) -> bool: ] try: - result = subprocess.run(['gcloud', 'projects', 'get-iam-policy', self.config.project_id, - '--flatten=bindings[].members', - '--format=value(bindings.role)'], + result = subprocess.run(['gcloud', 'projects', 'get-iam-policy', self.config.project_id, + '--flatten=bindings[].members', + '--format=value(bindings.role)'], capture_output=True, text=True, check=True) - user_email = subprocess.run(['gcloud', 'config', 'get-value', 'account'], + user_email = subprocess.run(['gcloud', 'config', 'get-value', 'account'], capture_output=True, text=True, check=True).stdout.strip(check=True) user_roles = result.stdout.split('\n') @@ -184,7 +184,7 @@ def generate_model_version(self) -> str: # Get git commit hash if available try: - result = subprocess.run(['git', 'rev-parse', '--short', 'HEAD'], + result = subprocess.run(['git', 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True, check=True) git_hash = result.stdout.strip() if result.returncode == 0 else "unknown" except Exception: @@ -649,8 +649,8 @@ def cleanup_old_versions(self, keep_versions: int = 3) -> None: # Sort by deployment time and keep only the latest versions sorted_deployments = sorted( - self.deployment_history, - key=lambda x: x["deployed_at"], + self.deployment_history, + key=lambda x: x["deployed_at"], reverse=True ) @@ -749,7 +749,7 @@ def main(): # Get project ID try: - result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], + result = subprocess.run(['gcloud', 'config', 'get-value', 'project'], capture_output=True, text=True, check=True) project_id = result.stdout.strip() except Exception: @@ -790,4 +790,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() + main() diff --git a/scripts/legacy/add_comprehensive_features.py b/scripts/legacy/add_comprehensive_features.py index a4fc9c308..533f97c07 100644 --- a/scripts/legacy/add_comprehensive_features.py +++ b/scripts/legacy/add_comprehensive_features.py @@ -13,7 +13,7 @@ def add_comprehensive_features(): """Add all advanced features to the comprehensive notebook.""" # Read the existing notebook - with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Add all the advanced features as new cells @@ -559,4 +559,4 @@ def add_comprehensive_features(): print('\\n๐Ÿš€ COMPREHENSIVE NOTEBOOK IS NOW COMPLETE!') if __name__ == "__main__": - add_comprehensive_features() \ No newline at end of file + add_comprehensive_features() diff --git a/scripts/legacy/add_wandb_setup.py b/scripts/legacy/add_wandb_setup.py index 35c8bb753..ddce4791b 100644 --- a/scripts/legacy/add_wandb_setup.py +++ b/scripts/legacy/add_wandb_setup.py @@ -13,7 +13,7 @@ def add_wandb_setup(): """Add wandb setup to the minimal notebook.""" # Read the existing notebook - with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Add wandb setup cell after the imports @@ -149,4 +149,4 @@ def add_wandb_setup(): print('3. Restart runtime and run the notebook') if __name__ == "__main__": - add_wandb_setup() \ No newline at end of file + add_wandb_setup() diff --git a/scripts/legacy/comprehensive_model_validation.py b/scripts/legacy/comprehensive_model_validation.py index 61aecd9a7..04ffa3947 100644 --- a/scripts/legacy/comprehensive_model_validation.py +++ b/scripts/legacy/comprehensive_model_validation.py @@ -24,7 +24,7 @@ def comprehensive_validation(): model_dir = Path(__file__).parent.parent / 'deployment' / 'model' required_files = ['config.json', 'model.safetensors', 'training_args.bin'] - print(f"\n๐Ÿ“ MODEL FILE VALIDATION") + print("\n๐Ÿ“ MODEL FILE VALIDATION") print("-" * 40) missing_files = [] @@ -41,13 +41,13 @@ def comprehensive_validation(): print(f"\nโŒ CRITICAL: Missing files: {missing_files}") return False - print(f"โœ… All model files present and valid") + print("โœ… All model files present and valid") # Load model configuration - print(f"\n๐Ÿ”ง MODEL CONFIGURATION VALIDATION") + print("\n๐Ÿ”ง MODEL CONFIGURATION VALIDATION") print("-" * 40) - with open(model_dir / 'config.json', 'r') as f: + with open(model_dir / 'config.json') as f: config = json.load(f) print(f"Model Type: {config.get('model_type', 'unknown')}") @@ -61,7 +61,7 @@ def comprehensive_validation(): print(f"Emotion Classes: {len(emotion_mapping)}") # Load model and tokenizer - print(f"\n๐Ÿ”ง MODEL LOADING VALIDATION") + print("\n๐Ÿ”ง MODEL LOADING VALIDATION") print("-" * 40) try: @@ -81,11 +81,11 @@ def comprehensive_validation(): print(f"โœ… Model moved to {device}") except Exception as e: - print(f"โŒ Model loading failed: {str(e)}") + print(f"โŒ Model loading failed: {e!s}") return False # Test 1: Basic Functionality - print(f"\n๐Ÿงช TEST 1: BASIC FUNCTIONALITY") + print("\n๐Ÿงช TEST 1: BASIC FUNCTIONALITY") print("-" * 40) test_cases = [ @@ -131,11 +131,11 @@ def comprehensive_validation(): print(f"{status} '{text}' โ†’ {predicted_emotion} (expected: {expected_emotion}, confidence: {confidence:.3f})") except Exception as e: - print(f"โŒ Error predicting '{text}': {str(e)}") + print(f"โŒ Error predicting '{text}': {e!s}") return False accuracy = correct_predictions / total_predictions - print(f"\n๐Ÿ“Š Basic Functionality Results:") + print("\n๐Ÿ“Š Basic Functionality Results:") print(f" Correct: {correct_predictions}/{total_predictions}") print(f" Accuracy: {accuracy:.1%}") @@ -144,7 +144,7 @@ def comprehensive_validation(): return False # Test 2: Confidence Distribution - print(f"\n๐Ÿงช TEST 2: CONFIDENCE DISTRIBUTION") + print("\n๐Ÿงช TEST 2: CONFIDENCE DISTRIBUTION") print("-" * 40) confidence_scores = [] @@ -170,7 +170,7 @@ def comprehensive_validation(): print(f"โš ๏ธ WARNING: Low average confidence ({avg_confidence:.3f})") # Test 3: Edge Cases - print(f"\n๐Ÿงช TEST 3: EDGE CASES") + print("\n๐Ÿงช TEST 3: EDGE CASES") print("-" * 40) edge_cases = [ @@ -201,12 +201,12 @@ def comprehensive_validation(): print(f"โœ… Edge case handled: '{text[:30]}...' โ†’ {predicted_emotion} ({confidence:.3f})") except Exception as e: - print(f"โŒ Edge case failed: '{text[:30]}...' - {str(e)}") + print(f"โŒ Edge case failed: '{text[:30]}...' - {e!s}") print(f"\n๐Ÿ“Š Edge Case Results: {edge_case_success}/{len(edge_cases)} successful") # Test 4: Performance Benchmark - print(f"\n๐Ÿงช TEST 4: PERFORMANCE BENCHMARK") + print("\n๐Ÿงช TEST 4: PERFORMANCE BENCHMARK") print("-" * 40) benchmark_text = "I'm feeling really happy today!" @@ -233,7 +233,7 @@ def comprehensive_validation(): print(f"โš ๏ธ WARNING: Slow inference time ({avg_time:.4f}s)") # Test 5: Consistency Check - print(f"\n๐Ÿงช TEST 5: CONSISTENCY CHECK") + print("\n๐Ÿงช TEST 5: CONSISTENCY CHECK") print("-" * 40) consistency_text = "I'm feeling happy today!" @@ -263,7 +263,7 @@ def comprehensive_validation(): return False # Final Validation Summary - print(f"\n๐ŸŽฏ FINAL VALIDATION SUMMARY") + print("\n๐ŸŽฏ FINAL VALIDATION SUMMARY") print("=" * 60) validation_results = { @@ -284,13 +284,13 @@ def comprehensive_validation(): print(f"\n{'๐ŸŽ‰ ALL TESTS PASSED!' if all_passed else 'โŒ SOME TESTS FAILED'}") if all_passed: - print(f"โœ… Your 99.54% F1 score model is 100% RELIABLE!") - print(f"๐Ÿš€ Ready for production deployment!") + print("โœ… Your 99.54% F1 score model is 100% RELIABLE!") + print("๐Ÿš€ Ready for production deployment!") else: - print(f"โš ๏ธ Model needs further validation before deployment") + print("โš ๏ธ Model needs further validation before deployment") return all_passed if __name__ == "__main__": success = comprehensive_validation() - exit(0 if success else 1) \ No newline at end of file + exit(0 if success else 1) diff --git a/scripts/legacy/create_bulletproof_cell.py b/scripts/legacy/create_bulletproof_cell.py index 4fa79be07..184f0e949 100644 --- a/scripts/legacy/create_bulletproof_cell.py +++ b/scripts/legacy/create_bulletproof_cell.py @@ -406,4 +406,4 @@ def forward(self, input_ids, attention_mask): print("6. This will work in a fresh kernel without any state corruption!") if __name__ == "__main__": - create_bulletproof_cell() \ No newline at end of file + create_bulletproof_cell() diff --git a/scripts/legacy/create_final_bulletproof_cell.py b/scripts/legacy/create_final_bulletproof_cell.py index 499fb7be0..3953ccb14 100644 --- a/scripts/legacy/create_final_bulletproof_cell.py +++ b/scripts/legacy/create_final_bulletproof_cell.py @@ -442,4 +442,4 @@ def forward(self, input_ids, attention_mask): print("๐ŸŽฏ This will solve the zero samples issue!") if __name__ == "__main__": - create_final_bulletproof_cell() \ No newline at end of file + create_final_bulletproof_cell() diff --git a/scripts/legacy/create_unique_fallback_dataset.py b/scripts/legacy/create_unique_fallback_dataset.py index 8386cc31d..b15f94a1d 100644 --- a/scripts/legacy/create_unique_fallback_dataset.py +++ b/scripts/legacy/create_unique_fallback_dataset.py @@ -234,4 +234,4 @@ def create_unique_fallback_dataset(): print("๐Ÿš€ CREATE UNIQUE FALLBACK DATASET") print("=" * 40) create_unique_fallback_dataset() - print("\n๐ŸŽ‰ Unique fallback dataset created successfully!") \ No newline at end of file + print("\n๐ŸŽ‰ Unique fallback dataset created successfully!") diff --git a/scripts/legacy/deep_model_analysis.py b/scripts/legacy/deep_model_analysis.py index c1683680a..997a71b26 100644 --- a/scripts/legacy/deep_model_analysis.py +++ b/scripts/legacy/deep_model_analysis.py @@ -28,14 +28,14 @@ def deep_model_analysis(): # Define emotion mapping emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - print(f"\n๐Ÿ“Š EMOTION MAPPING ANALYSIS") + print("\n๐Ÿ“Š EMOTION MAPPING ANALYSIS") print("-" * 40) print("Current mapping (LABEL_0 to LABEL_11):") for i, emotion in enumerate(emotion_mapping): print(f" LABEL_{i} โ†’ {emotion}") # Test with different variations - print(f"\n๐Ÿงช DETAILED PREDICTION ANALYSIS") + print("\n๐Ÿงช DETAILED PREDICTION ANALYSIS") print("-" * 40) test_cases = [ @@ -62,7 +62,7 @@ def deep_model_analysis(): # Get top 3 predictions top_probs, top_indices = torch.topk(probabilities[0], 3) - print(f"๐Ÿ” Top 3 predictions:") + print("๐Ÿ” Top 3 predictions:") for i, (prob, idx) in enumerate(zip(top_probs, top_indices)): emotion = emotion_mapping[idx.item()] print(f" {i+1}. {emotion}: {prob.item():.3f}") @@ -73,7 +73,7 @@ def deep_model_analysis(): print(f"๐Ÿ“Š Expected emotion '{expected_emotion}' probability: {expected_prob:.3f}") # Analyze model confidence patterns - print(f"\n๐Ÿ“ˆ CONFIDENCE PATTERN ANALYSIS") + print("\n๐Ÿ“ˆ CONFIDENCE PATTERN ANALYSIS") print("-" * 40) confidence_by_emotion = {emotion: [] for emotion in emotion_mapping} @@ -99,7 +99,7 @@ def deep_model_analysis(): print(f"'{word}' โ†’ {predicted_emotion} (confidence: {confidence:.3f})") # Check for bias towards certain emotions - print(f"\n๐ŸŽฏ EMOTION BIAS ANALYSIS") + print("\n๐ŸŽฏ EMOTION BIAS ANALYSIS") print("-" * 40) emotion_counts = {} @@ -118,7 +118,7 @@ def deep_model_analysis(): print(f"โŒ WARNING: Model shows bias towards '{most_common[0]}'") # Test with training-like data - print(f"\n๐ŸŽ“ TRAINING-LIKE DATA TEST") + print("\n๐ŸŽ“ TRAINING-LIKE DATA TEST") print("-" * 40) # These should be more similar to what the model was trained on @@ -171,20 +171,20 @@ def deep_model_analysis(): print(f"\n๐Ÿ“Š Training-like accuracy: {training_like_accuracy:.1%}") # Final analysis - print(f"\n๐Ÿ” ANALYSIS SUMMARY") + print("\n๐Ÿ” ANALYSIS SUMMARY") print("=" * 50) if training_like_accuracy > 0.8: print(f"โœ… Model performs well on training-like data ({training_like_accuracy:.1%})") - print(f"โš ๏ธ Issue: Model may be overfitting to specific training patterns") - print(f"๐Ÿ’ก Solution: Model needs more diverse training data or regularization") + print("โš ๏ธ Issue: Model may be overfitting to specific training patterns") + print("๐Ÿ’ก Solution: Model needs more diverse training data or regularization") else: print(f"โŒ Model performs poorly even on training-like data ({training_like_accuracy:.1%})") - print(f"โš ๏ธ Issue: Fundamental problem with model training or label mapping") - print(f"๐Ÿ’ก Solution: Retrain model with better data or check label mapping") + print("โš ๏ธ Issue: Fundamental problem with model training or label mapping") + print("๐Ÿ’ก Solution: Retrain model with better data or check label mapping") return training_like_accuracy > 0.8 if __name__ == "__main__": success = deep_model_analysis() - exit(0 if success else 1) \ No newline at end of file + exit(0 if success else 1) diff --git a/scripts/legacy/evaluate_whisper_wer.py b/scripts/legacy/evaluate_whisper_wer.py index 453cc96ce..f1f64dd56 100644 --- a/scripts/legacy/evaluate_whisper_wer.py +++ b/scripts/legacy/evaluate_whisper_wer.py @@ -166,25 +166,25 @@ def main(): """Main evaluation function.""" parser = argparse.ArgumentParser(description="Evaluate Whisper WER on LibriSpeech") parser.add_argument( - "--output-dir", - type=str, + "--output-dir", + type=str, help="Directory to save results and audio files" ) parser.add_argument( - "--max-samples", - type=int, - default=50, + "--max-samples", + type=int, + default=50, help="Maximum number of samples to evaluate" ) parser.add_argument( - "--model-size", - type=str, - default="base", + "--model-size", + type=str, + default="base", help="Whisper model size (tiny, base, small, medium, large)" ) parser.add_argument( - "--save-results", - action="store_true", + "--save-results", + action="store_true", help="Save detailed results to JSON file" ) @@ -199,7 +199,7 @@ def main(): # Download or load LibriSpeech samples samples = download_librispeech_sample( - output_dir=args.output_dir, + output_dir=args.output_dir, max_samples=args.max_samples ) diff --git a/scripts/legacy/expand_journal_dataset.py b/scripts/legacy/expand_journal_dataset.py index 0786d8f7d..f318843f0 100644 --- a/scripts/legacy/expand_journal_dataset.py +++ b/scripts/legacy/expand_journal_dataset.py @@ -5,11 +5,11 @@ import json import random -from typing import List, Dict +from typing import Dict def load_current_dataset(): """Load the current journal dataset.""" - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: return json.load(f) def save_expanded_dataset(data, filename='data/expanded_journal_dataset.json'): @@ -31,7 +31,7 @@ def create_balanced_dataset(target_size=1000): emotion = entry['emotion'] emotion_counts[emotion] = emotion_counts.get(emotion, 0) + 1 - print(f"๐Ÿ“Š Current emotion distribution:") + print("๐Ÿ“Š Current emotion distribution:") for emotion, count in sorted(emotion_counts.items()): print(f" {emotion}: {count} samples") @@ -42,7 +42,7 @@ def create_balanced_dataset(target_size=1000): # Create expanded dataset expanded_data = [] - for emotion in emotion_counts.keys(): + for emotion in emotion_counts: # Get existing samples for this emotion existing_samples = [entry for entry in current_data if entry['emotion'] == emotion] current_count = len(existing_samples) @@ -65,7 +65,7 @@ def create_balanced_dataset(target_size=1000): variation = create_variation(base_sample, emotion) expanded_data.append(variation) - print(f"\nโœ… Expanded dataset created:") + print("\nโœ… Expanded dataset created:") print(f" Original samples: {len(current_data)}") print(f" Expanded samples: {len(expanded_data)}") print(f" Target size: {target_size}") @@ -282,4 +282,4 @@ def main(): print(" 3. Expect 75-85% F1 score!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/legacy/finalize_emotion_model.py b/scripts/legacy/finalize_emotion_model.py index 014101800..17f3c0019 100755 --- a/scripts/legacy/finalize_emotion_model.py +++ b/scripts/legacy/finalize_emotion_model.py @@ -362,7 +362,7 @@ def save_ensemble_model( 'threshold': ensemble.threshold, }, output_path) - logger.info(f"Model saved successfully!") + logger.info("Model saved successfully!") logger.info(f"Final metrics: {metrics}") diff --git a/scripts/legacy/integrate_cmu_mosei.py b/scripts/legacy/integrate_cmu_mosei.py index 686b0c743..2d12548c4 100644 --- a/scripts/legacy/integrate_cmu_mosei.py +++ b/scripts/legacy/integrate_cmu_mosei.py @@ -50,7 +50,7 @@ def download_cmu_mosei(): valid_ids = mosei.valid() test_ids = mosei.test() - print(f"โœ… CMU-MOSEI downloaded successfully!") + print("โœ… CMU-MOSEI downloaded successfully!") print(f"๐Ÿ“Š Train videos: {len(train_ids)}") print(f"๐Ÿ“Š Validation videos: {len(valid_ids)}") print(f"๐Ÿ“Š Test videos: {len(test_ids)}") @@ -102,7 +102,7 @@ def map_sentiment_to_emotions(samples): emotion_mapping = { # Very negative sentiments (-3, -2.5): 'sad', - (-2.5, -2): 'frustrated', + (-2.5, -2): 'frustrated', (-2, -1.5): 'anxious', (-1.5, -1): 'tired', (-1, -0.5): 'overwhelmed', @@ -229,4 +229,4 @@ def main(): print(" 3. Upload to Colab and achieve 75-85% F1 score!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/legacy/reorganize_model_directory.py b/scripts/legacy/reorganize_model_directory.py index eaf859d7a..55aa57cc9 100644 --- a/scripts/legacy/reorganize_model_directory.py +++ b/scripts/legacy/reorganize_model_directory.py @@ -32,7 +32,7 @@ def reorganize_model_directory(): print(f"โœ… Created models directory: {models_dir}") # 1. Save current model as model_1 (fallback) - print(f"\n๐Ÿ’พ SAVING CURRENT MODEL AS FALLBACK") + print("\n๐Ÿ’พ SAVING CURRENT MODEL AS FALLBACK") print("-" * 40) if os.path.exists(current_model_path): @@ -77,7 +77,7 @@ def reorganize_model_directory(): return # 2. Create default model directory structure - print(f"\n๐Ÿ“‚ CREATING DEFAULT MODEL STRUCTURE") + print("\n๐Ÿ“‚ CREATING DEFAULT MODEL STRUCTURE") print("-" * 40) if os.path.exists(default_model_path): @@ -121,7 +121,7 @@ def reorganize_model_directory(): print(f"โœ… Created default model metadata: {default_metadata_path}") # 3. Create models index file - print(f"\n๐Ÿ“‹ CREATING MODELS INDEX") + print("\n๐Ÿ“‹ CREATING MODELS INDEX") print("-" * 40) models_index = { @@ -152,7 +152,7 @@ def reorganize_model_directory(): print(f"โœ… Created models index: {index_path}") # 4. Create README for models directory - print(f"\n๐Ÿ“– CREATING MODELS README") + print("\n๐Ÿ“– CREATING MODELS README") print("-" * 40) readme_content = """# Model Versions @@ -229,7 +229,7 @@ def reorganize_model_directory(): print(f"โœ… Created models README: {readme_path}") # 5. Create symlink for easy access - print(f"\n๐Ÿ”— CREATING SYMLINKS") + print("\n๐Ÿ”— CREATING SYMLINKS") print("-" * 40) # Create symlink from deployment/model to default model @@ -254,17 +254,17 @@ def reorganize_model_directory(): print(f" You can manually link {symlink_path} to {default_model_path}") # 6. Summary - print(f"\n๐Ÿ“‹ REORGANIZATION SUMMARY") + print("\n๐Ÿ“‹ REORGANIZATION SUMMARY") print("=" * 50) print("โœ… Model directory reorganized successfully!") print() print("๐Ÿ“ New Structure:") print(f" {models_dir}/") - print(f" โ”œโ”€โ”€ model_1_fallback/ # Your working model (91.67% accuracy)") - print(f" โ”œโ”€โ”€ default/ # Ready for comprehensive model") - print(f" โ”œโ”€โ”€ models_index.json # Model registry") - print(f" โ””โ”€โ”€ README.md # Documentation") + print(" โ”œโ”€โ”€ model_1_fallback/ # Your working model (91.67% accuracy)") + print(" โ”œโ”€โ”€ default/ # Ready for comprehensive model") + print(" โ”œโ”€โ”€ models_index.json # Model registry") + print(" โ””โ”€โ”€ README.md # Documentation") print() print("๐ŸŽฏ Next Steps:") print(" 1. Train the comprehensive model using COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb") @@ -278,4 +278,4 @@ def reorganize_model_directory(): print(" - Clear versioning and documentation") if __name__ == "__main__": - reorganize_model_directory() \ No newline at end of file + reorganize_model_directory() diff --git a/scripts/legacy/retrain_with_expanded_dataset.py b/scripts/legacy/retrain_with_expanded_dataset.py index a2845206f..5cb8987c8 100644 --- a/scripts/legacy/retrain_with_expanded_dataset.py +++ b/scripts/legacy/retrain_with_expanded_dataset.py @@ -5,7 +5,7 @@ import json import torch -import torch.nn as nn +from torch import nn from torch.utils.data import Dataset, DataLoader from transformers import AutoModel, AutoTokenizer from sklearn.preprocessing import LabelEncoder @@ -16,7 +16,7 @@ def load_expanded_dataset(): """Load the expanded journal dataset.""" print("๐Ÿ“Š Loading expanded dataset...") - with open('data/expanded_journal_dataset.json', 'r') as f: + with open('data/expanded_journal_dataset.json') as f: data = json.load(f) print(f"โœ… Loaded {len(data)} samples") @@ -99,7 +99,7 @@ def prepare_expanded_data(data, test_size=0.2, val_size=0.1): X_temp, y_temp, test_size=val_size/(1-test_size), random_state=42, stratify=y_temp ) - print(f"๐Ÿ“Š Data split:") + print("๐Ÿ“Š Data split:") print(f" Training: {len(X_train)} samples") print(f" Validation: {len(X_val)} samples") print(f" Test: {len(X_test)} samples") @@ -263,7 +263,7 @@ def save_expanded_results(training_history, best_f1, label_encoder, test_data): with open('expanded_training_results.json', 'w') as f: json.dump(results, f, indent=2) - print(f"โœ… Results saved!") + print("โœ… Results saved!") print(f"๐Ÿ“Š Final F1 Score: {final_f1:.4f}") print(f"๐Ÿ“Š Final Accuracy: {final_accuracy:.4f}") print(f"๐ŸŽฏ Target Achieved: {final_f1 >= 0.70}") @@ -292,4 +292,4 @@ def main(): print(" 3. Deploy if target achieved!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/legacy/retrain_with_validation.py b/scripts/legacy/retrain_with_validation.py index 8710f134a..6f4decae1 100644 --- a/scripts/legacy/retrain_with_validation.py +++ b/scripts/legacy/retrain_with_validation.py @@ -14,14 +14,14 @@ def create_improved_training_plan(): print("๐ŸŽฏ Goal: Retrain model to achieve reliable 75-85% F1 score") print("=" * 50) - print(f"\nโŒ CURRENT ISSUES IDENTIFIED:") + print("\nโŒ CURRENT ISSUES IDENTIFIED:") print("-" * 40) print("1. Model bias towards 'grateful' and 'happy' emotions") print("2. Poor generalization (58.3% accuracy on basic tests)") print("3. Overfitting to specific training patterns") print("4. Label mapping inconsistencies") - print(f"\nโœ… IMPROVED TRAINING STRATEGY:") + print("\nโœ… IMPROVED TRAINING STRATEGY:") print("-" * 40) print("1. Use balanced dataset with equal emotion distribution") print("2. Implement proper cross-validation") @@ -29,7 +29,7 @@ def create_improved_training_plan(): print("4. Use early stopping based on validation performance") print("5. Test on diverse, realistic examples") - print(f"\n๐Ÿ“Š VALIDATION REQUIREMENTS:") + print("\n๐Ÿ“Š VALIDATION REQUIREMENTS:") print("-" * 40) print("โœ… Basic functionality test: >80% accuracy") print("โœ… Training-like data test: >80% accuracy") @@ -37,7 +37,7 @@ def create_improved_training_plan(): print("โœ… No emotion bias: <30% predictions for any single emotion") print("โœ… Consistent predictions: 100% consistency for same input") - print(f"\n๐Ÿš€ RECOMMENDED ACTIONS:") + print("\n๐Ÿš€ RECOMMENDED ACTIONS:") print("-" * 40) print("1. Create balanced training dataset") print("2. Implement proper validation split") @@ -389,13 +389,13 @@ def create_improved_notebook(): f.write(notebook_content) print(f"โœ… Created improved training notebook: {notebook_path}") - print(f"๐Ÿ“‹ Instructions:") - print(f" 1. Download the notebook file") - print(f" 2. Upload to Google Colab") - print(f" 3. Set Runtime โ†’ GPU") - print(f" 4. Run all cells") - print(f" 5. Verify reliability before deployment") + print("๐Ÿ“‹ Instructions:") + print(" 1. Download the notebook file") + print(" 2. Upload to Google Colab") + print(" 3. Set Runtime โ†’ GPU") + print(" 4. Run all cells") + print(" 5. Verify reliability before deployment") if __name__ == "__main__": success = create_improved_training_plan() - exit(0 if success else 1) \ No newline at end of file + exit(0 if success else 1) diff --git a/scripts/legacy/simple_cmu_mosei_download.py b/scripts/legacy/simple_cmu_mosei_download.py index 1723581c8..aaecfee37 100644 --- a/scripts/legacy/simple_cmu_mosei_download.py +++ b/scripts/legacy/simple_cmu_mosei_download.py @@ -103,7 +103,7 @@ def map_sentiment_to_emotions(samples): emotion_mapping = { # Very negative sentiments (-3, -2.5): 'sad', - (-2.5, -2): 'frustrated', + (-2.5, -2): 'frustrated', (-2, -1.5): 'anxious', (-1.5, -1): 'tired', (-1, -0.5): 'overwhelmed', @@ -225,4 +225,4 @@ def main(): print(" 3. Upload to Colab and achieve 75-85% F1 score!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/legacy/simple_f1_evaluation.py b/scripts/legacy/simple_f1_evaluation.py index 66e99ccc4..c4aa7b01c 100644 --- a/scripts/legacy/simple_f1_evaluation.py +++ b/scripts/legacy/simple_f1_evaluation.py @@ -186,4 +186,4 @@ def evaluate_current_f1(): logger.info("โœ… Evaluation completed successfully") else: logger.error("โŒ Evaluation failed") - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/scripts/legacy/validate_model_performance.py b/scripts/legacy/validate_model_performance.py index 1a0d10045..7d35d1c13 100644 --- a/scripts/legacy/validate_model_performance.py +++ b/scripts/legacy/validate_model_performance.py @@ -21,7 +21,7 @@ def load_model_and_tokenizer(model_path): model = AutoModelForSequenceClassification.from_pretrained(model_path) return tokenizer, model except Exception as e: - print(f"โŒ Error loading model: {str(e)}") + print(f"โŒ Error loading model: {e!s}") return None, None def check_model_configuration(model_path): @@ -30,7 +30,7 @@ def check_model_configuration(model_path): print("=" * 50) try: - with open(os.path.join(model_path, 'config.json'), 'r') as f: + with open(os.path.join(model_path, 'config.json')) as f: config = json.load(f) print(f"Model type: {config.get('model_type', 'NOT FOUND')}") @@ -59,7 +59,7 @@ def check_model_configuration(model_path): return False except Exception as e: - print(f"โŒ Error reading configuration: {str(e)}") + print(f"โŒ Error reading configuration: {e!s}") return False def create_test_dataset(): @@ -142,7 +142,7 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): device = next(model.parameters()).device results = [] - predictions_by_emotion = {emotion: 0 for emotion in emotions} + predictions_by_emotion = dict.fromkeys(emotions, 0) print("Testing on unseen examples...") print("-" * 50) @@ -188,14 +188,14 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): correct = sum(1 for r in results if r['correct']) accuracy = correct / len(results) - print(f"\n๐Ÿ“Š PERFORMANCE SUMMARY") + print("\n๐Ÿ“Š PERFORMANCE SUMMARY") print("=" * 30) print(f"Total examples: {len(results)}") print(f"Correct predictions: {correct}") print(f"Accuracy: {accuracy:.1%}") # Bias analysis - print(f"\n๐ŸŽฏ BIAS ANALYSIS") + print("\n๐ŸŽฏ BIAS ANALYSIS") print("=" * 20) for emotion, count in predictions_by_emotion.items(): percentage = count / len(results) * 100 @@ -204,7 +204,7 @@ def evaluate_model_performance(model, tokenizer, test_examples, emotions): # Determine if model is reliable max_bias = max(predictions_by_emotion.values()) / len(results) - print(f"\n๐Ÿ” RELIABILITY ASSESSMENT") + print("\n๐Ÿ” RELIABILITY ASSESSMENT") print("=" * 30) if accuracy >= 0.8 and max_bias <= 0.3: print("๐ŸŽ‰ MODEL PASSES RELIABILITY TEST!") @@ -289,7 +289,7 @@ def main(): training_data_path = "./data/balanced_training_data.json" if os.path.exists(training_data_path): try: - with open(training_data_path, 'r') as f: + with open(training_data_path) as f: training_data = json.load(f) data_leakage = check_for_data_leakage(training_data, test_examples) except: @@ -298,7 +298,7 @@ def main(): print("โš ๏ธ Training data not found, skipping data leakage check") # Summary - print(f"\n๐Ÿ“‹ VALIDATION SUMMARY") + print("\n๐Ÿ“‹ VALIDATION SUMMARY") print("=" * 30) print(f"Configuration correct: {'โœ…' if config_ok else 'โŒ'}") print(f"Accuracy on unseen data: {accuracy:.1%}") @@ -306,7 +306,7 @@ def main(): print(f"Model reliable: {'โœ…' if accuracy >= 0.8 and max_bias <= 0.3 else 'โŒ'}") if accuracy < 0.8: - print(f"\n๐Ÿ’ก RECOMMENDATIONS:") + print("\n๐Ÿ’ก RECOMMENDATIONS:") print("1. Increase training dataset size") print("2. Use data augmentation techniques") print("3. Try different model architectures") @@ -314,4 +314,4 @@ def main(): print("5. Use cross-validation for better evaluation") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/maintenance/auto_fix_code_quality.py b/scripts/maintenance/auto_fix_code_quality.py index b2d4d1948..768dfd5f1 100644 --- a/scripts/maintenance/auto_fix_code_quality.py +++ b/scripts/maintenance/auto_fix_code_quality.py @@ -16,7 +16,7 @@ import sys import re from pathlib import Path -from typing import Dict, List, Set, Tuple, Any, Optional +from typing import Dict, List, Tuple, Any import logging # Configure logging @@ -49,7 +49,7 @@ def fix_file(self, file_path: Path) -> Dict[str, Any]: logger.info("Fixing: %s", file_path) try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() original_content = content diff --git a/scripts/maintenance/code_quality_enforcer.py b/scripts/maintenance/code_quality_enforcer.py index 5255c3f20..34f0bc21b 100644 --- a/scripts/maintenance/code_quality_enforcer.py +++ b/scripts/maintenance/code_quality_enforcer.py @@ -23,7 +23,7 @@ import ast import re from pathlib import Path -from typing import Dict, List, Set, Tuple, Any, Optional +from typing import Dict, List, Any import logging # Configure logging @@ -122,7 +122,7 @@ def check_file(self, file_path: Path) -> List[Dict[str, Any]]: issues = [] try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() lines = content.splitlines() diff --git a/scripts/maintenance/emergency_f1_fix.py b/scripts/maintenance/emergency_f1_fix.py index 947b378ac..393929756 100644 --- a/scripts/maintenance/emergency_f1_fix.py +++ b/scripts/maintenance/emergency_f1_fix.py @@ -19,10 +19,10 @@ import numpy as np import torch -import torch.nn as nn +from torch import nn +from torch.utils.data import DataLoader, TensorDataset import torch.nn.functional as F from sklearn.metrics import f1_score -from torch.utils.data import DataLoader, TensorDataset from transformers import AutoTokenizer, get_linear_schedule_with_warmup # Add src to path @@ -162,7 +162,7 @@ def train_with_focal_loss(model, train_loader, val_loader, device, epochs=5): # Learning rate scheduler total_steps = len(train_loader) * epochs scheduler = get_linear_schedule_with_warmup( - optimizer, + optimizer, num_warmup_steps=total_steps // 10, num_training_steps=total_steps ) @@ -389,4 +389,4 @@ def emergency_f1_fix(): logger.info("โœ… Emergency F1 fix completed successfully") else: logger.error("โŒ Emergency F1 fix failed") - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/scripts/maintenance/fix_code_quality.py b/scripts/maintenance/fix_code_quality.py index 0ff64ea22..78ed56920 100644 --- a/scripts/maintenance/fix_code_quality.py +++ b/scripts/maintenance/fix_code_quality.py @@ -9,7 +9,6 @@ import logging import re from pathlib import Path -from typing import List, Set # Configure logging logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -112,7 +111,7 @@ def fix_missing_newlines(self, content: str) -> str: def fix_file(self, file_path: Path) -> bool: """Fix code quality issues in a single file.""" try: - with open(file_path, "r", encoding="utf-8") as f: + with open(file_path, encoding="utf-8") as f: content = f.read() original_content = content @@ -150,7 +149,7 @@ def fix_project(self) -> None: if self.fix_file(file_path): self.total_issues += 1 - logger.info(f"โœ… Code quality fixes completed!") + logger.info("โœ… Code quality fixes completed!") logger.info(f" โ€ข Files fixed: {self.fixed_files}") logger.info(f" โ€ข Total issues resolved: {self.total_issues}") diff --git a/scripts/maintenance/fix_import_paths.py b/scripts/maintenance/fix_import_paths.py index 7743b243a..ac454479f 100644 --- a/scripts/maintenance/fix_import_paths.py +++ b/scripts/maintenance/fix_import_paths.py @@ -9,7 +9,7 @@ def fix_import_paths_in_file(file_path): """Fix import paths in a single file.""" try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() original_content = content @@ -30,9 +30,9 @@ def fix_import_paths_in_file(file_path): (r'from \.\.data\.', 'from data.'), # Fix sys.path insertions - (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent / "src"\)\)', + (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent / "src"\)\)', 'sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))'), - (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent\.parent / "src"\)\)', + (r'sys\.path\.insert\(0, str\(Path\(__file__\)\.parent\.parent\.parent / "src"\)\)', 'sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src"))'), ] @@ -73,4 +73,4 @@ def main(): print("Import path fixes completed!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/maintenance/fix_label_mapping.py b/scripts/maintenance/fix_label_mapping.py index a7f8fcca8..ff95f0bad 100644 --- a/scripts/maintenance/fix_label_mapping.py +++ b/scripts/maintenance/fix_label_mapping.py @@ -32,7 +32,7 @@ def analyze_label_mapping(): # Load datasets go_emotions = load_dataset("go_emotions", "simplified") - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) @@ -526,4 +526,4 @@ def forward(self, input_ids, attention_mask): print("\n๐ŸŽฏ SUMMARY:") print("The issue was that GoEmotions uses emotion names (like 'admiration')") print("while Journal uses different emotion names (like 'proud').") - print("The fixed version maps GoEmotions emotions to Journal emotions!") \ No newline at end of file + print("The fixed version maps GoEmotions emotions to Journal emotions!") diff --git a/scripts/maintenance/fix_linting.py b/scripts/maintenance/fix_linting.py index c3566416e..65cfb9ca0 100644 --- a/scripts/maintenance/fix_linting.py +++ b/scripts/maintenance/fix_linting.py @@ -14,7 +14,7 @@ def fix_file(file_path: str) -> None: Args: file_path: Path to the file to fix """ - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() original_content = content diff --git a/scripts/maintenance/fix_linting_issues_conservative.py b/scripts/maintenance/fix_linting_issues_conservative.py index a57f3b45c..c3631b079 100644 --- a/scripts/maintenance/fix_linting_issues_conservative.py +++ b/scripts/maintenance/fix_linting_issues_conservative.py @@ -160,7 +160,7 @@ def fix_file(self, file_path: Path) -> bool: True if file was modified """ try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() original_content = content @@ -239,4 +239,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/maintenance/fix_model_architecture_mismatch.py b/scripts/maintenance/fix_model_architecture_mismatch.py index bbfb75756..36ba10470 100644 --- a/scripts/maintenance/fix_model_architecture_mismatch.py +++ b/scripts/maintenance/fix_model_architecture_mismatch.py @@ -13,7 +13,7 @@ def fix_model_architecture(): """Fix the model architecture mismatch in the minimal notebook.""" # Read the existing notebook - with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Find and replace the model setup cell @@ -78,4 +78,4 @@ def fix_model_architecture(): print(' โœ… Added detailed logging of the reconfiguration process') if __name__ == "__main__": - fix_model_architecture() \ No newline at end of file + fix_model_architecture() diff --git a/scripts/maintenance/fix_model_reconfiguration.py b/scripts/maintenance/fix_model_reconfiguration.py index a3dc88310..53cfa017a 100644 --- a/scripts/maintenance/fix_model_reconfiguration.py +++ b/scripts/maintenance/fix_model_reconfiguration.py @@ -14,7 +14,7 @@ def fix_model_reconfiguration(): """Fix the model reconfiguration in the minimal notebook.""" # Read the existing notebook - with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/MINIMAL_WORKING_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Find and replace the model setup cell @@ -89,4 +89,4 @@ def fix_model_reconfiguration(): print(' โœ… Added detailed logging of the configuration process') if __name__ == "__main__": - fix_model_reconfiguration() \ No newline at end of file + fix_model_reconfiguration() diff --git a/scripts/maintenance/fix_remaining_py38_types.py b/scripts/maintenance/fix_remaining_py38_types.py index bc9d76a19..5b3692a46 100644 --- a/scripts/maintenance/fix_remaining_py38_types.py +++ b/scripts/maintenance/fix_remaining_py38_types.py @@ -158,7 +158,7 @@ def _add_typing_imports(content: str, imports_to_add: set, dry_run: bool) -> str def fix_file(file_path: Path, dry_run: bool = False) -> Dict[str, Any]: """Fix Python 3.8 compatibility issues in a single file.""" try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() original_content = content @@ -260,7 +260,7 @@ def _print_summary(results: List[Dict[str, Any]], total_changes: int, dry_run: b modified = [r for r in results if r.get('modified', False)] errors = [r for r in results if 'error' in r] no_changes = [ - r for r in results + r for r in results if not r.get('modified', False) and 'error' not in r ] diff --git a/scripts/maintenance/infer_mapping_and_eval.py b/scripts/maintenance/infer_mapping_and_eval.py index 9922c9506..145b106d9 100644 --- a/scripts/maintenance/infer_mapping_and_eval.py +++ b/scripts/maintenance/infer_mapping_and_eval.py @@ -115,7 +115,6 @@ def evaluate(th): # Optional: write corrected config.json with inferred labels in model-index order if os.getenv("WRITE_CONFIG", "0") == "1": - from transformers import AutoConfig cfg = mdl.config id2label = {int(mi): ds_names[dj] for mi, dj in mapping} for i in range(M): diff --git a/scripts/maintenance/metrics_test.py b/scripts/maintenance/metrics_test.py index 74a5624f9..e547ab325 100644 --- a/scripts/maintenance/metrics_test.py +++ b/scripts/maintenance/metrics_test.py @@ -83,20 +83,27 @@ def norm(s: str) -> str: if mapped_count >= 5: kept_ds_indices = [i for i in range(len(ds_names)) if i in ds_to_model] kept_model_indices = [ds_to_model[i] for i in kept_ds_indices] +elif num_labels == len(ds_names): + print( + "Low mapping coverage; identity mapping (assumes same order)." + ) + kept_ds_indices = list(range(num_labels)) + kept_model_indices = list(range(num_labels)) +elif num_labels < len(ds_names): + m = min(num_labels, len(ds_names)) + print( + f"Low mapping coverage; min-dim identity mapping ({m} labels)." + ) + kept_ds_indices = list(range(m)) + kept_model_indices = list(range(m)) else: - if num_labels == len(ds_names): - print( - "Low mapping coverage; identity mapping (assumes same order)." - ) - kept_ds_indices = list(range(num_labels)) - kept_model_indices = list(range(num_labels)) - else: - m = min(num_labels, len(ds_names)) - print( - f"Low mapping coverage; min-dim identity mapping ({m} labels)." - ) - kept_ds_indices = list(range(m)) - kept_model_indices = list(range(m)) + # Fallback case + m = min(num_labels, len(ds_names)) + print( + f"Low mapping coverage; min-dim identity mapping ({m} labels)." + ) + kept_ds_indices = list(range(m)) + kept_model_indices = list(range(m)) D = len(kept_ds_indices) kept_ds_pos = {ds_idx: pos for pos, ds_idx in enumerate(kept_ds_indices)} diff --git a/scripts/maintenance/quick_label_fix.py b/scripts/maintenance/quick_label_fix.py index 8fab9044a..0a69032e6 100644 --- a/scripts/maintenance/quick_label_fix.py +++ b/scripts/maintenance/quick_label_fix.py @@ -17,7 +17,7 @@ def quick_label_fix(): # Load datasets go_emotions = load_dataset("go_emotions", "simplified") - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) @@ -59,13 +59,13 @@ def quick_label_fix(): 'classes': label_encoder.classes_.tolist() }, f, indent=2) - print(f"โœ… Fixed label encoder saved!") + print("โœ… Fixed label encoder saved!") print(f"๐Ÿ“Š Use num_labels={len(label_encoder.classes_)} in your model") - print(f"๐Ÿ“Š Label encoder: fixed_label_encoder.pkl") - print(f"๐Ÿ“Š Mappings: label_mappings.json") + print("๐Ÿ“Š Label encoder: fixed_label_encoder.pkl") + print("๐Ÿ“Š Mappings: label_mappings.json") return len(label_encoder.classes_) if __name__ == "__main__": num_labels = quick_label_fix() - print(f"\n๐ŸŽ‰ Quick fix completed! Use num_labels={num_labels}") \ No newline at end of file + print(f"\n๐ŸŽ‰ Quick fix completed! Use num_labels={num_labels}") diff --git a/scripts/maintenance/typehint_codemod.py b/scripts/maintenance/typehint_codemod.py index 100b035ac..43ff21185 100644 --- a/scripts/maintenance/typehint_codemod.py +++ b/scripts/maintenance/typehint_codemod.py @@ -240,23 +240,22 @@ def _add_typing_imports_to_lines(lines: List[str], imports_to_add: set) -> None: else: lines[i] = f"from typing import {new_imports}" break + # Add new typing import after last import + elif last_import_line >= 0: + import_line = ( + f"from typing import {', '.join(sorted(imports_to_add))}" + ) + lines.insert(last_import_line + 1, import_line) else: - # Add new typing import after last import - if last_import_line >= 0: - import_line = ( - f"from typing import {', '.join(sorted(imports_to_add))}" - ) - lines.insert(last_import_line + 1, import_line) - else: - import_line = ( - f"from typing import {', '.join(sorted(imports_to_add))}" - ) - lines.insert(0, import_line) + import_line = ( + f"from typing import {', '.join(sorted(imports_to_add))}" + ) + lines.insert(0, import_line) def _read_file_content(file_path: Path) -> str: """Read file content.""" - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: return f.read() diff --git a/scripts/testing/check_model_health.py b/scripts/testing/check_model_health.py index a598c194c..3c670ab8c 100755 --- a/scripts/testing/check_model_health.py +++ b/scripts/testing/check_model_health.py @@ -5,7 +5,6 @@ """ import requests -import json from test_config import create_api_client, create_test_config diff --git a/scripts/testing/create_journal_test_dataset.py b/scripts/testing/create_journal_test_dataset.py index 7c31216a6..338ce0b33 100644 --- a/scripts/testing/create_journal_test_dataset.py +++ b/scripts/testing/create_journal_test_dataset.py @@ -306,4 +306,4 @@ def main(): print(" Target: 70% F1 score on journal-style text vs Reddit comments") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/debug_go_emotions_labels.py b/scripts/testing/debug_go_emotions_labels.py index c07515eb5..6b153734a 100644 --- a/scripts/testing/debug_go_emotions_labels.py +++ b/scripts/testing/debug_go_emotions_labels.py @@ -31,14 +31,14 @@ def debug_go_emotions(): # Load the dataset go_emotions = load_dataset("go_emotions", "simplified") - print(f"\n๐Ÿ“Š Dataset structure:") + print("\n๐Ÿ“Š Dataset structure:") print(f"Keys: {list(go_emotions.keys())}") print(f"Train size: {len(go_emotions['train'])}") print(f"Validation size: {len(go_emotions['validation'])}") print(f"Test size: {len(go_emotions['test'])}") # Check first few examples - print(f"\n๐Ÿ“Š First 5 examples:") + print("\n๐Ÿ“Š First 5 examples:") for i in range(min(5, len(go_emotions['train']))): example = go_emotions['train'][i] print(f"Example {i}:") @@ -48,7 +48,7 @@ def debug_go_emotions(): print() # Check if there's a label mapping - print(f"\n๐Ÿ” Checking for label mapping...") + print("\n๐Ÿ” Checking for label mapping...") # Try to get the dataset info try: @@ -65,7 +65,7 @@ def debug_go_emotions(): print("No features available") # Look for label names in the dataset - print(f"\n๐Ÿ” Looking for label names...") + print("\n๐Ÿ” Looking for label names...") # Check if there's a label_names field if hasattr(go_emotions, 'label_names'): @@ -81,7 +81,7 @@ def debug_go_emotions(): print(f"Labels feature: {features['labels']}") # Try to get the original dataset - print(f"\n๐Ÿ” Trying original dataset...") + print("\n๐Ÿ” Trying original dataset...") try: original_go_emotions = load_dataset("go_emotions") print(f"Original dataset keys: {list(original_go_emotions.keys())}") @@ -94,11 +94,11 @@ def debug_go_emotions(): print(f"Could not load original dataset: {e}") # Check the dataset card - print(f"\n๐Ÿ” Checking dataset documentation...") + print("\n๐Ÿ” Checking dataset documentation...") print("GoEmotions dataset should have emotion names like:") print("['admiration', 'amusement', 'anger', 'annoyance', 'approval', 'caring', 'confusion', 'curiosity', 'desire', 'disappointment', 'disapproval', 'disgust', 'embarrassment', 'excitement', 'fear', 'gratitude', 'grief', 'joy', 'love', 'nervousness', 'optimism', 'pride', 'realization', 'relief', 'remorse', 'sadness', 'surprise', 'neutral']") return go_emotions if __name__ == "__main__": - debug_go_emotions() \ No newline at end of file + debug_go_emotions() diff --git a/scripts/testing/debug_label_mismatch.py b/scripts/testing/debug_label_mismatch.py index 23ddc4daa..795865fde 100644 --- a/scripts/testing/debug_label_mismatch.py +++ b/scripts/testing/debug_label_mismatch.py @@ -26,7 +26,7 @@ def debug_label_mismatch(): logger.info(f"โœ… GoEmotions loaded: {len(go_emotions['train'])} training examples") # Load journal dataset - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) logger.info(f"โœ… Journal dataset loaded: {len(journal_df)} entries") @@ -214,8 +214,8 @@ def debug_label_mismatch(): if __name__ == "__main__": result = debug_label_mismatch() if result: - print(f"\n๐ŸŽ‰ Debugging completed successfully!") + print("\n๐ŸŽ‰ Debugging completed successfully!") print(f"๐Ÿ“Š Use num_labels={result['num_labels']} in your model") - print(f"๐Ÿ“Š Label encoder saved as 'fixed_label_encoder.pkl'") + print("๐Ÿ“Š Label encoder saved as 'fixed_label_encoder.pkl'") else: - print(f"\nโŒ Debugging failed!") \ No newline at end of file + print("\nโŒ Debugging failed!") diff --git a/scripts/testing/debug_model_loading.py b/scripts/testing/debug_model_loading.py index b44fa92ee..187f068e1 100644 --- a/scripts/testing/debug_model_loading.py +++ b/scripts/testing/debug_model_loading.py @@ -6,8 +6,6 @@ import requests import json -import time -import argparse from test_config import create_api_client, create_test_config diff --git a/scripts/testing/debug_rate_limiter.py b/scripts/testing/debug_rate_limiter.py index 13feead62..7ccc2eac7 100644 --- a/scripts/testing/debug_rate_limiter.py +++ b/scripts/testing/debug_rate_limiter.py @@ -11,7 +11,7 @@ import os sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) -from src.api_rate_limiter import TokenBucketRateLimiter, RateLimitConfig # noqa: E402 +from src.api_rate_limiter import TokenBucketRateLimiter, RateLimitConfig def debug_rate_limiter(): diff --git a/scripts/testing/debug_rate_limiter_test.py b/scripts/testing/debug_rate_limiter_test.py index 0519ecba6..8d1c8b69c 100644 --- a/scripts/testing/debug_rate_limiter_test.py +++ b/scripts/testing/debug_rate_limiter_test.py @@ -1 +1 @@ - \ No newline at end of file + diff --git a/scripts/testing/hf_serverless_smoke.py b/scripts/testing/hf_serverless_smoke.py index e776c8dba..d746235b3 100644 --- a/scripts/testing/hf_serverless_smoke.py +++ b/scripts/testing/hf_serverless_smoke.py @@ -66,7 +66,7 @@ def main() -> int: r = _post_with_retries(payload) dt = (time.time() - t0) * 1000 print("โ€”" * 40) - print(f"Input: {repr(text)}") + print(f"Input: {text!r}") print(f"Status: {r.status_code} ({dt:.1f} ms)") try: obj = r.json() diff --git a/scripts/testing/mega_comprehensive_model_test.py b/scripts/testing/mega_comprehensive_model_test.py index 7ae040331..efc134368 100644 --- a/scripts/testing/mega_comprehensive_model_test.py +++ b/scripts/testing/mega_comprehensive_model_test.py @@ -378,7 +378,7 @@ def test_bias_analysis(self): 'overall_confidence': np.mean(list(emotion_confidences.values())) } - print(f"๐Ÿ“Š Bias Analysis Results:") + print("๐Ÿ“Š Bias Analysis Results:") print(f" Overall accuracy: {np.mean(list(emotion_accuracies.values())):.2f}%") print(f" Overall confidence: {np.mean(list(emotion_confidences.values())):.3f}") print(f" Most accurate: {most_accurate[0]} ({most_accurate[1]:.2f}%)") @@ -605,7 +605,7 @@ def analyze_confidence_distribution(self): self.test_results['confidence_analysis'] = confidence_stats - print(f"๐Ÿ“Š Confidence Distribution:") + print("๐Ÿ“Š Confidence Distribution:") print(f" Mean: {confidence_stats['mean']:.3f}") print(f" Median: {confidence_stats['median']:.3f}") print(f" Std Dev: {confidence_stats['std']:.3f}") @@ -660,7 +660,7 @@ def generate_comprehensive_report(self): json.dump(report, f, indent=2) # Print summary - print(f"๐ŸŽฏ OVERALL PERFORMANCE SUMMARY") + print("๐ŸŽฏ OVERALL PERFORMANCE SUMMARY") print(f" Total Tests: {total_tests}") print(f" Overall Accuracy: {overall_accuracy:.2f}%") print(f" Overall Confidence: {overall_confidence:.3f}") @@ -697,7 +697,7 @@ def run_all_tests(self): # Generate comprehensive report report = self.generate_comprehensive_report() - print(f"\n๐ŸŽ‰ MEGA COMPREHENSIVE TESTING COMPLETE!") + print("\n๐ŸŽ‰ MEGA COMPREHENSIVE TESTING COMPLETE!") print("=" * 80) return report @@ -708,14 +708,14 @@ def main(): report = tester.run_all_tests() if report: - print(f"\nโœ… Testing completed successfully!") - print(f"๐Ÿ“Š Final Results:") + print("\nโœ… Testing completed successfully!") + print("๐Ÿ“Š Final Results:") print(f" Accuracy: {report['overall_metrics']['overall_accuracy']:.2f}%") print(f" Confidence: {report['overall_metrics']['overall_confidence']:.3f}") print(f" Status: {report['summary']['model_status']}") print(f" Ready for deployment: {'โœ… YES' if report['summary']['deployment_ready'] else 'โŒ NO'}") else: - print(f"\nโŒ Testing failed!") + print("\nโŒ Testing failed!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/mega_test_summary.py b/scripts/testing/mega_test_summary.py index 2954387f4..682399cd7 100644 --- a/scripts/testing/mega_test_summary.py +++ b/scripts/testing/mega_test_summary.py @@ -145,4 +145,4 @@ def display_mega_test_results(): print(" It's ready for production deployment with confidence.") if __name__ == "__main__": - display_mega_test_results() \ No newline at end of file + display_mega_test_results() diff --git a/scripts/testing/setup_model_testing.py b/scripts/testing/setup_model_testing.py index eeed16839..5c69f0cc2 100644 --- a/scripts/testing/setup_model_testing.py +++ b/scripts/testing/setup_model_testing.py @@ -42,7 +42,7 @@ def create_mock_results(): "go_samples": 43410, "journal_samples": 150, "all_emotions": [ - "anxious", "calm", "content", "excited", "frustrated", + "anxious", "calm", "content", "excited", "frustrated", "grateful", "happy", "hopeful", "overwhelmed", "proud", "sad", "tired" ], "emotion_mapping": { @@ -101,7 +101,7 @@ def find_model_file(): # Copy to current directory if not already here if location != "best_simple_model.pth": shutil.copy2(location, "best_simple_model.pth") - print(f"โœ… Copied to: best_simple_model.pth") + print("โœ… Copied to: best_simple_model.pth") return True @@ -165,4 +165,4 @@ def run_quick_test(): print("\n๐ŸŽ‰ Ready to test the model!") print("๐Ÿ“‹ Run: python scripts/test_emotion_model.py") else: - print("\nโŒ Setup failed. Please check the issues above.") \ No newline at end of file + print("\nโŒ Setup failed. Please check the issues above.") diff --git a/scripts/testing/simple_model_test.py b/scripts/testing/simple_model_test.py index 265b0b8f1..f1aa0fcdc 100644 --- a/scripts/testing/simple_model_test.py +++ b/scripts/testing/simple_model_test.py @@ -34,10 +34,10 @@ def test_model_files(): # Try to load and parse try: - with open(results_file, 'r') as f: + with open(results_file) as f: results = json.load(f) - print(f"โœ… Results file is valid JSON") + print("โœ… Results file is valid JSON") print(f"๐Ÿ“Š F1 Score: {results.get('best_f1', 'N/A')}") print(f"๐Ÿ“Š Emotions: {len(results.get('all_emotions', []))}") @@ -116,7 +116,7 @@ def main(): # Test environment env_ok = test_python_environment() - print(f"\n๐Ÿ“Š Test Results:") + print("\n๐Ÿ“Š Test Results:") print(f" Files: {'โœ…' if files_ok else 'โŒ'}") print(f" Environment: {'โœ…' if env_ok else 'โŒ'}") @@ -128,4 +128,4 @@ def main(): suggest_next_steps() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/simple_rate_limiter_test.py b/scripts/testing/simple_rate_limiter_test.py index 0519ecba6..8d1c8b69c 100644 --- a/scripts/testing/simple_rate_limiter_test.py +++ b/scripts/testing/simple_rate_limiter_test.py @@ -1 +1 @@ - \ No newline at end of file + diff --git a/scripts/testing/test_api_startup.py b/scripts/testing/test_api_startup.py index 0519ecba6..8d1c8b69c 100644 --- a/scripts/testing/test_api_startup.py +++ b/scripts/testing/test_api_startup.py @@ -1 +1 @@ - \ No newline at end of file + diff --git a/scripts/testing/test_cloud_run_api_endpoints.py b/scripts/testing/test_cloud_run_api_endpoints.py index 19782a9a2..084c4b152 100644 --- a/scripts/testing/test_cloud_run_api_endpoints.py +++ b/scripts/testing/test_cloud_run_api_endpoints.py @@ -10,7 +10,7 @@ import sys import os import argparse -from typing import Dict, Any, List +from typing import Dict, Any import logging from test_config import create_api_client, create_test_config @@ -66,7 +66,7 @@ def test_health_endpoint(self) -> Dict[str, Any]: except requests.exceptions.RequestException as e: return { "success": False, - "error": f"Health endpoint failed: {str(e)}" + "error": f"Health endpoint failed: {e!s}" } def _validate_emotion_response(self, data: Dict[str, Any]) -> Dict[str, Any]: @@ -111,7 +111,7 @@ def test_emotion_detection_endpoint(self) -> Dict[str, Any]: except requests.exceptions.RequestException as e: return { "success": False, - "error": f"Emotion detection failed: {str(e)}" + "error": f"Emotion detection failed: {e!s}" } def test_model_loading(self) -> Dict[str, Any]: @@ -450,4 +450,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/test_comprehensive_model.py b/scripts/testing/test_comprehensive_model.py index 34e7bf62b..705464fbb 100644 --- a/scripts/testing/test_comprehensive_model.py +++ b/scripts/testing/test_comprehensive_model.py @@ -46,7 +46,7 @@ def test_comprehensive_model(): return # 2. Analyze configuration - print(f"\n๐Ÿ“‹ COMPREHENSIVE MODEL CONFIGURATION") + print("\n๐Ÿ“‹ COMPREHENSIVE MODEL CONFIGURATION") print("-" * 40) print(f"Model type: {model.config.model_type}") @@ -62,7 +62,7 @@ def test_comprehensive_model(): print(f"label2id: {model.config.label2id}") # 3. Verify emotion classes - print(f"\n๐ŸŽฏ EMOTION CLASSES VERIFICATION") + print("\n๐ŸŽฏ EMOTION CLASSES VERIFICATION") print("-" * 40) expected_emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] @@ -90,7 +90,7 @@ def test_comprehensive_model(): return # 4. Test model architecture - print(f"\n๐Ÿ—๏ธ MODEL ARCHITECTURE TEST") + print("\n๐Ÿ—๏ธ MODEL ARCHITECTURE TEST") print("-" * 40) test_input = tokenizer("I feel happy today", return_tensors='pt', truncation=True, padding=True) @@ -110,7 +110,7 @@ def test_comprehensive_model(): return # 5. Comprehensive inference test - print(f"\n๐Ÿงช COMPREHENSIVE INFERENCE TEST") + print("\n๐Ÿงช COMPREHENSIVE INFERENCE TEST") print("-" * 40) # Test cases covering all emotions with various intensities and contexts @@ -209,7 +209,7 @@ def test_comprehensive_model(): print() # 6. Performance analysis - print(f"\n๐Ÿ“Š PERFORMANCE ANALYSIS") + print("\n๐Ÿ“Š PERFORMANCE ANALYSIS") print("-" * 40) accuracy = correct_predictions / len(test_cases) * 100 @@ -225,7 +225,7 @@ def test_comprehensive_model(): print(f"Low confidence predictions (<0.5): {sum(1 for c in confidence_scores if c < 0.5)}/{len(test_cases)}") # 7. Compare with fallback model - print(f"\n๐Ÿ”„ COMPARISON WITH FALLBACK MODEL") + print("\n๐Ÿ”„ COMPARISON WITH FALLBACK MODEL") print("-" * 40) try: @@ -264,11 +264,11 @@ def test_comprehensive_model(): fallback_accuracy = fallback_correct / 12 * 100 fallback_avg_confidence = fallback_confidence / 12 - print(f"Comprehensive Model (36 cases):") + print("Comprehensive Model (36 cases):") print(f" Accuracy: {accuracy:.2f}%") print(f" Average confidence: {average_confidence:.3f}") print() - print(f"Fallback Model (12 cases):") + print("Fallback Model (12 cases):") print(f" Accuracy: {fallback_accuracy:.2f}%") print(f" Average confidence: {fallback_avg_confidence:.3f}") print() @@ -289,7 +289,7 @@ def test_comprehensive_model(): print(f"โš ๏ธ Could not compare with fallback model: {e}") # 8. Configuration persistence verification - print(f"\n๐Ÿ” CONFIGURATION PERSISTENCE VERIFICATION") + print("\n๐Ÿ” CONFIGURATION PERSISTENCE VERIFICATION") print("-" * 40) # Check if all critical configuration is preserved @@ -315,7 +315,7 @@ def test_comprehensive_model(): print("โŒ Configuration persistence issues detected!") # 9. Final assessment - print(f"\n๐ŸŽฏ FINAL ASSESSMENT") + print("\n๐ŸŽฏ FINAL ASSESSMENT") print("-" * 40) print("Configuration Status:") @@ -346,7 +346,7 @@ def test_comprehensive_model(): print("โŒ Low confidence predictions") # 10. Summary - print(f"\n๐Ÿ“‹ SUMMARY") + print("\n๐Ÿ“‹ SUMMARY") print("-" * 40) print("โœ… Comprehensive model loads successfully") @@ -363,13 +363,13 @@ def test_comprehensive_model(): print("โŒ Configuration persistence issues need attention") # 11. Update model metadata - print(f"\n๐Ÿ“ UPDATING MODEL METADATA") + print("\n๐Ÿ“ UPDATING MODEL METADATA") print("-" * 40) metadata_path = os.path.join(comprehensive_model_path, "model_metadata.json") if os.path.exists(metadata_path): try: - with open(metadata_path, 'r') as f: + with open(metadata_path) as f: metadata = json.load(f) # Update with test results @@ -388,8 +388,8 @@ def test_comprehensive_model(): except Exception as e: print(f"โš ๏ธ Could not update metadata: {e}") - print(f"\n๐ŸŽ‰ COMPREHENSIVE MODEL TESTING COMPLETE!") + print("\n๐ŸŽ‰ COMPREHENSIVE MODEL TESTING COMPLETE!") print("=" * 60) if __name__ == "__main__": - test_comprehensive_model() \ No newline at end of file + test_comprehensive_model() diff --git a/scripts/testing/test_config.py b/scripts/testing/test_config.py index 15e3f2bd4..b9bd87928 100644 --- a/scripts/testing/test_config.py +++ b/scripts/testing/test_config.py @@ -28,8 +28,8 @@ def _get_base_url(self) -> str: return args.base_url.rstrip('/') # Check multiple environment variables for flexibility - env_url = (os.environ.get("API_BASE_URL") or - os.environ.get("CLOUD_RUN_API_URL") or + env_url = (os.environ.get("API_BASE_URL") or + os.environ.get("CLOUD_RUN_API_URL") or os.environ.get("MODEL_API_BASE_URL")) if env_url: @@ -86,9 +86,9 @@ def get(self, endpoint: str, **kwargs) -> dict: response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: - raise requests.exceptions.RequestException(f"GET {endpoint} failed: {str(e)}") + raise requests.exceptions.RequestException(f"GET {endpoint} failed: {e!s}") except ValueError as e: - raise ValueError(f"Invalid JSON response from {endpoint}: {str(e)}") + raise ValueError(f"Invalid JSON response from {endpoint}: {e!s}") def post(self, endpoint: str, data: dict, **kwargs) -> dict: """Make POST request with consistent error handling""" @@ -102,9 +102,9 @@ def post(self, endpoint: str, data: dict, **kwargs) -> dict: response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: - raise requests.exceptions.RequestException(f"POST {endpoint} failed: {str(e)}") + raise requests.exceptions.RequestException(f"POST {endpoint} failed: {e!s}") except ValueError as e: - raise ValueError(f"Invalid JSON response from {endpoint}: {str(e)}") + raise ValueError(f"Invalid JSON response from {endpoint}: {e!s}") def create_test_config() -> TestConfig: @@ -115,4 +115,4 @@ def create_test_config() -> TestConfig: def create_api_client() -> APIClient: """Factory function to create API client""" config = create_test_config() - return APIClient(config) \ No newline at end of file + return APIClient(config) diff --git a/scripts/testing/test_e2e_simple.py b/scripts/testing/test_e2e_simple.py index 0519ecba6..8d1c8b69c 100644 --- a/scripts/testing/test_e2e_simple.py +++ b/scripts/testing/test_e2e_simple.py @@ -1 +1 @@ - \ No newline at end of file + diff --git a/scripts/testing/test_emotion_model.py b/scripts/testing/test_emotion_model.py index bfcbb0c21..0d5b91206 100644 --- a/scripts/testing/test_emotion_model.py +++ b/scripts/testing/test_emotion_model.py @@ -5,7 +5,7 @@ import json import torch -import torch.nn as nn +from torch import nn from transformers import AutoModel, AutoTokenizer from sklearn.preprocessing import LabelEncoder import numpy as np @@ -24,7 +24,7 @@ def load_trained_model(): tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") # Load label encoder - with open('simple_training_results.json', 'r') as f: + with open('simple_training_results.json') as f: results = json.load(f) # Create label encoder from results @@ -126,7 +126,7 @@ def analyze_performance(): print("=" * 40) # Load results - with open('simple_training_results.json', 'r') as f: + with open('simple_training_results.json') as f: results = json.load(f) print(f"Final F1 Score: {results['best_f1']:.4f}") @@ -136,10 +136,10 @@ def analyze_performance(): print(f"Journal Samples: {results['journal_samples']}") # Show emotion mapping - print(f"\nEmotion Mapping Used:") + print("\nEmotion Mapping Used:") for go_emotion, journal_emotion in results['emotion_mapping'].items(): print(f" {go_emotion} โ†’ {journal_emotion}") if __name__ == "__main__": test_model() - analyze_performance() \ No newline at end of file + analyze_performance() diff --git a/scripts/testing/test_final_inference.py b/scripts/testing/test_final_inference.py index 949c4313f..49a0d4713 100644 --- a/scripts/testing/test_final_inference.py +++ b/scripts/testing/test_final_inference.py @@ -34,11 +34,11 @@ def test_final_inference(): print(f"\nโŒ Missing required files: {missing_files}") return False - print(f"\nโœ… All model files found!") + print("\nโœ… All model files found!") try: # Load the model config to understand the architecture - with open(model_dir / 'config.json', 'r') as f: + with open(model_dir / 'config.json') as f: config = json.load(f) print(f"๐Ÿ”ง Model type: {config.get('model_type', 'unknown')}") @@ -67,7 +67,7 @@ def test_final_inference(): model.to(device) model.eval() - print(f"โœ… Model loaded successfully!") + print("โœ… Model loaded successfully!") print(f"๐ŸŽฏ Device: {device}") # Test texts @@ -84,7 +84,7 @@ def test_final_inference(): "I'm hopeful that things will get better." ] - print(f"\n๐Ÿ“Š Testing predictions:") + print("\n๐Ÿ“Š Testing predictions:") print("-" * 50) for i, text in enumerate(test_texts, 1): @@ -117,7 +117,7 @@ def test_final_inference(): print(f"{i:2d}. Text: {text}") print(f" Predicted: {predicted_emotion} (confidence: {confidence:.3f})") - print(f" Top 3 predictions:") + print(" Top 3 predictions:") for emotion, conf in top3_predictions: print(f" - {emotion}: {conf:.3f}") print() @@ -178,13 +178,13 @@ def test_simple_prediction(): # Show top 3 top3_indices = torch.topk(probabilities[0], 3).indices - print(f"\n๐Ÿ† Top 3 predictions:") + print("\n๐Ÿ† Top 3 predictions:") for i, idx in enumerate(top3_indices): emotion = emotion_mapping[idx.item()] conf = probabilities[0][idx].item() print(f" {i+1}. {emotion}: {conf:.3f}") - print(f"\n๐ŸŽ‰ Simple prediction test completed!") + print("\n๐ŸŽ‰ Simple prediction test completed!") return True except Exception as e: @@ -204,9 +204,9 @@ def test_simple_prediction(): test_simple_prediction() if success: - print(f"\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") - print(f"๐Ÿ“‹ Next steps:") - print(f" - Deploy with: cd deployment && ./deploy.sh") - print(f" - API will be available at: http://localhost:5000") + print("\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") + print("๐Ÿ“‹ Next steps:") + print(" - Deploy with: cd deployment && ./deploy.sh") + print(" - API will be available at: http://localhost:5000") else: - print(f"\nโŒ Tests failed. Check the error messages above.") \ No newline at end of file + print("\nโŒ Tests failed. Check the error messages above.") diff --git a/scripts/testing/test_fixed_inference.py b/scripts/testing/test_fixed_inference.py index 2ed7ab6e7..bcaa7dfca 100644 --- a/scripts/testing/test_fixed_inference.py +++ b/scripts/testing/test_fixed_inference.py @@ -34,11 +34,11 @@ def test_fixed_inference(): print(f"\nโŒ Missing required files: {missing_files}") return False - print(f"\nโœ… All model files found!") + print("\nโœ… All model files found!") try: # Load the model config to understand the architecture - with open(model_dir / 'config.json', 'r') as f: + with open(model_dir / 'config.json') as f: config = json.load(f) print(f"๐Ÿ”ง Model type: {config.get('model_type', 'unknown')}") @@ -67,7 +67,7 @@ def test_fixed_inference(): model.to(device) model.eval() - print(f"โœ… Model loaded successfully!") + print("โœ… Model loaded successfully!") print(f"๐ŸŽฏ Device: {device}") # Test texts @@ -84,7 +84,7 @@ def test_fixed_inference(): "I'm hopeful that things will get better." ] - print(f"\n๐Ÿ“Š Testing predictions:") + print("\n๐Ÿ“Š Testing predictions:") print("-" * 50) for i, text in enumerate(test_texts, 1): @@ -117,7 +117,7 @@ def test_fixed_inference(): print(f"{i:2d}. Text: {text}") print(f" Predicted: {predicted_emotion} (confidence: {confidence:.3f})") - print(f" Top 3 predictions:") + print(" Top 3 predictions:") for emotion, conf in top3_predictions: print(f" - {emotion}: {conf:.3f}") print() @@ -143,9 +143,9 @@ def test_fixed_inference(): success = test_fixed_inference() if success: - print(f"\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") - print(f"๐Ÿ“‹ Next steps:") - print(f" - Deploy with: cd deployment && ./deploy.sh") - print(f" - API will be available at: http://localhost:5000") + print("\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") + print("๐Ÿ“‹ Next steps:") + print(" - Deploy with: cd deployment && ./deploy.sh") + print(" - API will be available at: http://localhost:5000") else: - print(f"\nโŒ Test failed. Check the error messages above.") \ No newline at end of file + print("\nโŒ Test failed. Check the error messages above.") diff --git a/scripts/testing/test_local_inference.py b/scripts/testing/test_local_inference.py index a5f33ddb1..64d5a7df5 100644 --- a/scripts/testing/test_local_inference.py +++ b/scripts/testing/test_local_inference.py @@ -81,4 +81,4 @@ def test_local_inference(): if __name__ == "__main__": success = test_local_inference() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/testing/test_model_status.py b/scripts/testing/test_model_status.py index 9a3d0e467..a48d1da33 100644 --- a/scripts/testing/test_model_status.py +++ b/scripts/testing/test_model_status.py @@ -102,4 +102,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/test_new_trained_model.py b/scripts/testing/test_new_trained_model.py index d21c77746..de959386a 100644 --- a/scripts/testing/test_new_trained_model.py +++ b/scripts/testing/test_new_trained_model.py @@ -41,7 +41,7 @@ def test_new_trained_model(): print("โœ… Model loaded successfully!") # Check model configuration - print(f"\n๐Ÿ“Š Model Configuration:") + print("\n๐Ÿ“Š Model Configuration:") print(f" Model type: {model.config.model_type}") print(f" Architecture: {model.config.architectures[0]}") print(f" Hidden layers: {model.config.num_hidden_layers}") @@ -52,7 +52,7 @@ def test_new_trained_model(): # Define emotion mapping emotions = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] - print(f"\n๐ŸŽฏ Testing predictions...") + print("\n๐ŸŽฏ Testing predictions...") # Test examples test_examples = [ @@ -105,7 +105,7 @@ def test_new_trained_model(): print(f"\n๐Ÿ“Š Test Accuracy: {accuracy:.1%} ({correct}/{len(test_examples)})") # Test on some edge cases - print(f"\n๐Ÿงช Testing edge cases...") + print("\n๐Ÿงช Testing edge cases...") edge_cases = [ "I'm not sure how I feel.", "This is amazing!", @@ -126,7 +126,7 @@ def test_new_trained_model(): print(f" \"{text}\" โ†’ {predicted_emotion} (confidence: {confidence:.3f})") # Overall assessment - print(f"\n๐ŸŽฏ MODEL ASSESSMENT:") + print("\n๐ŸŽฏ MODEL ASSESSMENT:") if accuracy >= 0.8: print("โœ… EXCELLENT: Model ready for deployment!") elif accuracy >= 0.7: @@ -136,15 +136,15 @@ def test_new_trained_model(): else: print("โŒ POOR: Model needs significant improvement") - print(f"\n๐Ÿ“‹ Next steps:") - print(f" 1. Model is ready for local testing") - print(f" 2. Can be deployed to API server") - print(f" 3. Consider retraining tomorrow for better results") + print("\n๐Ÿ“‹ Next steps:") + print(" 1. Model is ready for local testing") + print(" 2. Can be deployed to API server") + print(" 3. Consider retraining tomorrow for better results") return True except Exception as e: - print(f"โŒ Error testing model: {str(e)}") + print(f"โŒ Error testing model: {e!s}") return False if __name__ == "__main__": @@ -152,4 +152,4 @@ def test_new_trained_model(): if success: print("\n๐ŸŽ‰ Model testing completed successfully!") else: - print("\nโŒ Model testing failed!") \ No newline at end of file + print("\nโŒ Model testing failed!") diff --git a/scripts/testing/test_new_trained_model_comprehensive.py b/scripts/testing/test_new_trained_model_comprehensive.py index 75a6a5cb8..7e4f9c65a 100644 --- a/scripts/testing/test_new_trained_model_comprehensive.py +++ b/scripts/testing/test_new_trained_model_comprehensive.py @@ -37,7 +37,7 @@ def test_new_trained_model(): model = AutoModelForSequenceClassification.from_pretrained(model_path) print("โœ… Model and tokenizer loaded successfully") except Exception as e: - print(f"โŒ Error loading model: {str(e)}") + print(f"โŒ Error loading model: {e!s}") return # 2. Check configuration @@ -210,7 +210,7 @@ def test_new_trained_model(): print("โœ… Configuration persistence verified") print("โœ… Model should work correctly in deployment") - print(f"\nPerformance Status:") + print("\nPerformance Status:") if accuracy >= 0.8: print("โœ… Excellent performance (โ‰ฅ80% accuracy)") elif accuracy >= 0.6: @@ -218,7 +218,7 @@ def test_new_trained_model(): else: print("โŒ Poor performance (<60% accuracy)") - print(f"\nConfidence Status:") + print("\nConfidence Status:") if avg_confidence >= 0.7: print("โœ… High confidence predictions") elif avg_confidence >= 0.5: @@ -230,10 +230,10 @@ def test_new_trained_model(): print("\n๐Ÿ“‹ SUMMARY") print("-" * 40) - print(f"โœ… Model loads successfully") - print(f"โœ… Architecture is correct (DistilRoBERTa)") - print(f"โœ… Emotion classes are properly configured") - print(f"โœ… Inference works correctly") + print("โœ… Model loads successfully") + print("โœ… Architecture is correct (DistilRoBERTa)") + print("โœ… Emotion classes are properly configured") + print("โœ… Inference works correctly") print(f"๐Ÿ“Š Test accuracy: {accuracy:.2%}") print(f"๐Ÿ“Š Average confidence: {avg_confidence:.3f}") @@ -241,7 +241,7 @@ def test_new_trained_model(): print(f"โš ๏ธ Configuration issues: {len(config_issues)}") print(" Consider using the comprehensive notebook for better configuration persistence") else: - print(f"โœ… Configuration persistence verified") + print("โœ… Configuration persistence verified") print("โœ… Model ready for deployment!") return { @@ -252,4 +252,4 @@ def test_new_trained_model(): } if __name__ == "__main__": - test_new_trained_model() \ No newline at end of file + test_new_trained_model() diff --git a/scripts/testing/test_numpy_compatibility.py b/scripts/testing/test_numpy_compatibility.py index de68fb00c..c23a8cc83 100644 --- a/scripts/testing/test_numpy_compatibility.py +++ b/scripts/testing/test_numpy_compatibility.py @@ -59,4 +59,4 @@ def broadcast_to(array, shape): if __name__ == "__main__": success = test_numpy_compatibility() if not success: - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/scripts/testing/test_phase3_cloud_run_optimization.py b/scripts/testing/test_phase3_cloud_run_optimization.py index d063ecd45..9774c9278 100644 --- a/scripts/testing/test_phase3_cloud_run_optimization.py +++ b/scripts/testing/test_phase3_cloud_run_optimization.py @@ -10,7 +10,7 @@ import json import time from pathlib import Path -from typing import Dict, Any, List, Optional +from typing import Dict, Any import unittest from unittest.mock import patch import logging @@ -61,7 +61,7 @@ def test_01_cloudbuild_yaml_structure(self): cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' self.assertTrue(cloudbuild_path.exists(), "cloudbuild.yaml should exist") - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Validate required fields @@ -212,7 +212,7 @@ def test_04_dockerfile_optimization(self): dockerfile_path = self.cloud_run_dir / 'Dockerfile.secure' self.assertTrue(dockerfile_path.exists(), "Dockerfile.secure should exist") - with open(dockerfile_path, 'r') as f: + with open(dockerfile_path) as f: content = f.read() # Test security features @@ -275,7 +275,7 @@ def test_05_requirements_security(self): requirements_path = self.cloud_run_dir / 'requirements_secure.txt' self.assertTrue(requirements_path.exists(), "requirements_secure.txt should exist") - with open(requirements_path, 'r') as f: + with open(requirements_path) as f: content = f.read() # Test required dependencies (updated to match actual requirements format) @@ -296,7 +296,7 @@ def test_05_requirements_security(self): unpinned_deps = [] for line in lines: line = line.strip() - if (line and not line.startswith('#') and + if (line and not line.startswith('#') and '==' not in line and '>=' not in line and '<=' not in line): unpinned_deps.append(line) @@ -310,7 +310,7 @@ def test_06_auto_scaling_configuration(self): print("๐Ÿ” Testing auto-scaling configuration...") cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Find Cloud Run deployment step @@ -358,7 +358,7 @@ def test_07_health_check_integration(self): # Test health check endpoint configuration cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Check for health check environment variables @@ -481,7 +481,7 @@ def test_10_yaml_parsing_validation(self): # Test Cloud Build YAML parsing cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Validate YAML structure using enhanced approach @@ -586,4 +586,4 @@ def run_phase3_tests(): if __name__ == '__main__': success = run_phase3_tests() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/testing/test_phase3_cloud_run_optimization_fixed.py b/scripts/testing/test_phase3_cloud_run_optimization_fixed.py index a846c6b2b..a384b7812 100644 --- a/scripts/testing/test_phase3_cloud_run_optimization_fixed.py +++ b/scripts/testing/test_phase3_cloud_run_optimization_fixed.py @@ -6,7 +6,6 @@ import sys import yaml from pathlib import Path -from typing import Dict, Any, List, Optional import unittest # Add src to path for imports @@ -41,7 +40,7 @@ def test_01_cloudbuild_yaml_structure(self): cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' self.assertTrue(cloudbuild_path.exists(), "cloudbuild.yaml should exist") - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Validate required fields - individual assertions instead of loop @@ -141,7 +140,7 @@ def test_05_environment_config_validation(self): config_path = self.cloud_run_dir / 'config.py' self.assertTrue(config_path.exists(), "config.py should exist") - with open(config_path, 'r') as f: + with open(config_path) as f: content = f.read() # Check for required configuration elements @@ -169,7 +168,7 @@ def test_06_dockerfile_optimization(self): dockerfile_path = self.cloud_run_dir / 'Dockerfile.secure' self.assertTrue(dockerfile_path.exists(), "Dockerfile.secure should exist") - with open(dockerfile_path, 'r') as f: + with open(dockerfile_path) as f: content = f.read() # Check for optimization features @@ -199,7 +198,7 @@ def test_07_requirements_security(self): requirements_path = self.cloud_run_dir / 'requirements_secure.txt' self.assertTrue(requirements_path.exists(), "requirements_secure.txt should exist") - with open(requirements_path, 'r') as f: + with open(requirements_path) as f: content = f.read() # Check for required dependencies @@ -243,7 +242,7 @@ def test_08_auto_scaling_configuration(self): cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' self.assertTrue(cloudbuild_path.exists(), "cloudbuild.yaml should exist") - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Get deployment step @@ -274,7 +273,7 @@ def test_09_health_check_integration(self): cloudbuild_path = self.cloud_run_dir / 'cloudbuild.yaml' self.assertTrue(cloudbuild_path.exists(), "cloudbuild.yaml should exist") - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Get deployment step @@ -305,7 +304,7 @@ def test_10_yaml_parsing_validation(self): self.assertTrue(cloudbuild_path.exists(), "cloudbuild.yaml should exist") # Test YAML parsing - with open(cloudbuild_path, 'r') as f: + with open(cloudbuild_path) as f: config = yaml.safe_load(f) # Validate basic structure @@ -373,4 +372,4 @@ def run_phase3_tests_fixed(): if __name__ == "__main__": success = run_phase3_tests_fixed() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/testing/test_phase4_vertex_ai_automation.py b/scripts/testing/test_phase4_vertex_ai_automation.py index 47072f532..e3b2328e2 100644 --- a/scripts/testing/test_phase4_vertex_ai_automation.py +++ b/scripts/testing/test_phase4_vertex_ai_automation.py @@ -5,7 +5,6 @@ """ import sys from pathlib import Path -from typing import Dict, Any, List, Optional import unittest # Add src to path for imports @@ -39,7 +38,7 @@ def test_01_script_structure(self): self.assertTrue(self.vertex_ai_script.exists(), "Vertex AI automation script should exist") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for required classes and methods @@ -70,7 +69,7 @@ def test_02_deployment_config_dataclass(self): """Test DeploymentConfig dataclass structure""" print("๐Ÿ” Testing DeploymentConfig dataclass...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for dataclass import and usage @@ -98,7 +97,7 @@ def test_03_prerequisites_checking(self): """Test prerequisites checking functionality""" print("๐Ÿ” Testing prerequisites checking...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for prerequisite checks @@ -137,7 +136,7 @@ def test_04_model_versioning(self): """Test model versioning functionality""" print("๐Ÿ” Testing model versioning...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for version generation @@ -154,7 +153,7 @@ def test_05_deployment_package_creation(self): """Test deployment package creation""" print("๐Ÿ” Testing deployment package creation...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for deployment package creation @@ -181,7 +180,7 @@ def test_06_docker_image_handling(self): """Test Docker image building and pushing""" print("๐Ÿ” Testing Docker image handling...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for Docker operations @@ -199,7 +198,7 @@ def test_07_vertex_ai_model_creation(self): """Test Vertex AI model creation""" print("๐Ÿ” Testing Vertex AI model creation...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for model creation @@ -215,7 +214,7 @@ def test_08_endpoint_deployment(self): """Test endpoint deployment functionality""" print("๐Ÿ” Testing endpoint deployment...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for endpoint deployment @@ -232,7 +231,7 @@ def test_09_monitoring_and_alerting(self): """Test monitoring and alerting setup""" print("๐Ÿ” Testing monitoring and alerting...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for monitoring setup @@ -250,7 +249,7 @@ def test_10_cost_monitoring(self): """Test cost monitoring setup""" print("๐Ÿ” Testing cost monitoring...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for cost monitoring @@ -267,7 +266,7 @@ def test_11_rollback_capabilities(self): """Test rollback capabilities""" print("๐Ÿ” Testing rollback capabilities...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for rollback functionality @@ -281,7 +280,7 @@ def test_12_ab_testing_support(self): """Test A/B testing support""" print("๐Ÿ” Testing A/B testing support...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for A/B testing @@ -296,7 +295,7 @@ def test_13_performance_metrics(self): """Test performance metrics collection""" print("๐Ÿ” Testing performance metrics...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for performance metrics @@ -310,7 +309,7 @@ def test_14_cleanup_functionality(self): """Test cleanup functionality""" print("๐Ÿ” Testing cleanup functionality...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for cleanup @@ -324,7 +323,7 @@ def test_15_full_deployment_workflow(self): """Test full deployment workflow""" print("๐Ÿ” Testing full deployment workflow...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for full deployment workflow @@ -354,7 +353,7 @@ def test_16_error_handling(self): """Test error handling and logging""" print("๐Ÿ” Testing error handling...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for error handling @@ -371,7 +370,7 @@ def test_17_configuration_management(self): """Test configuration management""" print("๐Ÿ” Testing configuration management...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for configuration management @@ -389,7 +388,7 @@ def test_18_security_features(self): """Test security features""" print("๐Ÿ” Testing security features...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for security features @@ -404,7 +403,7 @@ def test_19_documentation_and_logging(self): """Test documentation and logging""" print("๐Ÿ” Testing documentation and logging...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for documentation @@ -422,7 +421,7 @@ def test_20_main_function(self): """Test main function""" print("๐Ÿ” Testing main function...") - with open(self.vertex_ai_script, 'r') as f: + with open(self.vertex_ai_script) as f: content = f.read() # Check for main function @@ -490,4 +489,4 @@ def run_phase4_tests(): if __name__ == "__main__": success = run_phase4_tests() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/testing/test_pr4_integration.py b/scripts/testing/test_pr4_integration.py index 761e39b50..a35ee3164 100644 --- a/scripts/testing/test_pr4_integration.py +++ b/scripts/testing/test_pr4_integration.py @@ -45,11 +45,11 @@ def run_all_tests(self) -> Dict[str, Any]: error_result = { "name": test.__name__, "passed": False, - "message": f"Test failed with exception: {str(e)}", + "message": f"Test failed with exception: {e!s}", "details": str(e) } self.test_results.append(error_result) - print(f"โŒ FAIL {test.__name__}: {str(e)}") + print(f"โŒ FAIL {test.__name__}: {e!s}") return self.generate_summary() @@ -64,7 +64,7 @@ def test_security_configuration(self) -> Dict[str, Any]: } try: - with open(self.security_config_path, 'r', encoding='utf-8') as f: + with open(self.security_config_path, encoding='utf-8') as f: config = yaml.safe_load(f) # Check required sections @@ -100,7 +100,7 @@ def test_security_configuration(self) -> Dict[str, Any]: return { "name": "Security Configuration", "passed": False, - "message": f"Invalid YAML in security configuration: {str(e)}", + "message": f"Invalid YAML in security configuration: {e!s}", "details": str(e) } @@ -115,7 +115,7 @@ def test_openapi_specification(self) -> Dict[str, Any]: } try: - with open(self.openapi_spec_path, 'r') as f: + with open(self.openapi_spec_path) as f: spec = yaml.safe_load(f) # Check OpenAPI version @@ -159,7 +159,7 @@ def test_openapi_specification(self) -> Dict[str, Any]: return { "name": "OpenAPI Specification", "passed": False, - "message": f"Invalid YAML in OpenAPI specification: {str(e)}", + "message": f"Invalid YAML in OpenAPI specification: {e!s}", "details": str(e) } @@ -174,7 +174,7 @@ def test_dependencies_security(self) -> Dict[str, Any]: } try: - with open(self.requirements_path, 'r') as f: + with open(self.requirements_path) as f: requirements = f.read() # Check for security scanning tools @@ -196,13 +196,13 @@ def test_dependencies_security(self) -> Dict[str, Any]: # - certifi: Ensures up-to-date CA certificates for secure HTTPS connections. # - urllib3: Secure HTTP client with robust TLS/SSL support. try: - with open(self.security_config_path, 'r') as secf: + with open(self.security_config_path) as secf: security_config = yaml.safe_load(secf) critical_packages = security_config.get('critical_packages', ['cryptography', 'certifi', 'urllib3']) if 'critical_packages' not in security_config: print("โš ๏ธ Warning: 'critical_packages' not found in security.yaml, using default list.") except Exception as e: - print(f"โš ๏ธ Warning: Could not read security.yaml for critical_packages: {str(e)}. Using default list.") + print(f"โš ๏ธ Warning: Could not read security.yaml for critical_packages: {e!s}. Using default list.") critical_packages = ['cryptography', 'certifi', 'urllib3'] missing_critical = [pkg for pkg in critical_packages if pkg not in requirements] @@ -225,7 +225,7 @@ def test_dependencies_security(self) -> Dict[str, Any]: return { "name": "Dependencies Security", "passed": False, - "message": f"Error reading requirements file: {str(e)}", + "message": f"Error reading requirements file: {e!s}", "details": str(e) } @@ -269,8 +269,8 @@ def test_security_scanning_tools(self) -> Dict[str, Any]: "message": "Bandit security scanner not found in PATH", "details": "Install bandit: pip install bandit" } - result = subprocess.run([bandit_path, '--version'], - capture_output=True, text=True, timeout=30) + result = subprocess.run([bandit_path, '--version'], + check=False, capture_output=True, text=True, timeout=30) if result.returncode != 0: return { "name": "Security Scanning Tools", @@ -289,7 +289,7 @@ def test_security_scanning_tools(self) -> Dict[str, Any]: "details": "Install safety and ensure it is in a secure location" } result = subprocess.run([safety_path, '--version'], - capture_output=True, text=True, timeout=30) + check=False, capture_output=True, text=True, timeout=30) if result.returncode != 0: return { "name": "Security Scanning Tools", @@ -374,4 +374,4 @@ def main(): print("Ready for final review and submission") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/testing/test_pr5_cicd_integration.py b/scripts/testing/test_pr5_cicd_integration.py index 064d4032a..7fe07d0a7 100644 --- a/scripts/testing/test_pr5_cicd_integration.py +++ b/scripts/testing/test_pr5_cicd_integration.py @@ -21,7 +21,7 @@ def test_yaml_syntax(): return False try: - with open(config_path, 'r') as f: + with open(config_path) as f: yaml.safe_load(f) print("โœ… CircleCI YAML syntax is valid") return True @@ -42,7 +42,7 @@ def test_conda_environment_setup(): conda_cmd = ['conda'] # fallback to PATH result = subprocess.run(conda_cmd + ['--version'], - capture_output=True, text=True, timeout=10) + check=False, capture_output=True, text=True, timeout=10) if result.returncode != 0: print("โŒ Conda not available") return False @@ -54,7 +54,7 @@ def test_conda_environment_setup(): return False # Validate environment.yml structure - with open(env_path, 'r') as f: + with open(env_path) as f: env_yaml = yaml.safe_load(f) # Check required fields @@ -85,7 +85,7 @@ def test_conda_environment_setup(): return False print(f"โœ… Found {len(found_packages)} packages in environment.yml") - print(f"โœ… Conda environment setup validation passed (fast mode)") + print("โœ… Conda environment setup validation passed (fast mode)") return True except Exception as e: @@ -100,7 +100,7 @@ def test_critical_fixes(): config_path = Path(".circleci/config.yml") try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = yaml.safe_load(f) except Exception as e: print(f"โŒ Failed to load config: {e}") @@ -187,7 +187,7 @@ def test_pipeline_structure(): config_path = Path(".circleci/config.yml") try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = yaml.safe_load(f) except Exception as e: print(f"โŒ Failed to load config: {e}") @@ -195,7 +195,7 @@ def test_pipeline_structure(): required_components = [ "executors", - "commands", + "commands", "jobs", "workflows" ] @@ -227,7 +227,7 @@ def test_pipeline_structure_edge_cases(): } required_components = [ "executors", - "commands", + "commands", "jobs", "workflows" ] @@ -241,9 +241,9 @@ def test_pipeline_structure_edge_cases(): malformed_configs = [None, [], "not_a_dict"] for idx, malformed in enumerate(malformed_configs): if not isinstance(malformed, dict): - print(f"โœ… Malformed config case {idx+1}: {repr(malformed)} correctly identified as invalid") + print(f"โœ… Malformed config case {idx+1}: {malformed!r} correctly identified as invalid") else: - print(f"โŒ Malformed config case {idx+1}: {repr(malformed)} incorrectly identified as valid") + print(f"โŒ Malformed config case {idx+1}: {malformed!r} incorrectly identified as valid") return True @@ -253,7 +253,7 @@ def test_job_dependencies(): config_path = Path(".circleci/config.yml") try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = yaml.safe_load(f) except Exception as e: print(f"โŒ Failed to load config: {e}") @@ -335,7 +335,7 @@ def test_environment_variables(): config_path = Path(".circleci/config.yml") try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = yaml.safe_load(f) except Exception as e: print(f"โŒ Failed to load config: {e}") @@ -344,7 +344,7 @@ def test_environment_variables(): # Check for hardcoded conda paths that should be abstracted content = "" try: - with open(config_path, 'r') as f: + with open(config_path) as f: content = f.read() except Exception as e: print(f"โŒ Failed to read config content: {e}") @@ -426,4 +426,4 @@ def main(): if __name__ == "__main__": success = main() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/scripts/testing/test_rate_limiter_no_threading.py b/scripts/testing/test_rate_limiter_no_threading.py index 0519ecba6..8d1c8b69c 100644 --- a/scripts/testing/test_rate_limiter_no_threading.py +++ b/scripts/testing/test_rate_limiter_no_threading.py @@ -1 +1 @@ - \ No newline at end of file + diff --git a/scripts/testing/test_working_inference.py b/scripts/testing/test_working_inference.py index 986e59ffd..d55f00a94 100644 --- a/scripts/testing/test_working_inference.py +++ b/scripts/testing/test_working_inference.py @@ -37,7 +37,7 @@ def test_working_inference(): print("\nโœ… All model files found!") # Load config to understand the model - with open(model_dir / 'config.json', 'r') as f: + with open(model_dir / 'config.json') as f: config = json.load(f) print(f"๐Ÿ”ง Model type: {config.get('model_type', 'unknown')}") @@ -48,7 +48,7 @@ def test_working_inference(): print(f"๐ŸŽฏ Emotion mapping: {emotion_mapping}") try: - print(f"\n๐Ÿ”ง Loading public tokenizer: roberta-base") + print("\n๐Ÿ”ง Loading public tokenizer: roberta-base") tokenizer = AutoTokenizer.from_pretrained("roberta-base") print(f"๐Ÿ”ง Loading model from: {model_dir}") @@ -69,7 +69,7 @@ def test_working_inference(): "I'm feeling overwhelmed with tasks." ] - print(f"\n๐Ÿงช Testing inference...") + print("\n๐Ÿงช Testing inference...") print("=" * 50) for i, text in enumerate(test_texts, 1): @@ -91,11 +91,11 @@ def test_working_inference(): print(f" Predicted: {emotion} (confidence: {confidence:.3f})") - print(f"\nโœ… Inference test completed successfully!") + print("\nโœ… Inference test completed successfully!") return True except Exception as e: - print(f"\nโŒ Error during inference: {str(e)}") + print(f"\nโŒ Error during inference: {e!s}") return False def test_simple_inference(): @@ -131,13 +131,13 @@ def test_simple_inference(): emotion_mapping = ['anxious', 'calm', 'content', 'excited', 'frustrated', 'grateful', 'happy', 'hopeful', 'overwhelmed', 'proud', 'sad', 'tired'] emotion = emotion_mapping[predicted_class] - print(f"โœ… Simple test successful!") + print("โœ… Simple test successful!") print(f" Text: {text}") print(f" Predicted: {emotion} (confidence: {confidence:.3f})") return True except Exception as e: - print(f"โŒ Error during simple inference: {str(e)}") + print(f"โŒ Error during simple inference: {e!s}") return False if __name__ == "__main__": @@ -153,7 +153,7 @@ def test_simple_inference(): success = test_simple_inference() if success: - print(f"\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") - print(f"๐Ÿ“Š Ready for deployment!") + print("\n๐ŸŽ‰ SUCCESS! Your 99.54% F1 score model is working!") + print("๐Ÿ“Š Ready for deployment!") else: - print(f"\nโŒ Test failed. Check the error messages above.") \ No newline at end of file + print("\nโŒ Test failed. Check the error messages above.") diff --git a/scripts/training/add_advanced_features_to_notebook.py b/scripts/training/add_advanced_features_to_notebook.py index 3f063dc9e..0af941757 100644 --- a/scripts/training/add_advanced_features_to_notebook.py +++ b/scripts/training/add_advanced_features_to_notebook.py @@ -15,7 +15,7 @@ def add_advanced_features(): """Add advanced features to the ultimate notebook.""" # Read the existing notebook - with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Add focal loss implementation @@ -627,4 +627,4 @@ def add_advanced_features(): return 'notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb' if __name__ == "__main__": - add_advanced_features() \ No newline at end of file + add_advanced_features() diff --git a/scripts/training/bulletproof_training.py b/scripts/training/bulletproof_training.py index 70695c761..ac6166e34 100644 --- a/scripts/training/bulletproof_training.py +++ b/scripts/training/bulletproof_training.py @@ -7,10 +7,10 @@ import json import pickle import torch -import torch.nn as nn +from torch import nn +from torch.utils.data import Dataset, DataLoader import pandas as pd from datasets import load_dataset -from torch.utils.data import Dataset, DataLoader from sklearn.model_selection import train_test_split from sklearn.metrics import f1_score, accuracy_score from sklearn.preprocessing import LabelEncoder @@ -54,7 +54,7 @@ def create_unified_label_encoder(): # Load datasets go_emotions = load_dataset("go_emotions", "simplified") - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) @@ -103,7 +103,7 @@ def prepare_filtered_data(label_encoder, label_to_id): # Load datasets go_emotions = load_dataset("go_emotions", "simplified") - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_entries = json.load(f) journal_df = pd.DataFrame(journal_entries) @@ -149,11 +149,11 @@ def prepare_filtered_data(label_encoder, label_to_id): logger.info(f"๐Ÿ“Š Expected range: {expected_range}") if go_label_range[0] < expected_range[0] or go_label_range[1] > expected_range[1]: - logger.error(f"โŒ GoEmotions labels out of range!") + logger.error("โŒ GoEmotions labels out of range!") return None, None, None, None if journal_label_range[0] < expected_range[0] or journal_label_range[1] > expected_range[1]: - logger.error(f"โŒ Journal labels out of range!") + logger.error("โŒ Journal labels out of range!") return None, None, None, None logger.info("โœ… All labels within expected range") @@ -446,4 +446,4 @@ def main(): if __name__ == "__main__": success = main() if not success: - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/scripts/training/complete_simple_notebook.py b/scripts/training/complete_simple_notebook.py index 752ebcb4e..f43838dc6 100644 --- a/scripts/training/complete_simple_notebook.py +++ b/scripts/training/complete_simple_notebook.py @@ -13,7 +13,7 @@ def complete_simple_notebook(): """Add all missing components to the simple notebook.""" # Read the existing notebook - with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Add all the missing cells @@ -488,4 +488,4 @@ def complete_simple_notebook(): print('\\n๐Ÿš€ The notebook is now COMPLETE and ready to use!') if __name__ == "__main__": - complete_simple_notebook() \ No newline at end of file + complete_simple_notebook() diff --git a/scripts/training/comprehensive_domain_adaptation_training.py b/scripts/training/comprehensive_domain_adaptation_training.py index 2abaa2fc5..3716e736c 100644 --- a/scripts/training/comprehensive_domain_adaptation_training.py +++ b/scripts/training/comprehensive_domain_adaptation_training.py @@ -25,7 +25,6 @@ import subprocess import logging from pathlib import Path -from typing import Dict, List, Optional, Tuple, Any, Union from dataclasses import dataclass # Suppress warnings for cleaner output @@ -107,19 +106,19 @@ def install_dependencies(self) -> bool: # Step 1: Clean slate - remove conflicting packages logger.info("๐Ÿงน Cleaning existing packages...") subprocess.run([ - "pip", "uninstall", "torch", "torchvision", "torchaudio", + "pip", "uninstall", "torch", "torchvision", "torchaudio", "transformers", "datasets", "-y" - ], capture_output=True) + ], check=False, capture_output=True) # Step 2: Install PyTorch with compatible CUDA version logger.info("๐Ÿ”ฅ Installing PyTorch with CUDA support...") result = subprocess.run([ - "pip", "install", f"torch=={dependencies['torch']}", - f"torchvision=={dependencies['torchvision']}", + "pip", "install", f"torch=={dependencies['torch']}", + f"torchvision=={dependencies['torchvision']}", f"torchaudio=={dependencies['torchaudio']}", - "--index-url", "https://download.pytorch.org/whl/cu118", + "--index-url", "https://download.pytorch.org/whl/cu118", "--no-cache-dir" - ], capture_output=True, text=True, timeout=600) + ], check=False, capture_output=True, text=True, timeout=600) if result.returncode != 0: logger.error(f"โŒ PyTorch installation failed: {result.stderr}") @@ -128,9 +127,9 @@ def install_dependencies(self) -> bool: # Step 3: Install Transformers with compatible version logger.info("๐Ÿค— Installing Transformers...") result = subprocess.run([ - "pip", "install", f"transformers=={dependencies['transformers']}", + "pip", "install", f"transformers=={dependencies['transformers']}", f"datasets=={dependencies['datasets']}", "--no-cache-dir" - ], capture_output=True, text=True, timeout=300) + ], check=False, capture_output=True, text=True, timeout=300) if result.returncode != 0: logger.error(f"โŒ Transformers installation failed: {result.stderr}") @@ -139,17 +138,17 @@ def install_dependencies(self) -> bool: # Step 4: Install additional dependencies logger.info("๐Ÿ“š Installing additional dependencies...") result = subprocess.run([ - "pip", "install", - f"evaluate=={dependencies['evaluate']}", - f"scikit-learn=={dependencies['scikit-learn']}", - f"pandas=={dependencies['pandas']}", - f"numpy=={dependencies['numpy']}", - f"matplotlib=={dependencies['matplotlib']}", - f"seaborn=={dependencies['seaborn']}", - f"accelerate=={dependencies['accelerate']}", - f"wandb=={dependencies['wandb']}", + "pip", "install", + f"evaluate=={dependencies['evaluate']}", + f"scikit-learn=={dependencies['scikit-learn']}", + f"pandas=={dependencies['pandas']}", + f"numpy=={dependencies['numpy']}", + f"matplotlib=={dependencies['matplotlib']}", + f"seaborn=={dependencies['seaborn']}", + f"accelerate=={dependencies['accelerate']}", + f"wandb=={dependencies['wandb']}", "--no-cache-dir" - ], capture_output=True, text=True, timeout=300) + ], check=False, capture_output=True, text=True, timeout=300) if result.returncode != 0: logger.error(f"โŒ Additional dependencies installation failed: {result.stderr}") @@ -217,7 +216,6 @@ def broadcast_to(array, shape): logger.info(" โœ… Numpy compatibility fix applied") # Try imports again - from transformers import AutoModel, AutoTokenizer logger.info(" โœ… Transformers imports successful after fix") else: raise e @@ -239,7 +237,6 @@ def broadcast_to(array, shape): logger.info("โœ… Numpy compatibility fix applied") # Try verification again - from transformers import AutoModel, AutoTokenizer logger.info("โœ… Transformers imports successful after fix") return True except Exception as fix_error: @@ -261,7 +258,10 @@ def run_command_safe(command: str, description: str) -> bool: """Execute command with comprehensive error handling.""" logger.info(f"๐Ÿ”„ {description}...") try: - result = subprocess.run(command, shell=True, capture_output=True, text=True, timeout=300) + result = subprocess.run( + command, check=False, shell=True, capture_output=True, + text=True, timeout=300 + ) if result.returncode == 0: logger.info(f" โœ… {description} completed") return True @@ -331,7 +331,7 @@ def load_datasets(self) -> bool: logger.info("โœ… GoEmotions dataset loaded") # Load journal dataset - with open('data/journal_test_dataset.json', 'r', encoding='utf-8') as f: + with open('data/journal_test_dataset.json', encoding='utf-8') as f: journal_entries = json.load(f) import pandas as pd @@ -464,9 +464,7 @@ def initialize_model(self, num_labels: int) -> bool: logger.info(f"๐Ÿ—๏ธ Initializing model with {num_labels} labels...") try: - import torch - import torch.nn as nn - from transformers import AutoModel, AutoTokenizer + from transformers import AutoTokenizer # Initialize tokenizer self.tokenizer = AutoTokenizer.from_pretrained(self.config.model_name) @@ -498,7 +496,6 @@ class FocalLoss: """Focal Loss for addressing class imbalance in emotion detection.""" def __init__(self, alpha=1, gamma=2, reduction='mean'): - import torch.nn as nn self.alpha = alpha self.gamma = gamma self.reduction = reduction @@ -531,7 +528,7 @@ def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3) logger.info(f"๐Ÿ—๏ธ Initializing DomainAdaptedEmotionClassifier with num_labels = {num_labels}") try: - import torch.nn as nn + from torch import nn from transformers import AutoModel self.bert = AutoModel.from_pretrained(model_name) @@ -589,7 +586,6 @@ def setup_training(self) -> bool: logger.info("๐ŸŽฏ Setting up training components...") try: - import torch from torch.optim import AdamW from transformers import get_linear_schedule_with_warmup @@ -706,4 +702,4 @@ def main(): if __name__ == "__main__": success = main() if not success: - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/scripts/training/create_bulletproof_colab_notebook.py b/scripts/training/create_bulletproof_colab_notebook.py index 66f7d214a..8bddb2890 100644 --- a/scripts/training/create_bulletproof_colab_notebook.py +++ b/scripts/training/create_bulletproof_colab_notebook.py @@ -714,4 +714,4 @@ def create_bulletproof_colab_notebook(): print(" - Robust error handling") if __name__ == "__main__": - create_bulletproof_colab_notebook() \ No newline at end of file + create_bulletproof_colab_notebook() diff --git a/scripts/training/create_colab_expanded_training.py b/scripts/training/create_colab_expanded_training.py index 59cf5ad5e..27d59775b 100644 --- a/scripts/training/create_colab_expanded_training.py +++ b/scripts/training/create_colab_expanded_training.py @@ -734,4 +734,4 @@ def create_colab_notebook(): print(" 5. Expect 75-85% F1 score!") if __name__ == "__main__": - create_colab_notebook() \ No newline at end of file + create_colab_notebook() diff --git a/scripts/training/create_colab_notebook.py b/scripts/training/create_colab_notebook.py index 44888870b..e939d3f18 100644 --- a/scripts/training/create_colab_notebook.py +++ b/scripts/training/create_colab_notebook.py @@ -673,4 +673,4 @@ def create_colab_notebook(): print(" - Model export for deployment") if __name__ == "__main__": - create_colab_notebook() \ No newline at end of file + create_colab_notebook() diff --git a/scripts/training/create_comprehensive_notebook.py b/scripts/training/create_comprehensive_notebook.py index 53aeac663..c5f4430af 100644 --- a/scripts/training/create_comprehensive_notebook.py +++ b/scripts/training/create_comprehensive_notebook.py @@ -600,4 +600,4 @@ def create_comprehensive_notebook(): return output_path if __name__ == "__main__": - create_comprehensive_notebook() \ No newline at end of file + create_comprehensive_notebook() diff --git a/scripts/training/create_corrected_specialized_notebook.py b/scripts/training/create_corrected_specialized_notebook.py index b3be8ffb6..72361f991 100644 --- a/scripts/training/create_corrected_specialized_notebook.py +++ b/scripts/training/create_corrected_specialized_notebook.py @@ -626,20 +626,20 @@ def create_corrected_notebook(): f.write(notebook_content) print(f"โœ… Created corrected specialized notebook: {notebook_path}") - print(f"๐Ÿ“‹ Key improvements:") - print(f" 1. Verifies access to j-hartmann/emotion-english-distilroberta-base") - print(f" 2. Confirms model architecture (should be DistilRoBERTa with 6 layers)") - print(f" 3. Includes comprehensive reliability testing") - print(f" 4. Saves training info for verification") - print(f" 5. Tests for bias and accuracy before deployment") - print(f"\n๐Ÿš€ Instructions:") - print(f" 1. Download the notebook file") - print(f" 2. Upload to Google Colab") - print(f" 3. Set Runtime โ†’ GPU") - print(f" 4. Run all cells") - print(f" 5. Verify the model is actually using the specialized architecture") - print(f" 6. Only deploy if reliability tests pass") + print("๐Ÿ“‹ Key improvements:") + print(" 1. Verifies access to j-hartmann/emotion-english-distilroberta-base") + print(" 2. Confirms model architecture (should be DistilRoBERTa with 6 layers)") + print(" 3. Includes comprehensive reliability testing") + print(" 4. Saves training info for verification") + print(" 5. Tests for bias and accuracy before deployment") + print("\n๐Ÿš€ Instructions:") + print(" 1. Download the notebook file") + print(" 2. Upload to Google Colab") + print(" 3. Set Runtime โ†’ GPU") + print(" 4. Run all cells") + print(" 5. Verify the model is actually using the specialized architecture") + print(" 6. Only deploy if reliability tests pass") if __name__ == "__main__": create_corrected_notebook() - print("โœ… Corrected specialized notebook created successfully!") \ No newline at end of file + print("โœ… Corrected specialized notebook created successfully!") diff --git a/scripts/training/create_emotion_specialized_notebook.py b/scripts/training/create_emotion_specialized_notebook.py index 031cb1c7e..7ad3b5165 100644 --- a/scripts/training/create_emotion_specialized_notebook.py +++ b/scripts/training/create_emotion_specialized_notebook.py @@ -499,4 +499,4 @@ def create_emotion_specialized_notebook(): print(" - Better hyperparameters") if __name__ == "__main__": - create_emotion_specialized_notebook() \ No newline at end of file + create_emotion_specialized_notebook() diff --git a/scripts/training/create_final_bulletproof_notebook.py b/scripts/training/create_final_bulletproof_notebook.py index d0359a26d..7a2785606 100644 --- a/scripts/training/create_final_bulletproof_notebook.py +++ b/scripts/training/create_final_bulletproof_notebook.py @@ -733,4 +733,4 @@ def create_final_bulletproof_notebook(): print("\n๐ŸŽฏ This should work perfectly now!") if __name__ == "__main__": - create_final_bulletproof_notebook() \ No newline at end of file + create_final_bulletproof_notebook() diff --git a/scripts/training/create_final_colab_notebook.py b/scripts/training/create_final_colab_notebook.py index a400b0c09..3c703d057 100644 --- a/scripts/training/create_final_colab_notebook.py +++ b/scripts/training/create_final_colab_notebook.py @@ -482,4 +482,4 @@ def main(): print(" 5. Expect 75-85% F1 score!") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/training/create_fixed_bulletproof_notebook.py b/scripts/training/create_fixed_bulletproof_notebook.py index 219cd8c78..f3ee61193 100644 --- a/scripts/training/create_fixed_bulletproof_notebook.py +++ b/scripts/training/create_fixed_bulletproof_notebook.py @@ -468,4 +468,4 @@ def create_fixed_bulletproof_notebook(): print(" - Robust error handling") if __name__ == "__main__": - create_fixed_bulletproof_notebook() \ No newline at end of file + create_fixed_bulletproof_notebook() diff --git a/scripts/training/create_fixed_colab_notebook.py b/scripts/training/create_fixed_colab_notebook.py index f30f8ddca..5bb570230 100644 --- a/scripts/training/create_fixed_colab_notebook.py +++ b/scripts/training/create_fixed_colab_notebook.py @@ -453,4 +453,4 @@ def create_fixed_colab_notebook(): print(" 5. Expect 75-85% F1 score!") if __name__ == "__main__": - create_fixed_colab_notebook() \ No newline at end of file + create_fixed_colab_notebook() diff --git a/scripts/training/create_fixed_notebook.py b/scripts/training/create_fixed_notebook.py index db7a1502e..d728c82f8 100644 --- a/scripts/training/create_fixed_notebook.py +++ b/scripts/training/create_fixed_notebook.py @@ -630,20 +630,20 @@ def create_fixed_notebook(): json.dump(notebook, f, indent=1) print(f"โœ… Created fixed specialized notebook: {notebook_path}") - print(f"๐Ÿ“‹ Key improvements:") - print(f" 1. Proper JSON formatting (no syntax errors)") - print(f" 2. Verifies access to j-hartmann/emotion-english-distilroberta-base") - print(f" 3. Confirms model architecture (should be DistilRoBERTa with 6 layers)") - print(f" 4. Includes comprehensive reliability testing") - print(f" 5. Saves training info for verification") - print(f"\n๐Ÿš€ Instructions:") - print(f" 1. Download the notebook file") - print(f" 2. Upload to Google Colab") - print(f" 3. Set Runtime โ†’ GPU") - print(f" 4. Run all cells") - print(f" 5. Verify the model is actually using the specialized architecture") - print(f" 6. Only deploy if reliability tests pass") + print("๐Ÿ“‹ Key improvements:") + print(" 1. Proper JSON formatting (no syntax errors)") + print(" 2. Verifies access to j-hartmann/emotion-english-distilroberta-base") + print(" 3. Confirms model architecture (should be DistilRoBERTa with 6 layers)") + print(" 4. Includes comprehensive reliability testing") + print(" 5. Saves training info for verification") + print("\n๐Ÿš€ Instructions:") + print(" 1. Download the notebook file") + print(" 2. Upload to Google Colab") + print(" 3. Set Runtime โ†’ GPU") + print(" 4. Run all cells") + print(" 5. Verify the model is actually using the specialized architecture") + print(" 6. Only deploy if reliability tests pass") if __name__ == "__main__": create_fixed_notebook() - print("โœ… Fixed specialized notebook created successfully!") \ No newline at end of file + print("โœ… Fixed specialized notebook created successfully!") diff --git a/scripts/training/create_fixed_specialized_training_notebook.py b/scripts/training/create_fixed_specialized_training_notebook.py index 874bdccfe..07dbbc09f 100644 --- a/scripts/training/create_fixed_specialized_training_notebook.py +++ b/scripts/training/create_fixed_specialized_training_notebook.py @@ -680,4 +680,4 @@ def create_fixed_notebook(): return output_path if __name__ == "__main__": - create_fixed_notebook() \ No newline at end of file + create_fixed_notebook() diff --git a/scripts/training/create_improved_expanded_notebook.py b/scripts/training/create_improved_expanded_notebook.py index 84bb4fa86..4a6a50221 100644 --- a/scripts/training/create_improved_expanded_notebook.py +++ b/scripts/training/create_improved_expanded_notebook.py @@ -764,4 +764,4 @@ def create_improved_notebook(): print(" - DataLoader optimizations (num_workers, pin_memory)") if __name__ == "__main__": - create_improved_notebook() \ No newline at end of file + create_improved_notebook() diff --git a/scripts/training/create_minimal_working_notebook.py b/scripts/training/create_minimal_working_notebook.py index 215da793b..17ebb8fef 100644 --- a/scripts/training/create_minimal_working_notebook.py +++ b/scripts/training/create_minimal_working_notebook.py @@ -379,4 +379,4 @@ def create_minimal_notebook(): return output_path if __name__ == "__main__": - create_minimal_notebook() \ No newline at end of file + create_minimal_notebook() diff --git a/scripts/training/create_model_ensemble_notebook.py b/scripts/training/create_model_ensemble_notebook.py index a5ee53d59..b221f7d4d 100644 --- a/scripts/training/create_model_ensemble_notebook.py +++ b/scripts/training/create_model_ensemble_notebook.py @@ -674,4 +674,4 @@ def create_model_ensemble_notebook(): print(" - Optimized hyperparameters") if __name__ == "__main__": - create_model_ensemble_notebook() \ No newline at end of file + create_model_ensemble_notebook() diff --git a/scripts/training/create_simple_ultimate_notebook.py b/scripts/training/create_simple_ultimate_notebook.py index 91af37aa3..da3d3610b 100644 --- a/scripts/training/create_simple_ultimate_notebook.py +++ b/scripts/training/create_simple_ultimate_notebook.py @@ -414,4 +414,4 @@ def create_simple_notebook(): return output_path if __name__ == "__main__": - create_simple_notebook() \ No newline at end of file + create_simple_notebook() diff --git a/scripts/training/create_ultimate_bulletproof_notebook.py b/scripts/training/create_ultimate_bulletproof_notebook.py index ccba22de0..2f99cff3f 100644 --- a/scripts/training/create_ultimate_bulletproof_notebook.py +++ b/scripts/training/create_ultimate_bulletproof_notebook.py @@ -417,4 +417,4 @@ def create_ultimate_notebook(): return output_path if __name__ == "__main__": - create_ultimate_notebook() \ No newline at end of file + create_ultimate_notebook() diff --git a/scripts/training/debug_colab_compatibility.py b/scripts/training/debug_colab_compatibility.py index 5f3b9b784..e2b097758 100644 --- a/scripts/training/debug_colab_compatibility.py +++ b/scripts/training/debug_colab_compatibility.py @@ -19,7 +19,9 @@ def run_command(command, description): """Run a command and return success status.""" print(f"๐Ÿ”ง {description}...") try: - result = subprocess.run(command, shell=True, capture_output=True, text=True) + result = subprocess.run( + command, check=False, shell=True, capture_output=True, text=True + ) if result.returncode == 0: print(f"โœ… {description} successful") return True, result.stdout @@ -52,7 +54,7 @@ def check_gpu_availability(): print(f"PyTorch version: {torch.__version__}") if torch.cuda.is_available(): - print(f"โœ… CUDA available") + print("โœ… CUDA available") print(f"GPU: {torch.cuda.get_device_name(0)}") print(f"Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB") print(f"CUDA version: {torch.version.cuda}") @@ -238,7 +240,7 @@ def check_dataset_loading(): # Test journal dataset import json - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_data = json.load(f) print(f"โœ… Journal dataset loaded: {len(journal_data)} samples") @@ -318,4 +320,4 @@ def main(): print(" 3. Check the Colab GPU development guide") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/training/final_combined_training.py b/scripts/training/final_combined_training.py index 0d278c1a2..3a3ec0775 100644 --- a/scripts/training/final_combined_training.py +++ b/scripts/training/final_combined_training.py @@ -18,9 +18,9 @@ import torch from torch.utils.data import Dataset from transformers import ( - AutoTokenizer, - AutoModelForSequenceClassification, - TrainingArguments, + AutoTokenizer, + AutoModelForSequenceClassification, + TrainingArguments, Trainer, EarlyStoppingCallback ) @@ -41,7 +41,7 @@ def load_combined_dataset(): # Load original journal dataset (150 high-quality samples) try: - with open('data/journal_test_dataset.json', 'r') as f: + with open('data/journal_test_dataset.json') as f: journal_data = json.load(f) for item in journal_data: @@ -56,7 +56,7 @@ def load_combined_dataset(): # Load CMU-MOSEI dataset try: - with open('data/cmu_mosei_balanced_dataset.json', 'r') as f: + with open('data/cmu_mosei_balanced_dataset.json') as f: cmu_data = json.load(f) for item in cmu_data: @@ -71,7 +71,7 @@ def load_combined_dataset(): # Load expanded journal dataset as backup try: - with open('data/expanded_journal_dataset.json', 'r') as f: + with open('data/expanded_journal_dataset.json') as f: expanded_data = json.load(f) # Only use a subset to avoid synthetic data issues @@ -268,8 +268,8 @@ def main(): print("๐ŸŽ‰ Training completed!") print(f"๐Ÿ“ˆ Final F1 Score: {results['eval_f1']*100:.2f}%") - print(f"๐ŸŽฏ Target: 75-85%") + print("๐ŸŽฏ Target: 75-85%") print(f"๐Ÿ“Š Improvement: {((results['eval_f1'] - 0.67) / 0.67 * 100):.1f}% from baseline") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/training/final_expanded_training.py b/scripts/training/final_expanded_training.py index 435792b4d..576e8003a 100644 --- a/scripts/training/final_expanded_training.py +++ b/scripts/training/final_expanded_training.py @@ -16,9 +16,9 @@ import torch from torch.utils.data import Dataset from transformers import ( - AutoTokenizer, - AutoModelForSequenceClassification, - TrainingArguments, + AutoTokenizer, + AutoModelForSequenceClassification, + TrainingArguments, Trainer, EarlyStoppingCallback ) @@ -33,7 +33,7 @@ # Load expanded dataset print("๐Ÿ“Š Loading expanded dataset...") -with open('data/expanded_journal_dataset.json', 'r') as f: +with open('data/expanded_journal_dataset.json') as f: expanded_data = json.load(f) print(f"โœ… Loaded {len(expanded_data)} expanded samples") @@ -92,7 +92,7 @@ def __getitem__(self, idx): model_name = "bert-base-uncased" tokenizer = AutoTokenizer.from_pretrained(model_name) model = AutoModelForSequenceClassification.from_pretrained( - model_name, + model_name, num_labels=num_labels, problem_type="single_label_classification" ) @@ -154,7 +154,7 @@ def compute_metrics(eval_pred): # Evaluate on test set print("๐Ÿงช Evaluating model...") results = trainer.evaluate() -print(f"๐Ÿ“Š Final Results:") +print("๐Ÿ“Š Final Results:") print(f" F1 Score: {results['eval_f1']:.4f} ({results['eval_f1']*100:.1f}%)") print(f" Accuracy: {results['eval_accuracy']:.4f} ({results['eval_accuracy']*100:.1f}%)") @@ -180,7 +180,7 @@ def compute_metrics(eval_pred): "I'm content with how things are going." ] -expected_emotions = ['happy', 'frustrated', 'anxious', 'grateful', 'overwhelmed', +expected_emotions = ['happy', 'frustrated', 'anxious', 'grateful', 'overwhelmed', 'proud', 'sad', 'excited', 'calm', 'hopeful', 'tired', 'content'] print("๐Ÿ“Š Testing Results:") @@ -213,7 +213,7 @@ def compute_metrics(eval_pred): print(f" Predicted: {predicted_emotion} (confidence: {confidence:.3f})") print(f" Expected: {expected}") print(f" {'โœ… CORRECT' if is_correct else 'โŒ WRONG'}") - print(f" Top 3 predictions:") + print(" Top 3 predictions:") for emotion, prob in zip(top_3_emotions, top_3_probs): print(f" - {emotion}: {prob:.3f}") print() @@ -221,17 +221,17 @@ def compute_metrics(eval_pred): test_accuracy = correct_predictions / len(test_samples) final_f1 = results['eval_f1'] -print(f"\n๐Ÿ“ˆ FINAL RESULTS:") +print("\n๐Ÿ“ˆ FINAL RESULTS:") print(f" Test Accuracy: {test_accuracy:.2%} ({correct_predictions}/{len(test_samples)})") print(f" F1 Score: {final_f1:.4f} ({final_f1*100:.1f}%)") print(f" Target Achieved: {'โœ… YES!' if final_f1 >= 0.75 else 'โŒ Not yet'}") if final_f1 >= 0.75: print(f"\n๐ŸŽ‰ SUCCESS! Model achieved {final_f1*100:.1f}% F1 score!") - print(f"๐Ÿš€ Ready for production deployment!") + print("๐Ÿš€ Ready for production deployment!") else: print(f"\n๐Ÿ“ˆ Good progress! Current F1: {final_f1*100:.1f}%") - print(f"๐Ÿ’ก Consider: more data, hyperparameter tuning, or different model architecture") + print("๐Ÿ’ก Consider: more data, hyperparameter tuning, or different model architecture") -print(f"\n๐Ÿ’พ Model saved to: ./best_emotion_model_final") -print(f"๐Ÿ“Š Training completed successfully!") \ No newline at end of file +print("\n๐Ÿ’พ Model saved to: ./best_emotion_model_final") +print("๐Ÿ“Š Training completed successfully!") diff --git a/scripts/training/fix_imports_in_notebook.py b/scripts/training/fix_imports_in_notebook.py index b65d8d307..8f4e749b0 100644 --- a/scripts/training/fix_imports_in_notebook.py +++ b/scripts/training/fix_imports_in_notebook.py @@ -13,7 +13,7 @@ def fix_imports(): """Add missing imports to the ultimate notebook.""" # Read the existing notebook - with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Find the imports cell and update it @@ -50,4 +50,4 @@ def fix_imports(): print(' โœ… CUDA availability check') if __name__ == "__main__": - fix_imports() \ No newline at end of file + fix_imports() diff --git a/scripts/training/fix_notebook_json.py b/scripts/training/fix_notebook_json.py index c3ff9a2f0..cfc8cb0e1 100644 --- a/scripts/training/fix_notebook_json.py +++ b/scripts/training/fix_notebook_json.py @@ -9,7 +9,7 @@ def fix_notebook_json(): """Fix JSON syntax errors in the notebook.""" # Read the notebook as text - with open('notebooks/expanded_dataset_training.ipynb', 'r') as f: + with open('notebooks/expanded_dataset_training.ipynb') as f: content = f.read() # Fix unescaped quotes in strings @@ -45,11 +45,11 @@ def fix_notebook_json(): # Test if the JSON is valid try: import json - with open('notebooks/expanded_dataset_training_fixed.ipynb', 'r') as f: + with open('notebooks/expanded_dataset_training_fixed.ipynb') as f: json.load(f) print("โœ… JSON syntax is now valid") except Exception as e: print(f"โŒ JSON still has issues: {e}") if __name__ == "__main__": - fix_notebook_json() \ No newline at end of file + fix_notebook_json() diff --git a/scripts/training/fix_preprocessing_in_notebook.py b/scripts/training/fix_preprocessing_in_notebook.py index 1bc9eae51..e70c00fcc 100644 --- a/scripts/training/fix_preprocessing_in_notebook.py +++ b/scripts/training/fix_preprocessing_in_notebook.py @@ -13,7 +13,7 @@ def fix_preprocessing(): """Fix the preprocessing function in the ultimate notebook.""" # Read the existing notebook - with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Find and replace the preprocessing cell @@ -139,4 +139,4 @@ def fix_preprocessing(): print(' โœ… Updated trainer initialization with data collator') if __name__ == "__main__": - fix_preprocessing() \ No newline at end of file + fix_preprocessing() diff --git a/scripts/training/fix_training_arguments.py b/scripts/training/fix_training_arguments.py index a9dcebb1b..5314268a9 100644 --- a/scripts/training/fix_training_arguments.py +++ b/scripts/training/fix_training_arguments.py @@ -13,7 +13,7 @@ def fix_training_arguments(): """Fix the training arguments in the simple notebook.""" # Read the existing notebook - with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/SIMPLE_ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) # Find and replace the training arguments cell @@ -55,4 +55,4 @@ def fix_training_arguments(): print(' โœ… Kept all other parameters intact') if __name__ == "__main__": - fix_training_arguments() \ No newline at end of file + fix_training_arguments() diff --git a/scripts/training/improve_expanded_training_notebook.py b/scripts/training/improve_expanded_training_notebook.py index 60273cc1b..9fb2cc516 100644 --- a/scripts/training/improve_expanded_training_notebook.py +++ b/scripts/training/improve_expanded_training_notebook.py @@ -11,7 +11,7 @@ def improve_notebook(): """Improve the expanded training notebook with enhancements.""" # Read the current notebook - with open('notebooks/expanded_dataset_training.ipynb', 'r') as f: + with open('notebooks/expanded_dataset_training.ipynb') as f: notebook = json.load(f) # Find the training function cell @@ -120,4 +120,4 @@ def improve_notebook(): print(" - Better memory management") if __name__ == "__main__": - improve_notebook() \ No newline at end of file + improve_notebook() diff --git a/scripts/training/robust_domain_adaptation_training.py b/scripts/training/robust_domain_adaptation_training.py index f605aee6f..adac4989c 100644 --- a/scripts/training/robust_domain_adaptation_training.py +++ b/scripts/training/robust_domain_adaptation_training.py @@ -13,7 +13,7 @@ import warnings import subprocess from pathlib import Path -from typing import Dict, List, Optional, Tuple, Any +from typing import Dict, List, Optional # Suppress warnings for cleaner output warnings.filterwarnings('ignore') @@ -40,26 +40,26 @@ def setup_environment(): # Step 1: Clean slate - remove conflicting packages subprocess.run([ - "pip", "uninstall", "torch", "torchvision", "torchaudio", + "pip", "uninstall", "torch", "torchvision", "torchaudio", "transformers", "datasets", "-y" - ], capture_output=True) + ], check=False, capture_output=True) # Step 2: Install PyTorch with compatible CUDA version subprocess.run([ "pip", "install", "torch==2.1.0", "torchvision==0.16.0", "torchaudio==2.1.0", "--index-url", "https://download.pytorch.org/whl/cu118", "--no-cache-dir" - ]) + ], check=False) # Step 3: Install Transformers with compatible version subprocess.run([ "pip", "install", "transformers==4.30.0", "datasets==2.13.0", "--no-cache-dir" - ]) + ], check=False) # Step 4: Install additional dependencies subprocess.run([ - "pip", "install", "evaluate", "scikit-learn", "pandas", "numpy", + "pip", "install", "evaluate", "scikit-learn", "pandas", "numpy", "matplotlib", "seaborn", "accelerate", "wandb", "--no-cache-dir" - ]) + ], check=False) print("โœ… Dependencies installed successfully") return is_colab @@ -84,7 +84,6 @@ def verify_installation(): print("โš ๏ธ No GPU available. Training will be slow on CPU.") # Test critical imports - from transformers import AutoModel, AutoTokenizer print(" โœ… Transformers imports successful") return True @@ -101,7 +100,9 @@ def run_command(command: str, description: str) -> bool: """Execute command with error handling.""" print(f"๐Ÿ”„ {description}...") try: - result = subprocess.run(command, shell=True, capture_output=True, text=True) + result = subprocess.run( + command, check=False, shell=True, capture_output=True, text=True + ) if result.returncode == 0: print(f" โœ… {description} completed") return True @@ -140,7 +141,7 @@ def safe_load_dataset(dataset_name: str, config: Optional[str] = None, split: Op def safe_load_json(file_path: str): """Safely load JSON file with error handling.""" try: - with open(file_path, 'r') as f: + with open(file_path) as f: data = json.load(f) print(f"โœ… Successfully loaded {file_path}") return data @@ -220,7 +221,6 @@ class FocalLoss: """Focal Loss for addressing class imbalance in emotion detection.""" def __init__(self, alpha=1, gamma=2, reduction='mean'): - import torch.nn as nn import torch.nn.functional as F self.alpha = alpha self.gamma = gamma @@ -243,7 +243,7 @@ class DomainAdaptedEmotionClassifier: """BERT-based emotion classifier with domain adaptation capabilities.""" def __init__(self, model_name="bert-base-uncased", num_labels=None, dropout=0.3): - import torch.nn as nn + from torch import nn from transformers import AutoModel # ROBUST: Validate num_labels @@ -307,7 +307,6 @@ def safe_model_initialization(model_name: str, num_labels: int, device: str): model = DomainAdaptedEmotionClassifier(model_name=model_name, num_labels=num_labels) # Move to device - import torch model = model.to(device) print(f"โœ… Model moved to {device}") @@ -360,4 +359,4 @@ def main(): print(" 4. Evaluate and save results") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/training/setup_colab_environment.py b/scripts/training/setup_colab_environment.py index e33c1902a..4fbb221a6 100644 --- a/scripts/training/setup_colab_environment.py +++ b/scripts/training/setup_colab_environment.py @@ -36,7 +36,7 @@ def install_dependencies(): # Core ML dependencies packages = [ "torch>=2.1.0,<2.2.0", - "torchvision>=0.16.0,<0.17.0", + "torchvision>=0.16.0,<0.17.0", "torchaudio>=2.1.0,<2.2.0", "transformers>=4.30.0,<5.0.0", "datasets>=2.10.0,<3.0.0", @@ -65,7 +65,7 @@ def install_dependencies(): for package in packages: try: logger.info(f"๐Ÿ“ฆ Installing {package}...") - subprocess.run([sys.executable, "-m", "pip", "install", package], + subprocess.run([sys.executable, "-m", "pip", "install", package], check=True, capture_output=True, text=True) logger.info(f"โœ… {package} installed successfully") except subprocess.CalledProcessError as e: @@ -226,7 +226,7 @@ def run_ci_pipeline(): try: result = subprocess.run( [sys.executable, "scripts/ci/run_full_ci_pipeline.py"], - capture_output=True, + check=False, capture_output=True, text=True, timeout=600 # 10 minute timeout ) @@ -288,4 +288,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/training/summarize_comprehensive_notebook.py b/scripts/training/summarize_comprehensive_notebook.py index fdaf4daca..7f7d1261d 100644 --- a/scripts/training/summarize_comprehensive_notebook.py +++ b/scripts/training/summarize_comprehensive_notebook.py @@ -13,7 +13,7 @@ def summarize_comprehensive_notebook(): """Summarize the comprehensive notebook.""" # Read the notebook - with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) print("๐Ÿš€ COMPREHENSIVE ULTIMATE TRAINING NOTEBOOK SUMMARY") @@ -24,7 +24,7 @@ def summarize_comprehensive_notebook(): markdown_cells = [cell for cell in notebook['cells'] if cell['cell_type'] == 'markdown'] code_cells = [cell for cell in notebook['cells'] if cell['cell_type'] == 'code'] - print(f"๐Ÿ“Š NOTEBOOK STATISTICS:") + print("๐Ÿ“Š NOTEBOOK STATISTICS:") print(f" Total cells: {len(notebook['cells'])}") print(f" Markdown cells: {len(markdown_cells)}") print(f" Code cells: {len(code_cells)}") @@ -101,10 +101,10 @@ def summarize_comprehensive_notebook(): print() print("๐Ÿ“ FILE LOCATION:") - print(f" notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb") + print(" notebooks/COMPREHENSIVE_ULTIMATE_TRAINING_COLAB.ipynb") print() print("๐Ÿš€ READY TO USE!") print(" Download, upload to Colab, set GPU runtime, and run!") if __name__ == "__main__": - summarize_comprehensive_notebook() \ No newline at end of file + summarize_comprehensive_notebook() diff --git a/scripts/training/summarize_ultimate_notebook.py b/scripts/training/summarize_ultimate_notebook.py index d6c83271e..d3d29c8c9 100644 --- a/scripts/training/summarize_ultimate_notebook.py +++ b/scripts/training/summarize_ultimate_notebook.py @@ -16,7 +16,7 @@ def summarize_notebook(): print() # Read the notebook - with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb', 'r') as f: + with open('notebooks/ULTIMATE_BULLETPROOF_TRAINING_COLAB.ipynb') as f: notebook = json.load(f) print("๐Ÿ“‹ NOTEBOOK OVERVIEW:") @@ -93,4 +93,4 @@ def summarize_notebook(): print(" Ready for production deployment") if __name__ == "__main__": - summarize_notebook() \ No newline at end of file + summarize_notebook() diff --git a/scripts/training/validate_improved_notebook.py b/scripts/training/validate_improved_notebook.py index eda4c6a03..8cf69d137 100644 --- a/scripts/training/validate_improved_notebook.py +++ b/scripts/training/validate_improved_notebook.py @@ -13,7 +13,7 @@ def validate_notebook(): # Load the notebook try: - with open('notebooks/expanded_dataset_training_improved.ipynb', 'r') as f: + with open('notebooks/expanded_dataset_training_improved.ipynb') as f: notebook = json.load(f) print("โœ… Notebook JSON is valid") except Exception as e: @@ -108,7 +108,7 @@ def validate_notebook(): all_passed = False # Summary - print(f"\n๐Ÿ“Š Validation Summary:") + print("\n๐Ÿ“Š Validation Summary:") print(f" Total cells: {len(cells)}") print(f" Code cells: {len(code_cells)}") print(f" Markdown cells: {len(markdown_cells)}") @@ -127,4 +127,4 @@ def validate_notebook(): return all_passed if __name__ == "__main__": - validate_notebook() \ No newline at end of file + validate_notebook() diff --git a/scripts/validation/check_dependencies.py b/scripts/validation/check_dependencies.py index f1f8149d8..da5fc8830 100644 --- a/scripts/validation/check_dependencies.py +++ b/scripts/validation/check_dependencies.py @@ -9,7 +9,7 @@ import re import sys from pathlib import Path -from typing import Set, List, Dict +from typing import Set class DependencyChecker: """Checker for dependency usage in the codebase.""" @@ -46,7 +46,7 @@ def _parse_requirements(self) -> Set[str]: """Parse requirements.txt and extract package names.""" deps = set() - with open(self.requirements_path, 'r') as f: + with open(self.requirements_path) as f: for line in f: line = line.strip() if line and not line.startswith('#'): @@ -78,7 +78,7 @@ def _find_used_dependencies(self) -> Set[str]: def _scan_file_for_imports(self, file_path: Path, used_deps: Set[str]) -> None: """Scan a Python file for import statements.""" try: - with open(file_path, 'r', encoding='utf-8') as f: + with open(file_path, encoding='utf-8') as f: content = f.read() # Find import statements @@ -104,7 +104,7 @@ def _scan_file_for_imports(self, file_path: Path, used_deps: Set[str]) -> None: def print_results(self) -> None: """Print dependency check results.""" - print(f"\n๐Ÿ“Š Dependency Usage Check Results") + print("\n๐Ÿ“Š Dependency Usage Check Results") print("=" * 50) if self.unused_deps: @@ -137,4 +137,4 @@ def main(): return 1 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/scripts/validation/validate_security_config.py b/scripts/validation/validate_security_config.py index 9d438eee0..d066d5e14 100644 --- a/scripts/validation/validate_security_config.py +++ b/scripts/validation/validate_security_config.py @@ -9,7 +9,7 @@ import yaml import sys from pathlib import Path -from typing import Dict, Any, List +from typing import Dict, Any class SecurityConfigValidator: """Validator for security configuration files.""" @@ -29,7 +29,7 @@ def validate(self) -> bool: return False try: - with open(self.config_path, 'r') as f: + with open(self.config_path) as f: config = yaml.safe_load(f) except yaml.YAMLError as e: self.errors.append(f"Invalid YAML in security configuration: {e}") @@ -219,7 +219,7 @@ def _validate_deployment_security(self, deploy_config: Dict[str, Any]) -> None: def print_results(self) -> None: """Print validation results.""" - print(f"\n๐Ÿ“Š Security Configuration Validation Results") + print("\n๐Ÿ“Š Security Configuration Validation Results") print("=" * 50) if self.errors: @@ -254,4 +254,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/api_rate_limiter.py b/src/api_rate_limiter.py index 8ec1995bf..1813aaff9 100644 --- a/src/api_rate_limiter.py +++ b/src/api_rate_limiter.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿ”’ API Rate Limiter +"""๐Ÿ”’ API Rate Limiter ================== Token bucket algorithm for API rate limiting. Includes security features. @@ -147,8 +146,7 @@ async def dispatch(self, request, call_next): # type: ignore[override] class TokenBucketRateLimiter: - """ - Token bucket rate limiter with security enhancements. + """Token bucket rate limiter with security enhancements. Features: - Token bucket algorithm for smooth rate limiting @@ -374,8 +372,7 @@ def allow_request( client_ip: str, user_agent: str = "", ) -> Tuple[bool, str, dict]: - """ - Check if request should be allowed. + """Check if request should be allowed. Returns: Tuple of (allowed, reason, metadata) diff --git a/src/data/pipeline.py b/src/data/pipeline.py index 51468e168..c4eccdb6f 100644 --- a/src/data/pipeline.py +++ b/src/data/pipeline.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Data Pipeline for SAMO Deep Learning. +"""Data Pipeline for SAMO Deep Learning. This module provides data processing pipelines for text and audio data, including preprocessing, feature extraction, and dataset management. @@ -9,7 +8,7 @@ import logging from datetime import datetime, timezone from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, Optional, Union import pandas as pd from .feature_engineering import FeatureEngineer from .validation import DataValidator diff --git a/src/data/preprocessing.py b/src/data/preprocessing.py index bffe26c5a..e501b694c 100644 --- a/src/data/preprocessing.py +++ b/src/data/preprocessing.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -Text Preprocessing Module for SAMO Deep Learning. +"""Text Preprocessing Module for SAMO Deep Learning. This module provides comprehensive text preprocessing functionality for journal entries and other text data. diff --git a/src/input_sanitizer.py b/src/input_sanitizer.py index bf72befe9..f6d1e7cd7 100644 --- a/src/input_sanitizer.py +++ b/src/input_sanitizer.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -๐Ÿงน Input Sanitizer +"""๐Ÿงน Input Sanitizer ================= Comprehensive input sanitization and validation for API security. """ @@ -8,7 +7,7 @@ import re import html import logging -from typing import Any, Dict, List, Optional, Union, Tuple +from typing import Any, Dict, List, Tuple from dataclasses import dataclass import unicodedata @@ -29,8 +28,7 @@ class SanitizationConfig: enable_content_type_validation: bool = True class InputSanitizer: - """ - Comprehensive input sanitization and validation. + """Comprehensive input sanitization and validation. Features: - XSS protection @@ -89,8 +87,7 @@ def __init__(self, config: SanitizationConfig): } def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[str]]: - """ - Sanitize text input. + """Sanitize text input. Args: text: Input text to sanitize @@ -106,7 +103,9 @@ def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[ # Check length if len(text) > self.config.max_text_length: - warnings.append(f"Text truncated from {len(text)} to {self.config.max_text_length} characters") + warnings.append( + f"Text truncated from {len(text)} to {self.config.max_text_length} characters" + ) text = text[:self.config.max_text_length] # Unicode normalization @@ -114,7 +113,8 @@ def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[ text = unicodedata.normalize('NFKC', text) # Check for blocked patterns - if self.config.enable_xss_protection or self.config.enable_sql_injection_protection: + if (self.config.enable_xss_protection or + self.config.enable_sql_injection_protection): for pattern in self.config.blocked_patterns: if re.search(pattern, text, re.IGNORECASE): warnings.append(f"Blocked pattern detected: {pattern}") @@ -134,8 +134,7 @@ def sanitize_text(self, text: str, context: str = "general") -> Tuple[str, List[ return text, warnings def sanitize_json(self, data: Any, max_depth: int = 10) -> Tuple[Any, List[str]]: - """ - Sanitize JSON data recursively. + """Sanitize JSON data recursively. Args: data: JSON data to sanitize @@ -168,8 +167,7 @@ def _sanitize_recursive(obj: Any, depth: int = 0) -> Any: return _sanitize_recursive(data), warnings def validate_emotion_request(self, data: Dict) -> Tuple[Dict, List[str]]: - """ - Validate and sanitize emotion detection request. + """Validate and sanitize emotion detection request. Args: data: Request data @@ -206,8 +204,7 @@ def validate_emotion_request(self, data: Dict) -> Tuple[Dict, List[str]]: return sanitized_data, warnings def validate_batch_request(self, data: Dict) -> Tuple[Dict, List[str]]: - """ - Validate and sanitize batch emotion detection request. + """Validate and sanitize batch emotion detection request. Args: data: Request data @@ -258,8 +255,7 @@ def validate_batch_request(self, data: Dict) -> Tuple[Dict, List[str]]: return sanitized_data, warnings def validate_content_type(self, content_type: str) -> bool: - """ - Validate content type header. + """Validate content type header. Args: content_type: Content type header value @@ -277,8 +273,7 @@ def validate_content_type(self, content_type: str) -> bool: return True def sanitize_headers(self, headers: Dict[str, str]) -> Tuple[Dict[str, str], List[str]]: - """ - Sanitize HTTP headers. + """Sanitize HTTP headers. Args: headers: HTTP headers @@ -305,8 +300,7 @@ def sanitize_headers(self, headers: Dict[str, str]) -> Tuple[Dict[str, str], Lis return sanitized_headers, warnings def detect_anomalies(self, data: Any) -> List[str]: - """ - Detect potential security anomalies in data. + """Detect potential security anomalies in data. Args: data: Data to analyze diff --git a/src/models/emotion_detection/bert_classifier.py b/src/models/emotion_detection/bert_classifier.py index 8c67223cf..dc2c2462d 100644 --- a/src/models/emotion_detection/bert_classifier.py +++ b/src/models/emotion_detection/bert_classifier.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -BERT-based Emotion Classifier for SAMO Deep Learning. +"""BERT-based Emotion Classifier for SAMO Deep Learning. This module provides a BERT-based multi-label emotion classification model trained on the GoEmotions dataset for journal entry analysis. @@ -12,10 +11,10 @@ import numpy as np import torch -import torch.nn as nn +from torch import nn +from torch.utils.data import Dataset, DataLoader import torch.nn.functional as F from sklearn.metrics import f1_score, precision_recall_fscore_support -from torch.utils.data import Dataset, DataLoader from transformers import AutoConfig, AutoModel, AutoTokenizer from .labels import GOEMOTIONS_EMOTIONS diff --git a/src/models/emotion_detection/dataset_loader.py b/src/models/emotion_detection/dataset_loader.py index 94d04862c..af3f3b7c5 100644 --- a/src/models/emotion_detection/dataset_loader.py +++ b/src/models/emotion_detection/dataset_loader.py @@ -27,7 +27,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -from .labels import GOEMOTIONS_EMOTIONS, EMOTION_ID_TO_LABEL, EMOTION_LABEL_TO_ID +from .labels import GOEMOTIONS_EMOTIONS class GoEmotionsDataset(Dataset): diff --git a/src/models/emotion_detection/labels.py b/src/models/emotion_detection/labels.py index 289e2556f..a9d35341a 100644 --- a/src/models/emotion_detection/labels.py +++ b/src/models/emotion_detection/labels.py @@ -33,4 +33,4 @@ ] EMOTION_ID_TO_LABEL = dict(enumerate(GOEMOTIONS_EMOTIONS)) -EMOTION_LABEL_TO_ID = {emotion: i for i, emotion in enumerate(GOEMOTIONS_EMOTIONS)} \ No newline at end of file +EMOTION_LABEL_TO_ID = {emotion: i for i, emotion in enumerate(GOEMOTIONS_EMOTIONS)} diff --git a/src/models/secure_loader/__init__.py b/src/models/secure_loader/__init__.py index d419401a6..625339478 100644 --- a/src/models/secure_loader/__init__.py +++ b/src/models/secure_loader/__init__.py @@ -1,5 +1,4 @@ -""" -Secure Model Loader Module for SAMO Deep Learning. +"""Secure Model Loader Module for SAMO Deep Learning. This module provides secure model loading capabilities with defense-in-depth against PyTorch RCE vulnerabilities and other security threats. @@ -11,8 +10,8 @@ from .model_validator import ModelValidator __all__ = [ - "SecureModelLoader", "IntegrityChecker", + "ModelValidator", "SandboxExecutor", - "ModelValidator" + "SecureModelLoader" ] diff --git a/src/models/secure_loader/integrity_checker.py b/src/models/secure_loader/integrity_checker.py index 4099edc2e..3b6b9098e 100644 --- a/src/models/secure_loader/integrity_checker.py +++ b/src/models/secure_loader/integrity_checker.py @@ -1,5 +1,4 @@ -""" -Model Integrity Checker for Secure Model Loading. +"""Model Integrity Checker for Secure Model Loading. This module provides integrity verification capabilities for model files, including checksums, digital signatures, and format validation. @@ -55,7 +54,7 @@ def _load_trusted_checksums(self) -> Dict[str, str]: return {} try: - with open(self.trusted_checksums_file, 'r') as f: + with open(self.trusted_checksums_file) as f: return json.load(f) except Exception as e: logger.error(f"Failed to load trusted checksums: {e}") diff --git a/src/models/secure_loader/model_validator.py b/src/models/secure_loader/model_validator.py index 7ba280679..b64822d65 100644 --- a/src/models/secure_loader/model_validator.py +++ b/src/models/secure_loader/model_validator.py @@ -1,5 +1,4 @@ -""" -Model Validator for Secure Model Loading. +"""Model Validator for Secure Model Loading. This module provides model validation capabilities including: - Model structure validation @@ -9,10 +8,10 @@ """ import logging import os -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple import torch -import torch.nn as nn +from torch import nn logger = logging.getLogger(__name__) @@ -235,8 +234,6 @@ def validate_version_compatibility(self, model_config: Dict[str, Any]) -> Tuple[ } try: - # Get current versions - import torch import transformers validation_info['current_versions'] = { diff --git a/src/models/secure_loader/sandbox_executor.py b/src/models/secure_loader/sandbox_executor.py index bcd30a963..97b44f37f 100644 --- a/src/models/secure_loader/sandbox_executor.py +++ b/src/models/secure_loader/sandbox_executor.py @@ -1,5 +1,4 @@ -""" -Sandbox Executor for Secure Model Loading. +"""Sandbox Executor for Secure Model Loading. This module provides sandboxed execution capabilities for model loading, preventing potential RCE vulnerabilities and malicious code execution. diff --git a/src/models/secure_loader/secure_model_loader.py b/src/models/secure_loader/secure_model_loader.py index c78c52180..9e2b153da 100644 --- a/src/models/secure_loader/secure_model_loader.py +++ b/src/models/secure_loader/secure_model_loader.py @@ -1,5 +1,4 @@ -""" -Secure Model Loader for SAMO Deep Learning. +"""Secure Model Loader for SAMO Deep Learning. This module provides the main secure model loading interface that integrates all security components: integrity checking, sandboxed execution, and validation. @@ -8,10 +7,10 @@ import logging import os import time -from typing import Any, Dict, Optional, Tuple, Type, Union +from typing import Any, Dict, Optional, Tuple, Type import torch -import torch.nn as nn +from torch import nn from .integrity_checker import IntegrityChecker from .sandbox_executor import SandboxExecutor diff --git a/src/models/summarization/t5_summarizer.py b/src/models/summarization/t5_summarizer.py index 5742a8e70..c4fbc2643 100644 --- a/src/models/summarization/t5_summarizer.py +++ b/src/models/summarization/t5_summarizer.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -""" -T5-based Text Summarization for SAMO Deep Learning. +"""T5-based Text Summarization for SAMO Deep Learning. This module provides T5-based text summarization capabilities for journal entries and other text content. @@ -9,10 +8,10 @@ import logging import warnings from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional import torch -import torch.nn as nn +from torch import nn from torch.utils.data import Dataset from transformers import ( AutoModelForSeq2SeqLM, diff --git a/src/monitoring/dashboard.py b/src/monitoring/dashboard.py index 035a58d48..beb2cebe3 100644 --- a/src/monitoring/dashboard.py +++ b/src/monitoring/dashboard.py @@ -1,5 +1,4 @@ -""" -Comprehensive Monitoring Dashboard for SAMO Deep Learning API +"""Comprehensive Monitoring Dashboard for SAMO Deep Learning API This module provides real-time monitoring capabilities including: - System resource monitoring @@ -9,11 +8,8 @@ - Performance metrics visualization """ -import asyncio -import json import logging import time -from datetime import datetime, timedelta from typing import Dict, List, Optional, Any from dataclasses import dataclass, asdict from collections import defaultdict, deque diff --git a/src/security_setup.py b/src/security_setup.py index 39c851c72..db2b74274 100644 --- a/src/security_setup.py +++ b/src/security_setup.py @@ -1,18 +1,15 @@ #!/usr/bin/env python3 -""" -๐Ÿ”’ Shared Security Setup +"""๐Ÿ”’ Shared Security Setup ======================== Common security configuration and middleware setup for deployment scripts. """ import os -from typing import Optional from security_headers import SecurityHeadersMiddleware, SecurityHeadersConfig def create_security_config(environment: str = "development") -> SecurityHeadersConfig: - """ - Create security configuration based on environment. + """Create security configuration based on environment. Args: environment: Environment name ('development', 'testing', 'production') @@ -45,8 +42,7 @@ def create_security_config(environment: str = "development") -> SecurityHeadersC def setup_security_middleware( app, environment: str = "development" ) -> SecurityHeadersMiddleware: - """ - Set up security headers middleware for a Flask app. + """Set up security headers middleware for a Flask app. Args: app: Flask application instance @@ -69,8 +65,7 @@ def setup_security_middleware( def get_environment() -> str: - """ - Determine current environment from environment variables. + """Determine current environment from environment variables. Returns: Environment name ('development', 'testing', 'production') diff --git a/src/unified_ai_api.py b/src/unified_ai_api.py index d39ec4e6c..3f3a778f9 100644 --- a/src/unified_ai_api.py +++ b/src/unified_ai_api.py @@ -1843,7 +1843,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query return except Exception as e: - await websocket.close(code=4001, reason=f"Authentication failed: {str(e)}") + await websocket.close(code=4001, reason=f"Authentication failed: {e!s}") return await websocket.accept() @@ -1878,7 +1878,7 @@ async def websocket_realtime_processing(websocket: WebSocket, token: str = Query logger.info("WebSocket authenticated for user: %s", payload.username) - except Exception as exc: + except Exception: await websocket.send_json({ "type": "error", "message": "Authentication failed" diff --git a/src/utils.py b/src/utils.py index 509717b8f..d8f3caf40 100644 --- a/src/utils.py +++ b/src/utils.py @@ -2,7 +2,6 @@ """Utility functions for the SAMO-DL project.""" import torch -from typing import Union def count_model_params(model: torch.nn.Module, only_trainable: bool = False) -> int: diff --git a/tests/integration/test_priority1_features.py b/tests/integration/test_priority1_features.py index 417f014cd..15eb77b5a 100644 --- a/tests/integration/test_priority1_features.py +++ b/tests/integration/test_priority1_features.py @@ -9,16 +9,13 @@ 5. Comprehensive Monitoring Dashboard """ -import asyncio -import json import os import tempfile from pathlib import Path import time -from typing import Dict, Any import pytest from fastapi.testclient import TestClient -from unittest.mock import Mock, patch +from unittest.mock import patch from src.unified_ai_api import app from src.security.jwt_manager import JWTManager @@ -1013,4 +1010,4 @@ def test_blacklist_token_cleanup(self): assert cleaned_count >= 0 # May or may not have expired tokens if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file + pytest.main([__file__]) diff --git a/tests/unit/test_admin_endpoints.py b/tests/unit/test_admin_endpoints.py index 632b9c7b0..5993ecfce 100644 --- a/tests/unit/test_admin_endpoints.py +++ b/tests/unit/test_admin_endpoints.py @@ -117,4 +117,4 @@ def test_admin_endpoints_missing_ip(self): self.assertIn('IP address required', response.get_json()['error']) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_anomaly_detection.py b/tests/unit/test_anomaly_detection.py index 0841eba08..679279c62 100644 --- a/tests/unit/test_anomaly_detection.py +++ b/tests/unit/test_anomaly_detection.py @@ -251,4 +251,4 @@ def test_anomaly_detection_performance(self): self.assertLess(processing_time, 1.0, f"Anomaly detection too slow: {processing_time:.3f}s") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_api_rate_limiter.py b/tests/unit/test_api_rate_limiter.py index 040d9ca01..8f0fef016 100644 --- a/tests/unit/test_api_rate_limiter.py +++ b/tests/unit/test_api_rate_limiter.py @@ -56,7 +56,7 @@ def test_allow_request_success(self): def test_allow_request_rate_limit_exceeded(self): """Test that allow_request returns False when rate limit exceeded.""" config = RateLimitConfig( - requests_per_minute=1, + requests_per_minute=1, burst_size=1, enable_user_agent_analysis=False, # Disable abuse detection for testing enable_request_pattern_analysis=False diff --git a/tests/unit/test_api_security.py b/tests/unit/test_api_security.py index ef4fadfb7..0ca0b76cd 100644 --- a/tests/unit/test_api_security.py +++ b/tests/unit/test_api_security.py @@ -455,4 +455,4 @@ def test_security_violation_handling(self): if __name__ == '__main__': # Run tests - unittest.main(verbosity=2) \ No newline at end of file + unittest.main(verbosity=2) diff --git a/tests/unit/test_csp_config.py b/tests/unit/test_csp_config.py index d5c4f9938..b5a8db9da 100644 --- a/tests/unit/test_csp_config.py +++ b/tests/unit/test_csp_config.py @@ -211,7 +211,7 @@ def test_enhanced_csp_policy_directives(self): # Test all directives in a single loop for directive, description in required_directives: - self.assertIn(directive, csp_policy, + self.assertIn(directive, csp_policy, f"Missing CSP directive: {description} ({directive})") def test_csp_policy_production_ready(self): @@ -231,8 +231,8 @@ def test_csp_policy_production_ready(self): # Test all production security features in a single loop for directive, description in production_security: - self.assertIn(directive, csp_policy, + self.assertIn(directive, csp_policy, f"Production security missing: {description} ({directive})") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_hash_security.py b/tests/unit/test_hash_security.py index 9df898345..f039eef2e 100644 --- a/tests/unit/test_hash_security.py +++ b/tests/unit/test_hash_security.py @@ -202,4 +202,4 @@ def test_special_characters_in_user_agent(self): self.fail("Client key with special characters is not a valid hex string") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_sandbox_executor.py b/tests/unit/test_sandbox_executor.py index d1d65ac6d..339a05c52 100644 --- a/tests/unit/test_sandbox_executor.py +++ b/tests/unit/test_sandbox_executor.py @@ -179,4 +179,4 @@ def network_function(): self.assertIn('error', meta) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_secure_model_loader.py b/tests/unit/test_secure_model_loader.py index f770129a9..ef1e77723 100644 --- a/tests/unit/test_secure_model_loader.py +++ b/tests/unit/test_secure_model_loader.py @@ -14,7 +14,7 @@ import unittest import torch -import torch.nn as nn +from torch import nn from src.models.secure_loader import ( SecureModelLoader, @@ -487,10 +487,10 @@ def test_audit_logging(self): self.assertTrue(os.path.exists(audit_log_path)) # Check audit log contains entries - with open(audit_log_path, 'r') as f: + with open(audit_log_path) as f: log_content = f.read() self.assertIn('AUDIT:', log_content) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/test_security_integration.py b/tests/unit/test_security_integration.py index 2f083d3e9..4bac98dc3 100644 --- a/tests/unit/test_security_integration.py +++ b/tests/unit/test_security_integration.py @@ -64,11 +64,11 @@ def test_comprehensive_security_headers(self): ] # Test all headers with consistent validation - for header, validation in required_headers: - self.assertIn(header, response.headers, f"Missing security header: {header}") - self.assertIsInstance(response.headers[header], str, f"Header {header} should be string") + for header_name, validation in required_headers: + self.assertIn(header_name, response.headers, f"Missing security header: {header_name}") + self.assertIsInstance(response.headers[header_name], str, f"Header {header_name} should be string") if validation == 'non-empty': - self.assertGreater(len(response.headers[header]), 0, f"Header {header} should not be empty") + self.assertGreater(len(response.headers[header_name]), 0, f"Header {header_name} should not be empty") def test_csp_policy_default_src(self): """Test that CSP policy includes default-src directive.""" diff --git a/tests/unit/test_validation_enhanced.py b/tests/unit/test_validation_enhanced.py index 8c530ac23..070cfccea 100644 --- a/tests/unit/test_validation_enhanced.py +++ b/tests/unit/test_validation_enhanced.py @@ -36,7 +36,7 @@ def test_check_missing_values_basic(self): def test_check_missing_values_with_required_columns(self): """Test missing values check with required columns.""" missing_stats = self.validator.check_missing_values( - self.test_df, + self.test_df, required_columns=['user_id', 'content'] ) @@ -203,4 +203,4 @@ def test_validate_text_input_invalid_types(self): # Test with non-string result = validate_text_input(123) - assert result['is_valid'] is False + assert result['is_valid'] is False