fix: filter ANSI escape sequences from Ollama logs in data filtering step

lpm_frontend/src/components/train/TrainingLog.tsx

@@ -13,6 +13,26 @@ const TrainingLog: React.FC<TrainingLogProps> = ({ trainingDetails }: TrainingLo
   const userScrollTimeout = useRef<NodeJS.Timeout | null>(null);
   const [isAutoScrollEnabled, setIsAutoScrollEnabled] = useState(true);
 
+  // Function to clean ANSI escape sequences from text
+  const cleanAnsiSequences = (text: string): string => {
+    // More comprehensive ANSI escape sequence regex pattern
+    // This handles various ANSI escape sequences including:
+    // - Cursor movement: \x1b[nA, \x1b[nB, \x1b[nC, \x1b[nD
+    // - Clear screen: \x1b[K, \x1b[J
+    // - Color codes: \x1b[38;5;n, \x1b[48;5;n, \x1b[0m, etc.
+    // - Progress bar characters: █, ▓, ▒, ░, ▏, ▕
+    const ansiEscape = /\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g;
+    
+    // Remove ANSI escape sequences
+    let cleaned = text.replace(ansiEscape, '');
+    
+    // Also remove common progress bar characters that might cause display issues
+    const progressChars = /[█▓▒░▏▕]/g;
+    cleaned = cleaned.replace(progressChars, '=');
+    
+    return cleaned;
+  };
+
   // Smooth scroll console to bottom
   const smoothScrollConsole = () => {
     if (consoleEndRef.current && !isUserScrolling) {
@@ -108,7 +128,7 @@ const TrainingLog: React.FC<TrainingLogProps> = ({ trainingDetails }: TrainingLo
           {trainingDetails.length > 0 ? (
             trainingDetails.map((detail, index) => (
               <div key={detail.timestamp + detail.message + index} className="text-gray-300">
-                {detail.message}
+                {cleanAnsiSequences(detail.message)}
               </div>
             ))
           ) : (

lpm_kernel/L2/utils.py

@@ -767,6 +767,7 @@ def save_ollama_model(model_name=None, log_file_path=None) -> str:
     import subprocess
     import time
     from lpm_kernel.configs.logging import TRAIN_LOG_FILE
+    from lpm_kernel.common.logging import clean_ansi_sequences
 
     if not model_name:
         raise ValueError("Ollama model_name must be specified, e.g. 'gemini:2b'")
@@ -781,12 +782,37 @@ def save_ollama_model(model_name=None, log_file_path=None) -> str:
         with open(log_file_path, "a") as logf:
             process = subprocess.Popen([
                 "ollama", "pull", model_name
-            ], stdout=logf, stderr=logf)
+            ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1, universal_newlines=True)
+            
+            # Read output in real-time and clean ANSI sequences
             while True:
                 ret = process.poll()
                 if ret is not None:
                     break
-                time.sleep(1)
+                
+                # Read available output
+                if process.stdout:
+                    line = process.stdout.readline()
+                    if line:
+                        # Clean ANSI sequences and write to log
+                        cleaned_line = clean_ansi_sequences(line)
+                        if cleaned_line.strip():  # Only write non-empty lines
+                            logf.write(cleaned_line)
+                            logf.flush()  # Ensure immediate write
+                
+                time.sleep(0.1)  # Small delay to prevent CPU spinning
+            
+            # Read any remaining output
+            remaining_output, remaining_error = process.communicate()
+            if remaining_output:
+                cleaned_output = clean_ansi_sequences(remaining_output)
+                if cleaned_output.strip():
+                    logf.write(cleaned_output)
+            if remaining_error:
+                cleaned_error = clean_ansi_sequences(remaining_error)
+                if cleaned_error.strip():
+                    logf.write(cleaned_error)
+                    
         if process.returncode == 0:
             logger.info(f"Ollama model '{model_name}' pulled successfully.")
         else:

lpm_kernel/common/logging.py

@@ -3,6 +3,9 @@ import logging
 import logging.config
 import os
 import sys
+import re
+from datetime import datetime
+from typing import Optional
 from lpm_kernel.configs.logging import LOGGING_CONFIG, LOG_BASE_DIR, TRAIN_LOG_DIR, rename_existing_log_file
 
 
@@ -56,3 +59,18 @@ def setup_logging():
 
 # Initialize global logger
 logger = setup_logging()
+
+
+def clean_ansi_sequences(text: str) -> str:
+    """
+    Remove ANSI escape sequences from text.
+    
+    Args:
+        text: Text that may contain ANSI escape sequences
+        
+    Returns:
+        Text with ANSI escape sequences removed
+    """
+    # ANSI escape sequence regex pattern
+    ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+    return ansi_escape.sub('', text)