mirror of
https://github.com/mindverse/Second-Me.git
synced 2026-08-27 09:31:42 +00:00
f5bb0dad59
* Add code for data filtering llm judge * Ignore log file created on root (mainly for synthetic_data_generation.log) * Fix metadata API compatibility issues by commenting out metadata tags in LLM API calls - Commented out metadata.tags parameters in all LLM API calls across the codebase - This fixes compatibility issues with custom LLM providers that don't support metadata - Affects shades generation, topics generation, wiki generation, bio QA, and question generation - Preserves the original code structure for future re-enabling if needed * feat: add data filtering pipeline with Ollama integration - Add MergedDataJudge class for intelligent data filtering using Ollama Gemma - Integrate automatic Ollama CLI installation into project setup process - Add DATA_FILTERING step to training pipeline with concurrent processing - Include testing for MergedDataJudge in its local main() function - Add Ollama dependency to pyproject.toml * feat: add automatic Ollama model cleanup after data filtering * Add logging for outputting data filtering parameters * fix: adjust error handling for MergedDataJudge: - Keep original merged.json unchanged when any error occurs - Exit filtering process immediately on errors instead of continuing with defaults - Ensure training pipeline continues safely even if data filtering fails * Add frontend for data filtering pipeline * resolve data filtering quality_level error by commenting out problematic fields, change TrainProcessService back to original class definition * fix: quote unquoted shade icons to prevent JSON parsing errors * Fixed wiki_res.json missing due to no database connection at wiki/base.py module import * Added scoring reasoning as part of the merged data * fix: filter ANSI escape sequences from Ollama logs in data filtering step * fix: Add data filtering steps to cloud training to resolve KeyError - Added 'Data Filtering' step to cloud training progress holder - Added data filtering step execution in cloud training service - Added data filtering parameters to cloud training routes - Updated frontend to send data filtering parameters - Fixed missing except clause in cloud training service This resolves the KeyError: 'data_filtering' when switching from cloud to local training.
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
# common/logging.py
|
|
import logging
|
|
import logging.config
|
|
import os
|
|
import sys
|
|
import re
|
|
from datetime import datetime
|
|
from typing import Optional
|
|
from lpm_kernel.configs.logging import LOGGING_CONFIG, LOG_BASE_DIR, TRAIN_LOG_DIR, rename_existing_log_file
|
|
|
|
|
|
def setup_logging():
|
|
try:
|
|
# Ensure log directories exist
|
|
os.makedirs(LOG_BASE_DIR, exist_ok=True)
|
|
os.makedirs(TRAIN_LOG_DIR, exist_ok=True)
|
|
|
|
# Rename existing log file if needed
|
|
rename_existing_log_file()
|
|
# Ensure directory permissions are correct
|
|
os.chmod(TRAIN_LOG_DIR, 0o755)
|
|
os.chmod(LOG_BASE_DIR, 0o755)
|
|
|
|
print(f"Log directory: {TRAIN_LOG_DIR}", file=sys.stderr)
|
|
print(
|
|
f"Log file: {LOGGING_CONFIG['handlers']['file']['filename']}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
except Exception as e:
|
|
print(f"Error creating log directory: {e}", file=sys.stderr)
|
|
# If unable to create directory, use standard output
|
|
LOGGING_CONFIG["handlers"]["file"] = LOGGING_CONFIG["handlers"]["console"]
|
|
|
|
try:
|
|
# Configure logging
|
|
logging.config.dictConfig(LOGGING_CONFIG)
|
|
root_logger = logging.getLogger()
|
|
root_logger.info("Logging system initialized successfully")
|
|
print(f"Log level: {root_logger.getEffectiveLevel()}", file=sys.stderr)
|
|
print(
|
|
f"Log handlers: {[h.__class__.__name__ for h in root_logger.handlers]}",
|
|
file=sys.stderr,
|
|
)
|
|
except Exception as e:
|
|
print(f"Error configuring logging: {e}", file=sys.stderr)
|
|
# If configuration fails, use basic configuration
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d - %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
|
|
# Get module logger
|
|
logger = logging.getLogger(__name__)
|
|
logger.info("Logging module initialization complete")
|
|
return logger
|
|
|
|
|
|
# Initialize global logger
|
|
logger = setup_logging()
|
|
|
|
|
|
def clean_ansi_sequences(text: str) -> str:
|
|
"""
|
|
Remove ANSI escape sequences from text.
|
|
|
|
Args:
|
|
text: Text that may contain ANSI escape sequences
|
|
|
|
Returns:
|
|
Text with ANSI escape sequences removed
|
|
"""
|
|
# ANSI escape sequence regex pattern
|
|
ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
|
|
return ansi_escape.sub('', text)
|