mirror of
https://github.com/mindverse/Second-Me.git
synced 2026-09-08 02:47:02 +00:00
f5bb0dad59
* Add code for data filtering llm judge * Ignore log file created on root (mainly for synthetic_data_generation.log) * Fix metadata API compatibility issues by commenting out metadata tags in LLM API calls - Commented out metadata.tags parameters in all LLM API calls across the codebase - This fixes compatibility issues with custom LLM providers that don't support metadata - Affects shades generation, topics generation, wiki generation, bio QA, and question generation - Preserves the original code structure for future re-enabling if needed * feat: add data filtering pipeline with Ollama integration - Add MergedDataJudge class for intelligent data filtering using Ollama Gemma - Integrate automatic Ollama CLI installation into project setup process - Add DATA_FILTERING step to training pipeline with concurrent processing - Include testing for MergedDataJudge in its local main() function - Add Ollama dependency to pyproject.toml * feat: add automatic Ollama model cleanup after data filtering * Add logging for outputting data filtering parameters * fix: adjust error handling for MergedDataJudge: - Keep original merged.json unchanged when any error occurs - Exit filtering process immediately on errors instead of continuing with defaults - Ensure training pipeline continues safely even if data filtering fails * Add frontend for data filtering pipeline * resolve data filtering quality_level error by commenting out problematic fields, change TrainProcessService back to original class definition * fix: quote unquoted shade icons to prevent JSON parsing errors * Fixed wiki_res.json missing due to no database connection at wiki/base.py module import * Added scoring reasoning as part of the merged data * fix: filter ANSI escape sequences from Ollama logs in data filtering step * fix: Add data filtering steps to cloud training to resolve KeyError - Added 'Data Filtering' step to cloud training progress holder - Added data filtering step execution in cloud training service - Added data filtering parameters to cloud training routes - Updated frontend to send data filtering parameters - Fixed missing except clause in cloud training service This resolves the KeyError: 'data_filtering' when switching from cloud to local training.
63 lines
2.1 KiB
Python
63 lines
2.1 KiB
Python
from enum import Enum
|
|
from typing import List
|
|
|
|
|
|
class ProcessStep(Enum):
|
|
"""Training process steps"""
|
|
|
|
LIST_DOCUMENTS = "list_documents"
|
|
GENERATE_DOCUMENT_EMBEDDINGS = "generate_document_embeddings"
|
|
CHUNK_DOCUMENT = "process_chunks"
|
|
CHUNK_EMBEDDING = "chunk_embedding"
|
|
EXTRACT_DIMENSIONAL_TOPICS = "extract_dimensional_topics"
|
|
GENERATE_SHADES = "generate_shades"
|
|
GENERATE_BIOGRAPHY = "generate_biography"
|
|
MODEL_DOWNLOAD = "model_download"
|
|
|
|
GENERATE_BASE = "generate_base"
|
|
BIO_QA_GENERATION = "bio_qa_generation"
|
|
WIKI_DATA_GENERATION = "wiki_data_generation"
|
|
GENERATE_MEMQA_ENTITY = "generate_memqa_entity"
|
|
GENERATE_MEMQA_RELATION = "generate_memqa_relation"
|
|
GENERATE_MEMQA_DESCRIPTION = "generate_memqa_description"
|
|
GENERATE_MEMQA_DIVERSITY = "generate_memqa_diversity"
|
|
SYNTHETIC_DATA_GENERATION = "synthetic_data_generation"
|
|
SYNTHETIC_NO_NOTES_DATA_GENERATION = "synthetic_no_notes_data_generation"
|
|
CONVERT_DATA = "convert_data"
|
|
DATA_FILTERING = "data_filtering"
|
|
|
|
TRAIN = "train"
|
|
MERGE_WEIGHTS = "merge_weights"
|
|
CONVERT_MODEL = "convert_model"
|
|
|
|
@classmethod
|
|
def get_ordered_steps(cls) -> List["ProcessStep"]:
|
|
"""Get ordered steps"""
|
|
return [
|
|
cls.MODEL_DOWNLOAD,
|
|
cls.LIST_DOCUMENTS,
|
|
cls.GENERATE_DOCUMENT_EMBEDDINGS,
|
|
cls.CHUNK_DOCUMENT,
|
|
cls.CHUNK_EMBEDDING,
|
|
cls.EXTRACT_DIMENSIONAL_TOPICS,
|
|
cls.GENERATE_SHADES,
|
|
cls.GENERATE_BIOGRAPHY,
|
|
cls.GENERATE_BASE,
|
|
cls.BIO_QA_GENERATION,
|
|
cls.WIKI_DATA_GENERATION,
|
|
cls.GENERATE_MEMQA_ENTITY,
|
|
cls.GENERATE_MEMQA_RELATION,
|
|
cls.GENERATE_MEMQA_DESCRIPTION,
|
|
cls.GENERATE_MEMQA_DIVERSITY,
|
|
cls.SYNTHETIC_DATA_GENERATION,
|
|
cls.SYNTHETIC_NO_NOTES_DATA_GENERATION,
|
|
cls.CONVERT_DATA,
|
|
cls.DATA_FILTERING,
|
|
cls.TRAIN,
|
|
cls.MERGE_WEIGHTS,
|
|
cls.CONVERT_MODEL,
|
|
]
|
|
|
|
def get_method_name(self) -> str:
|
|
"""Get the corresponding method name for this step"""
|
|
return self.value
|