From 4a5d47d93411e3b5b104fc494db30242376d5def Mon Sep 17 00:00:00 2001 From: LUIS NOVO Date: Mon, 18 Nov 2024 22:01:11 -0300 Subject: [PATCH] refactor transformation, add graph and admin --- migrations/5.surrealql | 11 ++ migrations/5_down.surrealql | 2 + open_notebook/config.py | 1 - open_notebook/database/repository.py | 14 +- open_notebook/domain/transformation.py | 26 ++-- open_notebook/graphs/multipattern.py | 59 ------- open_notebook/graphs/prompt.py | 46 ++++++ open_notebook/graphs/source.py | 49 +++--- open_notebook/graphs/transformation.py | 57 +++++++ open_notebook/graphs/utils.py | 23 --- open_notebook/models/llms.py | 19 +-- pages/8_💱_Transformations.py | 147 ++++++++++++++++++ pages/8_🛝_Playground.py | 42 ----- pages/components/source_panel.py | 25 +-- pages/stream_app/source.py | 11 +- pages/stream_app/utils.py | 6 - prompts/patterns/default/analyze_paper.jinja | 39 ----- prompts/patterns/default/cleanup.jinja | 7 - prompts/patterns/default/command.jinja | 6 - .../common_tranformation_instructions.jinja | 4 - prompts/patterns/default/keyinsights.jinja | 27 ---- prompts/patterns/default/makeitdense.jinja | 13 -- prompts/patterns/default/mermaid.jinja | 27 ---- .../default/reflection_questions.jinja | 23 --- prompts/patterns/default/summarize.jinja | 13 -- prompts/patterns/default/toc.jinja | 13 -- 26 files changed, 326 insertions(+), 384 deletions(-) create mode 100644 migrations/5.surrealql create mode 100644 migrations/5_down.surrealql delete mode 100644 open_notebook/graphs/multipattern.py create mode 100644 open_notebook/graphs/prompt.py create mode 100644 open_notebook/graphs/transformation.py create mode 100644 pages/8_💱_Transformations.py delete mode 100644 pages/8_🛝_Playground.py delete mode 100644 prompts/patterns/default/analyze_paper.jinja delete mode 100644 prompts/patterns/default/cleanup.jinja delete mode 100644 prompts/patterns/default/command.jinja delete mode 100644 prompts/patterns/default/common_tranformation_instructions.jinja delete mode 100644 prompts/patterns/default/keyinsights.jinja delete mode 100644 prompts/patterns/default/makeitdense.jinja delete mode 100644 prompts/patterns/default/mermaid.jinja delete mode 100644 prompts/patterns/default/reflection_questions.jinja delete mode 100644 prompts/patterns/default/summarize.jinja delete mode 100644 prompts/patterns/default/toc.jinja diff --git a/migrations/5.surrealql b/migrations/5.surrealql new file mode 100644 index 0000000..159f347 --- /dev/null +++ b/migrations/5.surrealql @@ -0,0 +1,11 @@ + +DEFINE TABLE IF NOT EXISTS transformation SCHEMAFULL; + +DEFINE FIELD IF NOT EXISTS name ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS title ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS description ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS prompt ON TABLE transformation TYPE string; +DEFINE FIELD IF NOT EXISTS apply_default ON TABLE transformation TYPE bool DEFAULT False; +DEFINE FIELD IF NOT EXISTS created ON transformation DEFAULT time::now() VALUE $before OR time::now(); +DEFINE FIELD IF NOT EXISTS updated ON transformation DEFAULT time::now() VALUE time::now(); + diff --git a/migrations/5_down.surrealql b/migrations/5_down.surrealql new file mode 100644 index 0000000..a9e3eb2 --- /dev/null +++ b/migrations/5_down.surrealql @@ -0,0 +1,2 @@ + +REMOVE TABLE IF EXISTS transformation SCHEMAFULL; diff --git a/open_notebook/config.py b/open_notebook/config.py index 6d2d655..450680a 100644 --- a/open_notebook/config.py +++ b/open_notebook/config.py @@ -16,7 +16,6 @@ except Exception: CONFIG = {} # ROOT DATA FOLDER -# todo: make this configurable once podcastfy supports it DATA_FOLDER = "./data" # LANGGRAPH CHECKPOINT FILE diff --git a/open_notebook/database/repository.py b/open_notebook/database/repository.py index aa442b0..f514c74 100644 --- a/open_notebook/database/repository.py +++ b/open_notebook/database/repository.py @@ -37,7 +37,6 @@ def repo_query(query_str: str, vars: Optional[Dict[str, Any]] = None): def repo_create(table: str, data: Dict[str, Any]): query = f"CREATE {table} CONTENT {data};" - # vars = {"table": table, "data": data} return repo_query(query) @@ -53,14 +52,7 @@ def repo_delete(id: str): return repo_query(query, vars) -def repo_relate(source: str, relationship: str, target: str): - query = f"RELATE {source}->{relationship}->{target}" - # CONTENT $content;" - # vars = { - # "source": source, - # "relationship": relationship, - # "target": target, - # # "content": {}, # You can add properties to the relation here if needed - # } - result = repo_query(query) +def repo_relate(source: str, relationship: str, target: str, data: Optional[Dict] = {}): + query = f"RELATE {source}->{relationship}->{target} CONTENT $content;" + result = repo_query(query, {"content": data}) return result diff --git a/open_notebook/domain/transformation.py b/open_notebook/domain/transformation.py index 6662365..4e97976 100644 --- a/open_notebook/domain/transformation.py +++ b/open_notebook/domain/transformation.py @@ -1,19 +1,21 @@ -from typing import ClassVar, List, Optional +from typing import ClassVar, Optional -import yaml from pydantic import Field -from open_notebook.domain.base import RecordModel +from open_notebook.domain.base import ObjectModel, RecordModel -class Transformation: - @classmethod - def get_all(cls): - with open("transformations.yaml", "r") as file: - transformations = yaml.safe_load(file) - return transformations +class Transformation(ObjectModel): + table_name: ClassVar[str] = "transformation" + name: str + title: str + description: str + prompt: str + apply_default: bool -class DefaultTransformations(RecordModel): - record_id: ClassVar[str] = "open_notebook:default_transformations" - source_insights: Optional[List[str]] = Field(default_factory=list) +class DefaultPrompts(RecordModel): + record_id: ClassVar[str] = "open_notebook:default_prompts" + transformation_instructions: Optional[str] = Field( + None, description="Instructions for executing a transformation" + ) diff --git a/open_notebook/graphs/multipattern.py b/open_notebook/graphs/multipattern.py deleted file mode 100644 index 9b95638..0000000 --- a/open_notebook/graphs/multipattern.py +++ /dev/null @@ -1,59 +0,0 @@ -import operator -from typing import List, Literal, Sequence - -from langchain_core.runnables import ( - RunnableConfig, -) -from langgraph.graph import END, START, StateGraph -from typing_extensions import Annotated, TypedDict - -from open_notebook.graphs.utils import run_pattern - - -class PatternChainState(TypedDict): - content_stack: Annotated[Sequence[str], operator.add] - patterns: List[str] - output: str - - -def call_model(state: dict, config: RunnableConfig) -> dict: - patterns = state["patterns"] - current_transformation = patterns.pop(0) - if current_transformation.startswith("patterns/"): - input_args = {"input_text": state["content_stack"][-1]} - else: - input_args = { - "input_text": state["content_stack"][-1], - "command": current_transformation, - } - current_transformation = "patterns/default/command" - - transformation_result = run_pattern( - pattern_name=current_transformation, - config=config, - state=input_args, - ) - return { - "content_stack": [transformation_result.content], - "output": transformation_result.content, - "patterns": state["patterns"], - } - - -def transform_condition(state: PatternChainState) -> Literal["agent", END]: # type: ignore - """ - Checks whether there are more chunks to process. - """ - if len(state["patterns"]) > 0: - return "agent" - return END - - -agent_state = StateGraph(PatternChainState) -agent_state.add_node("agent", call_model) -agent_state.add_edge(START, "agent") -agent_state.add_conditional_edges( - "agent", - transform_condition, -) -graph = agent_state.compile() diff --git a/open_notebook/graphs/prompt.py b/open_notebook/graphs/prompt.py new file mode 100644 index 0000000..176576c --- /dev/null +++ b/open_notebook/graphs/prompt.py @@ -0,0 +1,46 @@ +from typing import Any, Optional + +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.runnables import ( + RunnableConfig, +) +from langgraph.graph import END, START, StateGraph +from loguru import logger +from typing_extensions import TypedDict + +from open_notebook.graphs.utils import provision_langchain_model +from open_notebook.prompter import Prompter + + +class PatternChainState(TypedDict): + prompt: str + parser: Optional[Any] + input_text: str + output: str + + +def call_model(state: dict, config: RunnableConfig) -> dict: + content = state["input_text"] + system_prompt = Prompter( + prompt_text=state["prompt"], parser=state.get("parser") + ).render(data=state) + logger.warning(content) + payload = [SystemMessage(content=system_prompt)] + [HumanMessage(content=content)] + chain = provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id"), + "transformation", + max_tokens=5000, + ) + + response = chain.invoke(payload) + + return {"output": response.content} + + +agent_state = StateGraph(PatternChainState) +agent_state.add_node("agent", call_model) +agent_state.add_edge(START, "agent") +agent_state.add_edge("agent", END) + +graph = agent_state.compile() diff --git a/open_notebook/graphs/source.py b/open_notebook/graphs/source.py index d3e3f23..10c2e0e 100644 --- a/open_notebook/graphs/source.py +++ b/open_notebook/graphs/source.py @@ -1,5 +1,5 @@ import operator -from typing import List +from typing import List, Optional from langchain_core.runnables import ( RunnableConfig, @@ -13,22 +13,22 @@ from open_notebook.domain.notebook import Asset, Source from open_notebook.domain.transformation import Transformation from open_notebook.graphs.content_processing import ContentState from open_notebook.graphs.content_processing import graph as content_graph -from open_notebook.graphs.multipattern import graph as transform_graph +from open_notebook.graphs.transformation import graph as transform_graph from open_notebook.utils import surreal_clean class SourceState(TypedDict): content_state: ContentState - transformations: List[str] + apply_transformations: List[Transformation] notebook_id: str source: Source - transformations: Annotated[list, operator.add] + transformation: Annotated[list, operator.add] embed: bool class TransformationState(TypedDict): source: Source - transformation: dict + transformation: Transformation async def content_process(state: SourceState) -> dict: @@ -38,13 +38,6 @@ async def content_process(state: SourceState) -> dict: return {"content_state": processed_state} -async def run_patterns(input_text: str, patterns: List[dict]) -> str: - output = await transform_graph.ainvoke( - dict(content_stack=[input_text], patterns=patterns) - ) - return output["output"] - - def save_source(state: SourceState) -> dict: content_state = state["content_state"] @@ -69,15 +62,10 @@ def save_source(state: SourceState) -> dict: def trigger_transformations(state: SourceState, config: RunnableConfig) -> List[Send]: - if len(state["transformations"]) == 0: + if len(state["apply_transformations"]) == 0: return [] - transformations = Transformation.get_all() - to_apply = [ - t - for t in transformations["source_insights"] - if t["name"] in state["transformations"] - ] + to_apply = state["apply_transformations"] logger.debug(f"Applying transformations {to_apply}") return [ @@ -92,19 +80,26 @@ def trigger_transformations(state: SourceState, config: RunnableConfig) -> List[ ] -async def transform_content(state: TransformationState) -> dict: +async def transform_content(state: TransformationState) -> Optional[dict]: source = state["source"] content = source.full_text if not content: return None - transformation = state["transformation"] + transformation: Transformation = state["transformation"] - logger.debug(f"Applying transformation {transformation['name']}") - result = await run_patterns(content, patterns=transformation["patterns"]) - - source.add_insight(transformation["name"], surreal_clean(result)) - - return {"transformations": [{"name": transformation["name"], "content": result}]} + logger.debug(f"Applying transformation {transformation.name}") + result = await transform_graph.ainvoke( + dict(input_text=content, transformation=transformation) + ) + source.add_insight(transformation.title, surreal_clean(result["output"])) + return { + "transformation": [ + { + "output": result["output"], + "transformation_name": transformation.name, + } + ] + } # Create and compile the workflow diff --git a/open_notebook/graphs/transformation.py b/open_notebook/graphs/transformation.py new file mode 100644 index 0000000..f125c20 --- /dev/null +++ b/open_notebook/graphs/transformation.py @@ -0,0 +1,57 @@ +from executing import Source +from langchain_core.messages import HumanMessage, SystemMessage +from langchain_core.runnables import ( + RunnableConfig, +) +from langgraph.graph import END, START, StateGraph +from typing_extensions import TypedDict + +from open_notebook.domain.transformation import DefaultPrompts, Transformation +from open_notebook.graphs.utils import provision_langchain_model +from open_notebook.prompter import Prompter + + +class TransformationState(TypedDict): + input_text: str + source: Source + transformation: Transformation + output: str + + +def run_transformation(state: dict, config: RunnableConfig) -> dict: + source: Source = state.get("source") + content = state.get("input_text") + assert source or content, "No content to transform" + transformation: Transformation = state["transformation"] + if not content: + content = source.full_text + transformation_prompt_text = transformation.prompt + default_prompts: DefaultPrompts = DefaultPrompts().load() + if default_prompts.transformation_instructions: + transformation_prompt_text = f"{default_prompts.transformation_instructions}\n\n{transformation_prompt_text}" + + transformation_prompt_text = f"{transformation_prompt_text}\n\n# INPUT" + + system_prompt = Prompter(prompt_text=transformation_prompt_text).render(data=state) + payload = [SystemMessage(content=system_prompt)] + [HumanMessage(content=content)] + chain = provision_langchain_model( + str(payload), + config.get("configurable", {}).get("model_id"), + "transformation", + max_tokens=5000, + ) + + response = chain.invoke(payload) + if source: + source.add_insight(transformation.title, response.content) + + return { + "output": response.content, + } + + +agent_state = StateGraph(TransformationState) +agent_state.add_node("agent", run_transformation) +agent_state.add_edge(START, "agent") +agent_state.add_edge("agent", END) +graph = agent_state.compile() diff --git a/open_notebook/graphs/utils.py b/open_notebook/graphs/utils.py index e8be911..ced6ed0 100644 --- a/open_notebook/graphs/utils.py +++ b/open_notebook/graphs/utils.py @@ -1,10 +1,8 @@ from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from loguru import logger from open_notebook.domain.models import model_manager from open_notebook.models.llms import LanguageModel -from open_notebook.prompter import Prompter from open_notebook.utils import token_count @@ -31,24 +29,3 @@ def provision_langchain_model( assert isinstance(model, LanguageModel), f"Model is not a LanguageModel: {model}" return model.to_langchain() - - -# todo: turn into a graph -def run_pattern( - pattern_name: str, - config, - state: dict = {}, - parser=None, -) -> BaseMessage: - system_prompt = Prompter(prompt_template=pattern_name, parser=parser).render( - data=state - ) - payload = [SystemMessage(content=system_prompt)] + [ - HumanMessage(content=state["input_text"]) - ] - chain = provision_langchain_model( - str(payload), config.get("configurable", {}).get("model_id"), "transformation" - ) - - response = chain.invoke(payload) - return response diff --git a/open_notebook/models/llms.py b/open_notebook/models/llms.py index 4a5ed9b..2f70670 100644 --- a/open_notebook/models/llms.py +++ b/open_notebook/models/llms.py @@ -284,22 +284,11 @@ class OpenAILanguageModel(LanguageModel): if self.json: kwargs["response_format"] = {"type": "json_object"} - # Set the token limit in kwargs with the appropriate key - if self.model_name in ["o1-mini", "o1-preview"]: - kwargs["max_completion_tokens"] = self.max_tokens - top_p = 1 - streaming = False - max_tokens = None - else: - max_tokens = self.max_tokens - top_p = self.top_p - streaming = self.streaming - return ChatOpenAI( model=self.model_name, - temperature=self.temperature, - streaming=streaming, - max_tokens=max_tokens, - top_p=top_p, + temperature=self.temperature or 0.5, + streaming=self.streaming, + max_tokens=self.max_tokens, + top_p=self.top_p, model_kwargs=kwargs, ) diff --git a/pages/8_💱_Transformations.py b/pages/8_💱_Transformations.py new file mode 100644 index 0000000..686f52b --- /dev/null +++ b/pages/8_💱_Transformations.py @@ -0,0 +1,147 @@ +import streamlit as st + +from open_notebook.domain.transformation import DefaultPrompts, Transformation +from open_notebook.graphs.transformation import graph as transformation_graph +from pages.components.model_selector import model_selector +from pages.stream_app.utils import setup_page + +setup_page("🧩 Transformations") + +transformations_tab, playground_tab = st.tabs(["🧩 Transformations", "🛝 Playground"]) + + +if "transformations" not in st.session_state: + st.session_state.transformations = Transformation.get_all(order_by="name asc") +else: + # work-around for streamlit losing typing on session state + st.session_state.transformations = [ + Transformation(**trans.model_dump()) + for trans in st.session_state.transformations + ] + +with transformations_tab: + st.header("🧩 Transformations") + + st.markdown( + "Transformations are prompts that will be used by the LLM to process a source and extract insights, summaries, etc. " + ) + default_prompts: DefaultPrompts = DefaultPrompts().load() + with st.expander("**⚙️ Default Transformation Prompt**"): + default_prompts.transformation_instructions = st.text_area( + "Default Transformation Prompt", + default_prompts.transformation_instructions, + height=300, + ) + st.caption("This will be added to all your transformation prompts.") + if st.button("Save", key="save_default_prompt"): + default_prompts.update(default_prompts.model_dump()) + st.toast("Default prompt saved successfully!") + if st.button("Create new Transformation", icon="➕", key="new_transformation"): + new_transformation = Transformation( + name="New Tranformation", + title="New Transformation Title", + description="New Transformation Description", + prompt="New Transformation Prompt", + apply_default=False, + ) + st.session_state.transformations.insert(0, new_transformation) + st.rerun() + + st.divider() + st.markdown("Your Transformations") + if len(st.session_state.transformations) == 0: + st.markdown( + "No transformation created yet. Click 'Create new transformation' to get started." + ) + else: + for idx, transformation in enumerate(st.session_state.transformations): + transform_expander = f"**{transformation.name}**" + ( + " - default" if transformation.apply_default else "" + ) + with st.expander( + transform_expander, + expanded=(transformation.id is None), + ): + name = st.text_input( + "Transformation Name", + transformation.name, + key=f"{transformation.id}_name", + ) + title = st.text_input( + "Card Title (this will be the title of all cards created by this transformation). ie 'Key Topics'", + transformation.title, + key=f"{transformation.id}_title", + ) + description = st.text_area( + "Description (displayed as a hint in the UI so you know what you are selecting)", + transformation.description, + key=f"{transformation.id}_description", + ) + prompt = st.text_area( + "Prompt", + transformation.prompt, + key=f"{transformation.id}_prompt", + height=300, + ) + st.markdown( + "You can use the prompt to summarize, expand, extract insights and much more. Example: `Translate this text to French`. For inspiration, check out this [great resource](https://github.com/danielmiessler/fabric/tree/main/patterns)." + ) + + apply_default = st.checkbox( + "Suggest by default on new sources", + transformation.apply_default, + key=f"{transformation.id}_apply_default", + ) + if st.button("Save", key=f"{transformation.id}_save"): + transformation.name = name + transformation.title = title + transformation.description = description + transformation.prompt = prompt + transformation.apply_default = apply_default + st.toast(f"Transformation '{name}' saved successfully!") + transformation.save() + st.rerun() + + if transformation.id: + with st.popover("Other actions"): + if st.button( + "Use in Playground", + icon="🛝", + key=f"{transformation.id}_playground", + ): + st.stop() + if st.button( + "Delete", icon="❌", key=f"{transformation.id}_delete" + ): + transformation.delete() + st.session_state.transformations.remove(transformation) + st.toast(f"Transformation '{name}' deleted successfully!") + st.rerun() + +with playground_tab: + st.title("🛝 Playground") + + transformation = st.selectbox( + "Pick a transformation", + st.session_state.transformations, + format_func=lambda x: x.name, + ) + + model = model_selector( + "Pick a pattern model", + key="model", + help="This is the model that will be used to run the transformation", + model_type="language", + ) + + input_text = st.text_area("Enter some text", height=200) + + if st.button("Run"): + output = transformation_graph.invoke( + dict( + input_text=input_text, + transformation=transformation, + ), + config=dict(configurable={"model_id": model.id}), + ) + st.markdown(output["output"]) diff --git a/pages/8_🛝_Playground.py b/pages/8_🛝_Playground.py deleted file mode 100644 index a094d19..0000000 --- a/pages/8_🛝_Playground.py +++ /dev/null @@ -1,42 +0,0 @@ -import streamlit as st -import yaml - -from open_notebook.graphs.multipattern import graph as pattern_graph -from pages.components.model_selector import model_selector -from pages.stream_app.utils import setup_page - -setup_page("🛝 Playground") - -st.title("🛝 Playground") -with open("transformations.yaml", "r") as file: - transformations = yaml.safe_load(file) - -insight_transformations = transformations["source_insights"] - -transformation: dict = st.selectbox( - "Pick a transformation", - insight_transformations, - format_func=lambda x: x.get("name", "No Name"), -) - -with st.expander("Details"): - st.json(transformation) - -model = model_selector( - "Pick a pattern model", - key="model", - help="This is the model that will be used to run the transformation", - model_type="language", -) - -input_text = st.text_area("Enter some text", height=200) - -if st.button("Run"): - output = pattern_graph.invoke( - dict( - content_stack=[input_text], - patterns=transformation["patterns"], - ), - config=dict(configurable={"model_id": model.id}), - ) - st.markdown(output["output"]) diff --git a/pages/components/source_panel.py b/pages/components/source_panel.py index 499e80f..682fb5e 100644 --- a/pages/components/source_panel.py +++ b/pages/components/source_panel.py @@ -1,3 +1,5 @@ +import asyncio + import streamlit as st import streamlit_scrollable_textbox as stx # type: ignore from humanize import naturaltime @@ -5,8 +7,8 @@ from humanize import naturaltime from open_notebook.domain.models import model_manager from open_notebook.domain.notebook import Source from open_notebook.domain.transformation import Transformation -from open_notebook.utils import surreal_clean -from pages.stream_app.utils import check_models, run_patterns +from open_notebook.graphs.transformation import graph as transform_graph +from pages.stream_app.utils import check_models def source_panel(source_id: str, notebook_id=None, modal=False): @@ -23,7 +25,7 @@ def source_panel(source_id: str, notebook_id=None, modal=False): process_tab, source_tab = st.tabs(["Process", "Source"]) with process_tab: - c1, c2 = st.columns([3, 1]) + c1, c2 = st.columns([4, 2]) with c1: title = st.empty() if source.title: @@ -43,7 +45,7 @@ def source_panel(source_id: str, notebook_id=None, modal=False): "Delete", type="primary", key=f"delete_insight_{insight.id}" ): insight.delete() - # st.rerun(scope="fragment" if modal else "app") + st.rerun(scope="fragment" if modal else "app") st.toast("Source deleted") if notebook_id: if x2.button( @@ -53,19 +55,20 @@ def source_panel(source_id: str, notebook_id=None, modal=False): st.toast("Saved as Note. Refresh the Notebook to see it.") with c2: - transformations = Transformation.get_all() + transformations = Transformation.get_all(order_by="name asc") with st.container(border=True): transformation = st.selectbox( "Run a transformation", - transformations["source_insights"], + transformations, key=f"transformation_{source.id}", - format_func=lambda x: x["name"], + format_func=lambda x: x.name, ) - st.caption(transformation["description"]) + st.caption(transformation.description) if st.button("Run"): - result = run_patterns(source.full_text, transformation["patterns"]) - source.add_insight( - transformation["insight_type"], surreal_clean(result) + asyncio.run( + transform_graph.ainvoke( + input=dict(source=source, transformation=transformation) + ) ) st.rerun(scope="fragment" if modal else "app") diff --git a/pages/stream_app/source.py b/pages/stream_app/source.py index ad84896..0e92c60 100644 --- a/pages/stream_app/source.py +++ b/pages/stream_app/source.py @@ -9,7 +9,7 @@ from loguru import logger from open_notebook.config import UPLOADS_FOLDER from open_notebook.domain.models import model_manager from open_notebook.domain.notebook import Source -from open_notebook.domain.transformation import DefaultTransformations, Transformation +from open_notebook.domain.transformation import Transformation from open_notebook.exceptions import UnsupportedTypeException from open_notebook.graphs.source import source_graph from pages.components import source_panel @@ -45,11 +45,12 @@ def add_source(notebook_id): source_text = st.text_area("Text") req["content"] = source_text - default_transformations = [t for t in DefaultTransformations().source_insights] - available_transformations = [t["name"] for t in transformations["source_insights"]] + transformations = Transformation.get_all() + default_transformations = [t for t in transformations if t.apply_default] apply_transformations = st.multiselect( "Apply transformations", - options=available_transformations, + options=transformations, + format_func=lambda t: t.name, default=default_transformations, ) run_embed = st.checkbox( @@ -85,7 +86,7 @@ def add_source(notebook_id): { "content_state": req, "notebook_id": notebook_id, - "transformations": apply_transformations, + "apply_transformations": apply_transformations, "embed": run_embed, } ) diff --git a/pages/stream_app/utils.py b/pages/stream_app/utils.py index acb0eb7..d56d52a 100644 --- a/pages/stream_app/utils.py +++ b/pages/stream_app/utils.py @@ -9,7 +9,6 @@ from open_notebook.database.migrate import MigrationManager from open_notebook.domain.models import model_manager from open_notebook.domain.notebook import ChatSession, Notebook from open_notebook.graphs.chat import ThreadState, graph -from open_notebook.graphs.multipattern import graph as transform_graph from open_notebook.utils import ( compare_versions, get_installed_version, @@ -17,11 +16,6 @@ from open_notebook.utils import ( ) -def run_patterns(input_text, patterns): - output = transform_graph.invoke(dict(content_stack=[input_text], patterns=patterns)) - return output["output"] - - def version_sidebar(): with st.sidebar: try: diff --git a/prompts/patterns/default/analyze_paper.jinja b/prompts/patterns/default/analyze_paper.jinja deleted file mode 100644 index 6558834..0000000 --- a/prompts/patterns/default/analyze_paper.jinja +++ /dev/null @@ -1,39 +0,0 @@ - -# IDENTITY and PURPOSE - -You are an insightful and analytical reader of academic papers, extracting the key components, significance, and broader implications. Your focus is to uncover the core contributions, practical applications, methodological strengths or weaknesses, and any surprising findings. You are especially attuned to the clarity of arguments, the relevance to existing literature, and potential impacts on both the specific field and broader contexts. - -# STEPS - -1. **READ AND UNDERSTAND THE PAPER**: Thoroughly read the paper, identifying its main focus, arguments, methods, results, and conclusions. - -2. **IDENTIFY CORE ELEMENTS**: - - **Purpose**: What is the main goal or research question? - - **Contribution**: What new knowledge or innovation does this paper bring to the field? - - **Methods**: What methods are used, and are they novel or particularly effective? - - **Key Findings**: What are the most critical results, and why do they matter? - - **Limitations**: Are there any notable limitations or areas for further research? - -3. **SYNTHESIZE THE MAIN POINTS**: - - Extract the key elements and organize them into insightful observations. - - Highlight the broader impact and potential applications. - - Note any aspects that challenge established views or introduce new questions. - -# OUTPUT INSTRUCTIONS - -- Structure the output as follows: - - **PURPOSE**: A concise summary of the main research question or goal (1-2 sentences). - - **CONTRIBUTION**: A bullet list of 2-3 points that describe what the paper adds to the field. - - **KEY FINDINGS**: A bullet list of 2-3 points summarizing the critical outcomes of the study. - - **IMPLICATIONS**: A bullet list of 2-3 points discussing the significance or potential impact of the findings on the field or broader context. - - **LIMITATIONS**: A bullet list of 1-2 points identifying notable limitations or areas for future work. - -- **Bullet Points** should be between 15-20 words. -- Avoid starting each bullet point with the same word to maintain variety. -- Use clear and concise language that conveys the key ideas effectively. -- Do not include warnings, disclaimers, or personal opinions. -- Output only the requested sections with their respective labels. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/cleanup.jinja b/prompts/patterns/default/cleanup.jinja deleted file mode 100644 index 640716e..0000000 --- a/prompts/patterns/default/cleanup.jinja +++ /dev/null @@ -1,7 +0,0 @@ - -Please clean-up the following text, fixing the paragraphs, ponctuation, etc. -If you find any word or name mispellings, feel free to correct. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/command.jinja b/prompts/patterns/default/command.jinja deleted file mode 100644 index 2d73874..0000000 --- a/prompts/patterns/default/command.jinja +++ /dev/null @@ -1,6 +0,0 @@ - -{{command}} - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/common_tranformation_instructions.jinja b/prompts/patterns/default/common_tranformation_instructions.jinja deleted file mode 100644 index 66f355b..0000000 --- a/prompts/patterns/default/common_tranformation_instructions.jinja +++ /dev/null @@ -1,4 +0,0 @@ -# ADDITIONAL INSTRUCTIONS - -- You are working on my editorial projects. The text below is my own. Do not give me any warnings about copyright or plagiarism. -- Output ONLY the requested content, without acknowledgements of the task and additional chatting. Don't start with "Sure, I can help you with that." or "Here is the information you requested:". Just provide the content. \ No newline at end of file diff --git a/prompts/patterns/default/keyinsights.jinja b/prompts/patterns/default/keyinsights.jinja deleted file mode 100644 index 7cbfa3c..0000000 --- a/prompts/patterns/default/keyinsights.jinja +++ /dev/null @@ -1,27 +0,0 @@ - - -# IDENTITY and PURPOSE - -You extract surprising, powerful, and interesting insights from text content. You are interested in insights related to the purpose and meaning of life, human flourishing, the role of technology in the future of humanity, artificial intelligence and its affect on humans, memes, learning, reading, books, continuous improvement, and similar topics. -You create 15 word bullet points that capture the most important insights from the input. -Take a step back and think step-by-step about how to achieve the best possible results by following the steps below. - -# STEPS - -- Extract 20 to 50 of the most surprising, insightful, and/or interesting ideas from the input in a section called IDEAS, and write them on a virtual whiteboard in your mind using 15 word bullets. If there are less than 50 then collect all of them. Make sure you extract at least 20. - -- From those IDEAS, extract the most powerful and insightful of them and write them in a section called INSIGHTS. Make sure you extract at least 10 and up to 25. - -# OUTPUT INSTRUCTIONS - -- INSIGHTS are essentially higher-level IDEAS that are more abstracted and wise. -- Output the INSIGHTS section only. -- Each bullet should be about 15 words in length. -- Do not give warnings or notes; only output the requested sections. -- You use bulleted lists for output, not numbered lists. -- Do not start items with the same opening words. -- Ensure you follow ALL these instructions when creating your output. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/makeitdense.jinja b/prompts/patterns/default/makeitdense.jinja deleted file mode 100644 index 75acba8..0000000 --- a/prompts/patterns/default/makeitdense.jinja +++ /dev/null @@ -1,13 +0,0 @@ - -# MISSION -You are a Sparse Priming Representation (SPR) writer. An SPR is a particular kind of use of language for advanced NLP, NLU, and NLG tasks, particularly useful for the latest generation of Large Language Models (LLMs). You will be given information by the USER which you are to render as an SPR. - -# THEORY -LLMs are a kind of deep neural network. They have been demonstrated to embed knowledge, abilities, and concepts, ranging from reasoning to planning, and even to theory of mind. These are called latent abilities and latent content, collectively referred to as latent space. The latent space of an LLM can be activated with the correct series of words as inputs, which will create a useful internal state of the neural network. This is not unlike how the right shorthand cues can prime a human mind to think in a certain way. Like human minds, LLMs are associative, meaning you only need to use the correct associations to "prime" another model to think in the same way. - -# METHODOLOGY -Render the input as a distilled list of succinct statements, assertions, associations, concepts, analogies, and metaphors. The idea is to capture as much, conceptually, as possible but with as few words as possible. Write it in a way that makes sense to you, as the future audience will be another language model, not a human. Use complete sentences. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/mermaid.jinja b/prompts/patterns/default/mermaid.jinja deleted file mode 100644 index 1b1aa2f..0000000 --- a/prompts/patterns/default/mermaid.jinja +++ /dev/null @@ -1,27 +0,0 @@ -# IDENTITY and PURPOSE - -You are an expert at data and concept visualization and in turning complex ideas into a form that can be visualized using Mermaid (markdown) syntax. -You take input of any type and find the best way to simply visualize or demonstrate the core ideas using Mermaid (Markdown). -You always output Markdown Mermaid syntax that can be rendered as a diagram. - -# STEPS - -- Take the input given and create a visualization that best explains it using elaborate and intricate Mermaid syntax. -- Ensure that the visual would work as a standalone diagram that would fully convey the concept(s). -- Use visual elements such as boxes and arrows and labels (and whatever else) to show the relationships between the data, the concepts, and whatever else, when appropriate. -- Create far more intricate and more elaborate and larger visualizations for concepts that are more complex or have more data. -- Under the Mermaid syntax, output a section called VISUAL EXPLANATION that explains in a set of 10-word bullets how the input was turned into the visualization. Ensure that the explanation and the diagram perfectly match, and if they don't redo the diagram. -- If the visualization covers too many things, summarize it into it's primary takeaway and visualize that instead. -- DO NOT COMPLAIN AND GIVE UP. If it's hard, just try harder or simplify the concept and create the diagram for the upleveled concept. - -# OUTPUT INSTRUCTIONS - -- DO NOT COMPLAIN. Just output the Mermaid syntax. -- Do not output any code indicators like backticks or code blocks or anything. -- Ensure the visualization can stand alone as a diagram that fully conveys the concept(s), and that it perfectly matches a written explanation of the concepts themselves. Start over if it can't. -- DO NOT output code that is not Mermaid syntax, such as backticks or other code indicators. -- Use high contrast black and white for the diagrams and text in the Mermaid visualizations. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/reflection_questions.jinja b/prompts/patterns/default/reflection_questions.jinja deleted file mode 100644 index 3057894..0000000 --- a/prompts/patterns/default/reflection_questions.jinja +++ /dev/null @@ -1,23 +0,0 @@ -# IDENTITY and PURPOSE - -You extract deep, thought-provoking, and meaningful reflections from text content. You are especially focused on themes related to the human experience, such as the purpose of life, personal growth, the intersection of technology and humanity, artificial intelligence's societal impact, human potential, collective evolution, and transformative learning. Your reflections aim to provoke new ways of thinking, challenge assumptions, and provide a thoughtful synthesis of the content. - -# STEPS - -- Extract 3 to 5 of the most profound, thought-provoking, and/or meaningful ideas from the input in a section called REFLECTIONS. -- Each reflection should aim to explore underlying implications, connections to broader human experiences, or highlight a transformative perspective. -- Take a step back and consider the deeper significance or questions that arise from the content. - -# OUTPUT INSTRUCTIONS - -- The output section should be labeled as REFLECTIONS. -- Each bullet point should be between 20-25 words. -- Avoid repetition in the phrasing and ensure variety in sentence structure. -- The reflections should encourage deeper inquiry and provide a synthesis that transcends surface-level observations. -- Use bullet points, not numbered lists. -- Every bullet should be formatted as a question that elicits contemplation or a statement that offers a profound insight. -- Do not give warnings or notes; only output the requested section. - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/summarize.jinja b/prompts/patterns/default/summarize.jinja deleted file mode 100644 index 90a60ba..0000000 --- a/prompts/patterns/default/summarize.jinja +++ /dev/null @@ -1,13 +0,0 @@ - -# SYSTEM ROLE -You are a content summarization assistant that creates dense, information-rich summaries optimized for machine understanding. Your summaries should capture key concepts with minimal words while maintaining complete, clear sentences. - -# TASK -Analyze the provided content and create a summary that: -- Captures the core concepts and key information -- Uses clear, direct language -- Maintains context from any previous summaries - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT diff --git a/prompts/patterns/default/toc.jinja b/prompts/patterns/default/toc.jinja deleted file mode 100644 index f8fca4b..0000000 --- a/prompts/patterns/default/toc.jinja +++ /dev/null @@ -1,13 +0,0 @@ - -# SYSTEM ROLE -You are a content analysis assistant that reads through documents and provides a Table of Contents (ToC) to help users identify what the document covers more easily. -Your ToC should capture all major topics and transitions in the content and should mention them in the order theh appear. - -# TASK -Analyze the provided content and create a Table of Contents: -- Captures the core topics included in the text -- Gives a small description of what is covered - -{% include 'patterns/default/common_tranformation_instructions.jinja' %} - -# INPUT