open-notebook/open_notebook/database/migrations/1.surrealql

DEFINE TABLE IF NOT EXISTS source SCHEMAFULL;

DEFINE FIELD IF NOT EXISTS
    asset
    ON TABLE source
    FLEXIBLE TYPE option<object>;

DEFINE FIELD IF NOT EXISTS title ON TABLE source TYPE option<string>;
DEFINE FIELD IF NOT EXISTS topics ON TABLE source TYPE option<array<string>>;
DEFINE FIELD IF NOT EXISTS full_text ON TABLE source TYPE option<string>;

DEFINE FIELD IF NOT EXISTS created ON source DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON source DEFAULT time::now() VALUE time::now();

DEFINE TABLE IF NOT EXISTS source_embedding SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS source ON TABLE source_embedding TYPE record<source>;
DEFINE FIELD IF NOT EXISTS order ON TABLE source_embedding TYPE int;
DEFINE FIELD IF NOT EXISTS content ON TABLE source_embedding TYPE string;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_embedding TYPE array<float>;

DEFINE TABLE IF NOT EXISTS source_insight SCHEMAFULL;
DEFINE FIELD IF NOT EXISTS source ON TABLE source_insight TYPE record<source>;
DEFINE FIELD IF NOT EXISTS insight_type ON TABLE source_insight TYPE string;
DEFINE FIELD IF NOT EXISTS content ON TABLE source_insight TYPE string;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE source_insight TYPE array<float>;


DEFINE EVENT IF NOT EXISTS source_delete ON TABLE source WHEN ($after == NONE) THEN {
    delete source_embedding where source == $before.id;
    delete source_insight where source == $before.id;
};

DEFINE TABLE IF NOT EXISTS note SCHEMAFULL;

DEFINE FIELD IF NOT EXISTS title ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS summary ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS content ON TABLE note TYPE option<string>;
DEFINE FIELD IF NOT EXISTS embedding ON TABLE note TYPE array<float>;

DEFINE FIELD IF NOT EXISTS created ON note DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON note DEFAULT time::now() VALUE time::now();

DEFINE TABLE IF NOT EXISTS notebook SCHEMAFULL;

DEFINE FIELD IF NOT EXISTS name ON TABLE notebook TYPE option<string>;
DEFINE FIELD IF NOT EXISTS description ON TABLE notebook TYPE option<string>;
DEFINE FIELD IF NOT EXISTS archived ON TABLE notebook TYPE option<bool> DEFAULT False;


DEFINE FIELD IF NOT EXISTS created ON notebook DEFAULT time::now() VALUE $before OR time::now();
DEFINE FIELD IF NOT EXISTS updated ON notebook DEFAULT time::now() VALUE time::now();

DEFINE TABLE IF NOT EXISTS reference
TYPE RELATION 
FROM source TO notebook;

DEFINE TABLE IF NOT EXISTS artifact
TYPE RELATION 
FROM note TO notebook;

DEFINE TABLE IF NOT EXISTS podcast_config SCHEMALESS;

-- entender o analyzer
DEFINE ANALYZER IF NOT EXISTS my_analyzer TOKENIZERS blank,class,camel,punct FILTERS snowball(english), lowercase;

DEFINE INDEX IF NOT EXISTS idx_source_title ON TABLE source COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_full_text ON TABLE source COLUMNS full_text SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_embed_chunk ON TABLE source_embedding COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_source_insight ON TABLE source_insight COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_note ON TABLE note COLUMNS content SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;
DEFINE INDEX IF NOT EXISTS idx_note_title ON TABLE note COLUMNS title SEARCH ANALYZER my_analyzer BM25 HIGHLIGHTS;

DEFINE FUNCTION IF NOT EXISTS fn::text_search($query_text: string, $match_count: int, $sources:bool, $show_notes:bool) {
  
    let $source_title_search = 
        IF $sources {(
            SELECT id as item_id, math::max(search::score(1)) AS relevance
            FROM source
            WHERE title @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };
    
    let $source_embedding_search = 
         IF $sources {(
             SELECT source as item_id, math::max(search::score(1)) AS relevance
            FROM source_embedding
            WHERE content @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };

    let $source_full_search = 
         IF $sources {(
            SELECT source as item_id, math::max(search::score(1)) AS relevance
            FROM source
            WHERE full_text @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };
    
    let $source_insight_search = 
         IF $sources {(
             SELECT source as item_id, math::max(search::score(1)) AS relevance
            FROM source_insight
            WHERE content @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };

    let $note_title_search = 
         IF $show_notes {(
             SELECT id as item_id, math::max(search::score(1)) AS relevance
            FROM note
            WHERE title @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };

     let $note_content_search = 
         IF $show_notes {(
             SELECT id as item_id, math::max(search::score(1)) AS relevance
            FROM note
            WHERE content @1@ $query_text
            GROUP BY item_id)}
        ELSE { [] };

    let $source_chunk_results = array::union($source_embedding_search, $source_full_search);
    
    let $source_asset_results = array::union($source_title_search, $source_insight_search);

    let $source_results = array::union($source_chunk_results, $source_asset_results );
    let $note_results = array::union($note_title_search, $note_content_search );
    let $final_results = array::union($source_results, $note_results );

    RETURN (SELECT item_id, math::max(relevance) as relevance from $final_results
        group by item_id ORDER BY relevance DESC LIMIT $match_count);
    
    
};


DEFINE FUNCTION IF NOT EXISTS fn::vector_search($query: array<float>, $match_count: int, $sources:bool, $show_notes:bool) {
   
    let $source_embedding_search = 
         IF $sources {(
            SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
            FROM source_embedding LIMIT $match_count)}
        ELSE { [] };

    
    let $source_insight_search = 
         IF $sources {(
             SELECT source as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
                FROM source_insight LIMIT $match_count)}
        ELSE { [] };

    
     let $note_content_search = 
         IF $show_notes {(
                SELECT id as item_id, content, vector::similarity::cosine(embedding, $query) as similarity
                FROM note LIMIT $match_count)}

        ELSE { [] };

    let $source_chunk_results = array::union($source_embedding_search, $source_insight_search);
    
    let $source_results = array::union($source_chunk_results, $source_insight_search);

    let $note_results = $note_content_search;
    let $final_results = array::union($source_results, $note_results );

    RETURN (SELECT item_id, math::max(similarity) as similarity from $final_results
        group by item_id ORDER BY similarity DESC LIMIT $match_count);
    
    
};

IF array::len(select * from open_notebook:default_models) == 0 THEN
    CREATE open_notebook:default_models SET
    default_chat_model= ""
END;