From f105a2fd92c25a75668c3615cd8328fa875050d8 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sat, 1 Jun 2024 21:21:52 -0500 Subject: [PATCH] commented the backend code with it's limitations + optimised and removed a bunch of unnecessary queries --- apps/cf-ai-backend/src/index.ts | 51 ++++++-- apps/cf-ai-backend/src/prompts/prompt1.ts | 6 +- apps/cf-ai-backend/src/types.ts | 1 + apps/web/app/chat/page.tsx | 23 +++- apps/web/app/globals.css | 48 ------- apps/web/app/helpers/lib/searchParams.ts | 7 + apps/web/app/home/actions.ts | 6 - apps/web/app/home/page.tsx | 1 + apps/web/app/home/queryinput.tsx | 152 ++++++++++++---------- apps/web/app/layout.tsx | 1 - package.json | 1 + 11 files changed, 152 insertions(+), 145 deletions(-) delete mode 100644 apps/web/app/globals.css diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 3e65ac90..19770dec 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -23,8 +23,11 @@ app.use("*", timing()); app.use("*", logger()); app.use("/api/", async (c, next) => { - const auth = bearerAuth({ token: c.env.SECURITY_KEY }); - return auth(c, next); + if (c.env.NODE_ENV !== "development") { + const auth = bearerAuth({ token: c.env.SECURITY_KEY }); + return auth(c, next); + } + return next(); }); // ------- MIDDLEWARES END ------- @@ -71,6 +74,11 @@ app.get( }, ); +/* TODO: Eventually, we should not have to save each user's content in a seperate vector. +Lowkey, it makes sense. The user may save their own version of a page - like selected text from twitter.com url. +But, it's not scalable *enough*. How can we store the same vectors for the same content, without needing to duplicate for each uer? +Hard problem to solve, Vectorize doesn't have an OR filter, so we can't just filter by URL and user. +*/ app.post( "/api/chat", zValidator( @@ -99,23 +107,30 @@ app.post( const sourcesOnly = query.sourcesOnly === "true"; const spaces = query.spaces?.split(",") || [undefined]; + // Get the AI model maker and vector store const { model, store } = await initQuery(c, query.model); const filter: VectorizeVectorMetadataFilter = { user: query.user }; + // Converting the query to a vector so that we can search for similar vectors const queryAsVector = await store.embeddings.embedQuery(query.query); const responses: VectorizeMatches = { matches: [], count: 0 }; - for (const space of spaces) { + // SLICED to 5 to avoid too many queries + for (const space of spaces.slice(0, 5)) { if (space !== undefined) { + // it's possible for space list to be [undefined] so we only add space filter conditionally filter.space = space; } + // Because there's no OR operator in the filter, we have to make multiple queries const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { topK: query.topK, filter, + returnMetadata: true, }); + // Basically recreating the response object if (resp.count > 0) { responses.matches.push(...resp.matches); responses.count += resp.count; @@ -125,6 +140,8 @@ app.post( const minScore = Math.min(...responses.matches.map(({ score }) => score)); const maxScore = Math.max(...responses.matches.map(({ score }) => score)); + // We are "normalising" the scores - if all of them are on top, we want to make sure that + // we have a way to filter out the noise. const normalizedData = responses.matches.map((data) => ({ ...data, normalizedScore: @@ -136,6 +153,9 @@ app.post( let highScoreData = normalizedData.filter( ({ normalizedScore }) => normalizedScore > 50, ); + + // If the normalsation is not done properly, we have a fallback to just get the + // top 3 scores if (highScoreData.length === 0) { highScoreData = normalizedData .sort((a, b) => b.score - a.score) @@ -146,13 +166,16 @@ app.post( (a, b) => b.normalizedScore - a.normalizedScore, ); - console.log(JSON.stringify(sortedHighScoreData)); - + // So this is kinda hacky, but the frontend needs to do 2 calls to get sources and chat. + // I think this is fine for now, but we can improve this later. if (sourcesOnly) { const idsAsStrings = sortedHighScoreData.map((dataPoint) => dataPoint.id.toString(), ); + // We are getting the content ID back, so that the frontend can show the actual sources properly. + // it IS a lot of DB calls, i completely agree. + // TODO: return metadata value here, so that the frontend doesn't have to re-fetch anything. const storedContent = await Promise.all( idsAsStrings.map(async (id) => await c.env.KV.get(id)), ); @@ -160,19 +183,21 @@ app.post( return c.json({ ids: storedContent }); } - const vec = await c.env.VECTORIZE_INDEX.getByIds( - sortedHighScoreData.map(({ id }) => id), - ); + const vec = responses.matches.map((data) => ({ metadata: data.metadata })); const vecWithScores = vec.map((v, i) => ({ ...v, score: sortedHighScoreData[i].score, + normalisedScore: sortedHighScoreData[i].normalizedScore, })); - const preparedContext = vecWithScores.map(({ metadata, score }) => ({ - context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - score, - })); + const preparedContext = vecWithScores.map( + ({ metadata, score, normalisedScore }) => ({ + context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + normalisedScore, + }), + ); const initialMessages: CoreMessage[] = [ { role: "user", content: systemPrompt }, @@ -193,7 +218,7 @@ app.post( ...((body.chatHistory || []) as CoreMessage[]), userMessage, ], - temperature: 0.4, + // temperature: 0.4, }); return response.toTextStreamResponse(); diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts index 6e1bdf7b..aa7694d3 100644 --- a/apps/cf-ai-backend/src/prompts/prompt1.ts +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -11,6 +11,7 @@ To generate your answer: - Provide a direct answer to the question - Use markdown formatting in your answer, including bold, italics, and bullet points as appropriate to improve readability and highlight key points - Give detailed and accurate responses for things like 'write a blog' or long-form questions. +- The normalisedScore is a value in which the scores are 'balanced' to give a better representation of the relevance of the context, between 1 and 100, out of the top 10 results Provide your justification between tags and your final answer between tags, formatting both in markdown. @@ -20,13 +21,14 @@ export const template = ({ contexts, question }) => { // Map over contexts to generate the context and score parts const contextParts = contexts .map( - ({ context, score }) => ` + ({ context, score, normalisedScore }) => ` ${context} - ${score} + score: ${score} + normalisedScore: ${normalisedScore} `, ) .join("\n"); diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts index 3b6db589..bea4bf80 100644 --- a/apps/cf-ai-backend/src/types.ts +++ b/apps/cf-ai-backend/src/types.ts @@ -10,6 +10,7 @@ export type Env = { KV: KVNamespace; MYBROWSER: unknown; ANTHROPIC_API_KEY: string; + NODE_ENV: string; }; export interface TweetData { diff --git a/apps/web/app/chat/page.tsx b/apps/web/app/chat/page.tsx index fce4bfd1..bfe0c362 100644 --- a/apps/web/app/chat/page.tsx +++ b/apps/web/app/chat/page.tsx @@ -1,7 +1,24 @@ -import React from "react"; +import { chatSearchParamsCache } from "../helpers/lib/searchParams"; +import Menu from "../home/menu"; +import Header from "../home/header"; +import ChatWindow from "./chatWindow"; -function Page() { - return
Page
; +function Page({ + searchParams, +}: { + searchParams: Record; +}) { + const { firstTime, q, spaces } = chatSearchParamsCache.parse(searchParams); + + return ( +
+ + +
+ + +
+ ); } export default Page; diff --git a/apps/web/app/globals.css b/apps/web/app/globals.css deleted file mode 100644 index b1902464..00000000 --- a/apps/web/app/globals.css +++ /dev/null @@ -1,48 +0,0 @@ -@tailwind base; -@tailwind components; -@tailwind utilities; - -/* :root { - --foreground-rgb: 0, 0, 0; - --background-start-rgb: 214, 219, 220; - --background-end-rgb: 255, 255, 255; -} */ - -@media (prefers-color-scheme: dark) { - :root { - --foreground: rgba(179, 188, 197, 1); - --foreground-menu: rgba(106, 115, 125, 1); - --background: rgba(23, 27, 31, 1); - --secondary: rgba(31, 36, 40, 1); - --primary: rgba(54, 157, 253, 1); - --border: rgba(51, 57, 67, 1); - } -} - -body { - color: var(--foreground); - background: var(--background); - font-size: 14px; -} - -@layer base { - .all-center { - display: flex; - align-items: center; - justify-content: center; - } -} - -@layer utilities { - .text-balance { - text-wrap: balance; - } -} - -.gradient-background { - background: linear-gradient( - 150deg, - rgba(255, 255, 255, 0.1) 0%, - rgba(255, 255, 255, 0) - ); -} diff --git a/apps/web/app/helpers/lib/searchParams.ts b/apps/web/app/helpers/lib/searchParams.ts index a43a28fe..2e02aa3e 100644 --- a/apps/web/app/helpers/lib/searchParams.ts +++ b/apps/web/app/helpers/lib/searchParams.ts @@ -3,8 +3,15 @@ import { parseAsInteger, parseAsString, parseAsBoolean, + parseAsArrayOf, } from "nuqs/server"; export const homeSearchParamsCache = createSearchParamsCache({ firstTime: parseAsBoolean.withDefault(false), }); + +export const chatSearchParamsCache = createSearchParamsCache({ + firstTime: parseAsBoolean.withDefault(false), + q: parseAsString.withDefault(""), + spaces: parseAsArrayOf(parseAsInteger, ","), +}); diff --git a/apps/web/app/home/actions.ts b/apps/web/app/home/actions.ts index 0bb2d051..908fe79e 100644 --- a/apps/web/app/home/actions.ts +++ b/apps/web/app/home/actions.ts @@ -1,7 +1 @@ "use server"; - -import { redirect } from "next/navigation"; - -export async function navigate(q: string) { - redirect(`/chat?q=${q}`); -} diff --git a/apps/web/app/home/page.tsx b/apps/web/app/home/page.tsx index d9025a9d..9908c017 100644 --- a/apps/web/app/home/page.tsx +++ b/apps/web/app/home/page.tsx @@ -9,6 +9,7 @@ function Page({ }: { searchParams: Record; }) { + // TODO: use this to show a welcome page/modal const { firstTime } = homeSearchParamsCache.parse(searchParams); return ( diff --git a/apps/web/app/home/queryinput.tsx b/apps/web/app/home/queryinput.tsx index c394d9c6..a0e25d83 100644 --- a/apps/web/app/home/queryinput.tsx +++ b/apps/web/app/home/queryinput.tsx @@ -1,88 +1,96 @@ "use client"; -import { ArrowRightIcon, MemoriesIcon, SelectIcon } from "@repo/ui/icons"; +import { ArrowRightIcon } from "@repo/ui/icons"; import Image from "next/image"; -import React from "react"; +import React, { useCallback, useState } from "react"; import Divider from "@repo/ui/shadcn/divider"; -import { redirect } from "next/navigation"; -import { navigate } from "./actions"; -import { FilterSpaces } from "@repo/ui/components/filterSpaces"; +import { MultipleSelector, Option } from "@repo/ui/shadcn/combobox"; +import { AnimatePresence } from "framer-motion"; +import { useRouter } from "next/navigation"; -function QueryInput() { - const [q, setQ] = React.useState(""); +const OPTIONS: Option[] = [ + { label: "nextjs", value: "0" }, + { label: "React", value: "1" }, + { label: "Remix", value: "2" }, + { label: "Vite", value: "3" }, + { label: "Nuxt", value: "4" }, + { label: "Vue", value: "5" }, + { label: "Svelte", value: "6" }, + { label: "Angular", value: "7" }, + { label: "Ember", value: "8" }, + { label: "Gatsby", value: "9" }, +]; + +function QueryInput({ + initialQuery = "", + initialSpaces = [], + disabled = false, +}: { + initialQuery?: string; + initialSpaces?: number[]; + disabled?: boolean; +}) { + const [q, setQ] = useState(initialQuery); + + const [selectedSpaces, setSelectedSpaces] = useState(initialSpaces); + + const { push } = useRouter(); + + const parseQ = () => { + const newQ = + "/chat?q=" + + encodeURI(q) + + (selectedSpaces ? "&spaces=" + selectedSpaces.join(",") : ""); - const parseQ = React.useCallback(() => { - const newQ = q.replace(/\n/g, "\\n"); return newQ; - }, [q]); - - const [selectedSpaces, setSelectedSpaces] = React.useState([]); + }; return ( -
- {/* input and action button */} -
navigate(parseQ())} className="flex gap-4 p-3"> -