From c68433d9d86017134079854866c67059198759f2 Mon Sep 17 00:00:00 2001 From: Dhravya Date: Sat, 18 May 2024 17:32:14 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20Rewrite=20backend=20to=20hono=20?= =?UTF-8?q?=E2=9A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/cf-ai-backend/.editorconfig | 13 - apps/cf-ai-backend/.gitignore | 178 +--------- apps/cf-ai-backend/.prettierrc | 6 - apps/cf-ai-backend/README.md | 58 +++ apps/cf-ai-backend/jest.config.js | 7 + apps/cf-ai-backend/package.json | 29 +- apps/cf-ai-backend/src/OpenAIEmbedder.ts | 43 --- apps/cf-ai-backend/src/env.d.ts | 19 - apps/cf-ai-backend/src/helper.ts | 139 ++++++++ apps/cf-ai-backend/src/index.test.ts | 13 + apps/cf-ai-backend/src/index.ts | 249 +++++++++++-- apps/cf-ai-backend/src/prompts/prompt1.ts | 47 +++ apps/cf-ai-backend/src/routes.ts | 43 --- apps/cf-ai-backend/src/routes/add.ts | 52 --- apps/cf-ai-backend/src/routes/ask.ts | 35 -- .../src/routes/batchUploadTweets.ts | 38 -- apps/cf-ai-backend/src/routes/chat.ts | 141 -------- apps/cf-ai-backend/src/routes/delete.ts | 27 -- apps/cf-ai-backend/src/routes/edit.ts | 61 ---- .../src/routes/getPageContent.ts | 33 -- apps/cf-ai-backend/src/routes/query.ts | 94 ----- apps/cf-ai-backend/src/routes/queue.ts | 95 ----- apps/cf-ai-backend/src/routes/wipedata.ts | 19 - apps/cf-ai-backend/src/types.ts | 48 +++ apps/cf-ai-backend/src/util.ts | 7 - .../cf-ai-backend/src/utils/OpenAIEmbedder.ts | 43 +++ apps/cf-ai-backend/src/utils/chonker.ts | 44 +++ apps/cf-ai-backend/src/utils/seededRandom.ts | 18 + apps/cf-ai-backend/tsconfig.json | 105 +----- apps/cf-ai-backend/vite.config.ts | 6 + apps/cf-ai-backend/wrangler.toml | 21 +- apps/web/src/app/content.tsx | 16 +- apps/web/src/app/layout.tsx | 5 +- apps/web/src/app/page.tsx | 3 + apps/web/src/assets/Memories.tsx | 12 +- apps/web/src/assets/MemoryWithImages.tsx | 15 +- apps/web/src/components/ProfileDrawer.tsx | 35 +- .../components/Sidebar/AddMemoryDialog.tsx | 141 +++++--- .../src/components/Sidebar/EditNoteDialog.tsx | 105 +++--- .../src/components/Sidebar/FilterCombobox.tsx | 185 +++++----- .../src/components/Sidebar/MemoriesBar.tsx | 330 ++++++++++-------- apps/web/src/components/Sidebar/index.tsx | 294 +++++----------- apps/web/src/components/ui/avatar.tsx | 2 +- apps/web/src/components/ui/badge.tsx | 2 +- apps/web/src/components/ui/dialog.tsx | 11 +- apps/web/src/components/ui/dropdown-menu.tsx | 8 +- apps/web/src/components/ui/input.tsx | 6 +- package.json | 23 +- 48 files changed, 1268 insertions(+), 1656 deletions(-) delete mode 100644 apps/cf-ai-backend/.editorconfig delete mode 100644 apps/cf-ai-backend/.prettierrc create mode 100644 apps/cf-ai-backend/README.md create mode 100644 apps/cf-ai-backend/jest.config.js delete mode 100644 apps/cf-ai-backend/src/OpenAIEmbedder.ts delete mode 100644 apps/cf-ai-backend/src/env.d.ts create mode 100644 apps/cf-ai-backend/src/helper.ts create mode 100644 apps/cf-ai-backend/src/index.test.ts create mode 100644 apps/cf-ai-backend/src/prompts/prompt1.ts delete mode 100644 apps/cf-ai-backend/src/routes.ts delete mode 100644 apps/cf-ai-backend/src/routes/add.ts delete mode 100644 apps/cf-ai-backend/src/routes/ask.ts delete mode 100644 apps/cf-ai-backend/src/routes/batchUploadTweets.ts delete mode 100644 apps/cf-ai-backend/src/routes/chat.ts delete mode 100644 apps/cf-ai-backend/src/routes/delete.ts delete mode 100644 apps/cf-ai-backend/src/routes/edit.ts delete mode 100644 apps/cf-ai-backend/src/routes/getPageContent.ts delete mode 100644 apps/cf-ai-backend/src/routes/query.ts delete mode 100644 apps/cf-ai-backend/src/routes/queue.ts delete mode 100644 apps/cf-ai-backend/src/routes/wipedata.ts create mode 100644 apps/cf-ai-backend/src/types.ts delete mode 100644 apps/cf-ai-backend/src/util.ts create mode 100644 apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts create mode 100644 apps/cf-ai-backend/src/utils/chonker.ts create mode 100644 apps/cf-ai-backend/src/utils/seededRandom.ts create mode 100644 apps/cf-ai-backend/vite.config.ts diff --git a/apps/cf-ai-backend/.editorconfig b/apps/cf-ai-backend/.editorconfig deleted file mode 100644 index 64ab2601..00000000 --- a/apps/cf-ai-backend/.editorconfig +++ /dev/null @@ -1,13 +0,0 @@ -# http://editorconfig.org -root = true - -[*] -indent_style = tab -tab_width = 2 -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true - -[*.yml] -indent_style = space diff --git a/apps/cf-ai-backend/.gitignore b/apps/cf-ai-backend/.gitignore index c51412d9..e5b4f2d1 100644 --- a/apps/cf-ai-backend/.gitignore +++ b/apps/cf-ai-backend/.gitignore @@ -1,173 +1,13 @@ -# Logs -wrangler.toml - -logs -_.log -npm-debug.log_ -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) - -report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json - -# Runtime data - -pids -_.pid -_.seed -\*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover - -lib-cov - -# Coverage directory used by tools like istanbul - -coverage -\*.lcov - -# nyc test coverage - -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) - -.grunt - -# Bower dependency directory (https://bower.io/) - -bower_components - -# node-waf configuration - -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) - -build/Release - -# Dependency directories - -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) - -web_modules/ - -# TypeScript cache - -\*.tsbuildinfo - -# Optional npm cache directory - -.npm - -# Optional eslint cache - -.eslintcache - -# Optional stylelint cache - -.stylelintcache - -# Microbundle cache - -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history - -.node_repl_history +dist +node_modules +worker +package-lock.json +yarn.lock +.cargo-ok # Output of 'npm pack' +*.tgz -\*.tgz - -# Yarn Integrity file - -.yarn-integrity - -# dotenv environment variable files - -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) - -.cache -.parcel-cache - -# Next.js build output - -.next -out - -# Nuxt.js build / generate output - -.nuxt -dist - -# Gatsby files - -.cache/ - -# Comment in the public line in if your project uses Gatsby and not Next.js - -# https://nextjs.org/blog/next-9-1#public-directory-support - -# public - -# vuepress build output - -.vuepress/dist - -# vuepress v2.x temp and cache directory - -.temp -.cache - -# Docusaurus cache and generated files - -.docusaurus - -# Serverless directories - -.serverless/ - -# FuseBox cache - -.fusebox/ - -# DynamoDB Local files - -.dynamodb/ - -# TernJS port file - -.tern-port - -# Stores VSCode versions used for testing VSCode extensions - -.vscode-test - -# yarn v2 - -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.\* - -# wrangler project - +# wrangler files +.wrangler .dev.vars -.wrangler/ diff --git a/apps/cf-ai-backend/.prettierrc b/apps/cf-ai-backend/.prettierrc deleted file mode 100644 index 5c7b5d3c..00000000 --- a/apps/cf-ai-backend/.prettierrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "printWidth": 140, - "singleQuote": true, - "semi": true, - "useTabs": true -} diff --git a/apps/cf-ai-backend/README.md b/apps/cf-ai-backend/README.md new file mode 100644 index 00000000..86409f29 --- /dev/null +++ b/apps/cf-ai-backend/README.md @@ -0,0 +1,58 @@ +# Hono minimal project + +This is a minimal project with [Hono](https://github.com/honojs/hono/) for Cloudflare Workers. + +## Features + +- Minimal +- TypeScript +- Wrangler to develop and deploy. +- [Jest](https://jestjs.io/ja/) for testing. + +## Usage + +Initialize + +``` +npx create-cloudflare my-app https://github.com/honojs/hono-minimal +``` + +Install + +``` +yarn install +``` + +Develop + +``` +yarn dev +``` + +Test + +``` +yarn test +``` + +Deploy + +``` +yarn deploy +``` + +## Examples + +See: + +## For more information + +See: + +## Author + +Yusuke Wada + +## License + +MIT diff --git a/apps/cf-ai-backend/jest.config.js b/apps/cf-ai-backend/jest.config.js new file mode 100644 index 00000000..85c5da79 --- /dev/null +++ b/apps/cf-ai-backend/jest.config.js @@ -0,0 +1,7 @@ +module.exports = { + testEnvironment: "miniflare", + testMatch: ["**/test/**/*.+(ts|tsx)", "**/src/**/(*.)+(spec|test).+(ts|tsx)"], + transform: { + "^.+\\.(ts|tsx)$": "esbuild-jest", + }, +}; diff --git a/apps/cf-ai-backend/package.json b/apps/cf-ai-backend/package.json index 30df6ec7..480f9601 100644 --- a/apps/cf-ai-backend/package.json +++ b/apps/cf-ai-backend/package.json @@ -1,16 +1,17 @@ { - "name": "cf-ai-backend", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev", - "start": "wrangler dev", - "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20240222.0", - "typescript": "^5.0.4", - "wrangler": "^3.0.0" - } + "name": "new-cf-ai-backend", + "private": true, + "version": "0.0.1", + "main": "src/index.ts", + "scripts": { + "test": "jest --verbose", + "deploy": "wrangler deploy", + "dev": "wrangler dev", + "start": "wrangler dev", + "unsafe-reset-vector-db": "wrangler vectorize delete supermem-vector && wrangler vectorize create --dimensions=1536 supermem-vector-1 --metric=cosine" + }, + "license": "MIT", + "dependencies": { + "@hono/zod-validator": "^0.2.1" + } } diff --git a/apps/cf-ai-backend/src/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/OpenAIEmbedder.ts deleted file mode 100644 index e227d1e3..00000000 --- a/apps/cf-ai-backend/src/OpenAIEmbedder.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { AiTextGenerationOutput } from '@cloudflare/ai/dist/ai/tasks/text-generation'; - -interface OpenAIEmbeddingsParams { - apiKey: string; - modelName: string; -} - -export class OpenAIEmbeddings { - private apiKey: string; - private modelName: string; - - constructor({ apiKey, modelName }: OpenAIEmbeddingsParams) { - this.apiKey = apiKey; - this.modelName = modelName; - } - - async embedDocuments(texts: string[]): Promise { - const responses = await Promise.all(texts.map((text) => this.embedQuery(text))); - return responses; - } - - async embedQuery(text: string): Promise { - const response = await fetch('https://api.openai.com/v1/embeddings', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify({ - input: text, - model: this.modelName, - }), - }); - - const data = (await response.json()) as { - data: { - embedding: number[]; - }[]; - }; - - return data.data[0].embedding; - } -} diff --git a/apps/cf-ai-backend/src/env.d.ts b/apps/cf-ai-backend/src/env.d.ts deleted file mode 100644 index e4ae9a1b..00000000 --- a/apps/cf-ai-backend/src/env.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -interface Env { - VECTORIZE_INDEX: VectorizeIndex; - AI: Fetcher; - SECURITY_KEY: string; - OPENAI_API_KEY: string; - GOOGLE_AI_API_KEY: string; - MY_QUEUE: Queue; - KV: KVNamespace; - MYBROWSER: BrowserWorker; -} - -interface TweetData { - tweetText: string; - postUrl: string; - authorName: string; - handle: string; - time: string; - saveToUser: string; -} diff --git a/apps/cf-ai-backend/src/helper.ts b/apps/cf-ai-backend/src/helper.ts new file mode 100644 index 00000000..87495c59 --- /dev/null +++ b/apps/cf-ai-backend/src/helper.ts @@ -0,0 +1,139 @@ +import { Context } from "hono"; +import { Env, vectorObj } from "./types"; +import { CloudflareVectorizeStore } from "@langchain/cloudflare"; +import { OpenAIEmbeddings } from "./utils/OpenAIEmbedder"; +import { createOpenAI } from "@ai-sdk/openai"; +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { createAnthropic } from "@ai-sdk/anthropic"; +import { z } from "zod"; +import { seededRandom } from "./utils/seededRandom"; + +export async function initQuery( + c: Context<{ Bindings: Env }>, + model: string = "gpt-4o", +) { + const embeddings = new OpenAIEmbeddings({ + apiKey: c.env.OPENAI_API_KEY, + modelName: "text-embedding-3-small", + }); + + const store = new CloudflareVectorizeStore(embeddings, { + index: c.env.VECTORIZE_INDEX, + }); + + const DEFAULT_MODEL = "gpt-4o"; + + let selectedModel: + | ReturnType> + | ReturnType> + | ReturnType>; + + switch (model) { + case "claude-3-opus": + const anthropic = createAnthropic({ + apiKey: c.env.ANTHROPIC_API_KEY, + }); + selectedModel = anthropic.chat("claude-3-opus-20240229"); + console.log("Selected model: ", selectedModel); + break; + case "gemini-1.5-pro": + const googleai = createGoogleGenerativeAI({ + apiKey: c.env.GOOGLE_AI_API_KEY, + }); + selectedModel = googleai.chat("models/gemini-1.5-pro-latest"); + console.log("Selected model: ", selectedModel); + break; + case "gpt-4o": + default: + const openai = createOpenAI({ + apiKey: c.env.OPENAI_API_KEY, + }); + selectedModel = openai.chat("gpt-4o"); + break; + } + + if (!selectedModel) { + throw new Error( + `Model ${model} not found and default model ${DEFAULT_MODEL} is also not available.`, + ); + } + + return { store, model: selectedModel }; +} + +export async function deleteDocument({ + url, + user, + c, + store, +}: { + url: string; + user: string; + c: Context<{ Bindings: Env }>; + store: CloudflareVectorizeStore; +}) { + const toBeDeleted = `${url}-${user}`; + const random = seededRandom(toBeDeleted); + + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15); + + await c.env.KV.list({ prefix: uuid }).then(async (keys) => { + for (const key of keys.keys) { + await c.env.KV.delete(key.name); + await store.delete({ ids: [key.name] }); + } + }); +} + +export async function batchCreateChunksAndEmbeddings({ + store, + body, + chunks, + context, +}: { + store: CloudflareVectorizeStore; + body: z.infer; + chunks: string[]; + context: Context<{ Bindings: Env }>; +}) { + const ourID = `${body.url}-${body.user}`; + + await deleteDocument({ url: body.url, user: body.user, c: context, store }); + + const random = seededRandom(ourID); + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const uuid = + random().toString(36).substring(2, 15) + + random().toString(36).substring(2, 15) + + "-" + + i; + + const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${chunk}`; + + const docs = await store.addDocuments( + [ + { + pageContent: newPageContent, + metadata: { + title: body.title?.slice(0, 50) ?? "", + description: body.description ?? "", + space: body.space ?? "", + url: body.url, + user: body.user, + }, + }, + ], + { + ids: [uuid], + }, + ); + + console.log("Docs added: ", docs); + + await context.env.KV.put(uuid, ourID); + } +} diff --git a/apps/cf-ai-backend/src/index.test.ts b/apps/cf-ai-backend/src/index.test.ts new file mode 100644 index 00000000..bbf66fb5 --- /dev/null +++ b/apps/cf-ai-backend/src/index.test.ts @@ -0,0 +1,13 @@ +import app from "."; + +// TODO: write more tests +describe("Test the application", () => { + it("Should return 200 response", async () => { + const res = await app.request("http://localhost/"); + expect(res.status).toBe(200); + }), + it("Should return 404 response", async () => { + const res = await app.request("http://localhost/404"); + expect(res.status).toBe(404); + }); +}); diff --git a/apps/cf-ai-backend/src/index.ts b/apps/cf-ai-backend/src/index.ts index 317203c4..3e65ac90 100644 --- a/apps/cf-ai-backend/src/index.ts +++ b/apps/cf-ai-backend/src/index.ts @@ -1,50 +1,223 @@ -import type { VectorizeIndex, Fetcher, Request } from '@cloudflare/workers-types'; +import { z } from "zod"; +import { Hono } from "hono"; +import { CoreMessage, streamText } from "ai"; +import { chatObj, Env, vectorObj } from "./types"; +import { + batchCreateChunksAndEmbeddings, + deleteDocument, + initQuery, +} from "./helper"; +import { timing } from "hono/timing"; +import { logger } from "hono/logger"; +import { poweredBy } from "hono/powered-by"; +import { bearerAuth } from "hono/bearer-auth"; +import { zValidator } from "@hono/zod-validator"; +import chunkText from "./utils/chonker"; +import { systemPrompt, template } from "./prompts/prompt1"; -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from './OpenAIEmbedder'; -import { GoogleGenerativeAI } from '@google/generative-ai'; -import routeMap from './routes'; -import { queue } from './routes/queue'; +const app = new Hono<{ Bindings: Env }>(); -function isAuthorized(request: Request, env: Env): boolean { - return request.headers.get('X-Custom-Auth-Key') === env.SECURITY_KEY; -} +// ------- MIDDLEWARES ------- +app.use("*", poweredBy()); +app.use("*", timing()); +app.use("*", logger()); -export default { - async fetch(request: Request, env: Env, ctx: ExecutionContext) { - if (!isAuthorized(request, env)) { - return new Response('Unauthorized', { status: 401 }); - } +app.use("/api/", async (c, next) => { + const auth = bearerAuth({ token: c.env.SECURITY_KEY }); + return auth(c, next); +}); +// ------- MIDDLEWARES END ------- - const embeddings = new OpenAIEmbeddings({ - apiKey: env.OPENAI_API_KEY, - modelName: 'text-embedding-3-small', - }); +app.get("/", (c) => { + return c.text("Supermemory backend API is running!"); +}); - const store = new CloudflareVectorizeStore(embeddings, { - index: env.VECTORIZE_INDEX, - }); +app.get("/api/health", (c) => { + return c.json({ status: "ok" }); +}); - const genAI = new GoogleGenerativeAI(env.GOOGLE_AI_API_KEY); +app.post("/api/add", zValidator("json", vectorObj), async (c) => { + const body = c.req.valid("json"); - const model = genAI.getGenerativeModel({ model: 'gemini-pro' }); + const { store } = await initQuery(c); - const url = new URL(request.url); - const path = url.pathname; - const method = request.method.toUpperCase(); + await batchCreateChunksAndEmbeddings({ + store, + body, + chunks: chunkText(body.pageContent, 1536), + context: c, + }); - const routeHandlers = routeMap.get(path); + return c.json({ status: "ok" }); +}); - if (!routeHandlers) { - return new Response('Not Found', { status: 404 }); - } +app.get( + "/api/ask", + zValidator( + "query", + z.object({ + query: z.string(), + }), + ), + async (c) => { + const query = c.req.valid("query"); - const handler = routeHandlers[method]; + const { model } = await initQuery(c); - if (!handler) { - return new Response('Method Not Allowed', { status: 405 }); - } - return await handler(request, store, embeddings, model, env, ctx); - }, - queue, -}; + const response = await streamText({ model, prompt: query.query }); + const r = response.toTextStreamResponse(); + + return r; + }, +); + +app.post( + "/api/chat", + zValidator( + "query", + z.object({ + query: z.string(), + topK: z.number().optional().default(10), + user: z.string(), + spaces: z.string().optional(), + sourcesOnly: z.string().optional().default("false"), + model: z.string().optional().default("gpt-4o"), + }), + ), + zValidator("json", chatObj), + async (c) => { + const query = c.req.valid("query"); + const body = c.req.valid("json"); + + if (body.chatHistory) { + body.chatHistory = body.chatHistory.map((i) => ({ + ...i, + content: i.parts.length > 0 ? i.parts.join(" ") : i.content, + })); + } + + const sourcesOnly = query.sourcesOnly === "true"; + const spaces = query.spaces?.split(",") || [undefined]; + + const { model, store } = await initQuery(c, query.model); + + const filter: VectorizeVectorMetadataFilter = { user: query.user }; + + const queryAsVector = await store.embeddings.embedQuery(query.query); + const responses: VectorizeMatches = { matches: [], count: 0 }; + + for (const space of spaces) { + if (space !== undefined) { + filter.space = space; + } + + const resp = await c.env.VECTORIZE_INDEX.query(queryAsVector, { + topK: query.topK, + filter, + }); + + if (resp.count > 0) { + responses.matches.push(...resp.matches); + responses.count += resp.count; + } + } + + const minScore = Math.min(...responses.matches.map(({ score }) => score)); + const maxScore = Math.max(...responses.matches.map(({ score }) => score)); + + const normalizedData = responses.matches.map((data) => ({ + ...data, + normalizedScore: + maxScore !== minScore + ? 1 + ((data.score - minScore) / (maxScore - minScore)) * 98 + : 50, // If all scores are the same, set them to the middle of the scale + })); + + let highScoreData = normalizedData.filter( + ({ normalizedScore }) => normalizedScore > 50, + ); + if (highScoreData.length === 0) { + highScoreData = normalizedData + .sort((a, b) => b.score - a.score) + .slice(0, 3); + } + + const sortedHighScoreData = highScoreData.sort( + (a, b) => b.normalizedScore - a.normalizedScore, + ); + + console.log(JSON.stringify(sortedHighScoreData)); + + if (sourcesOnly) { + const idsAsStrings = sortedHighScoreData.map((dataPoint) => + dataPoint.id.toString(), + ); + + const storedContent = await Promise.all( + idsAsStrings.map(async (id) => await c.env.KV.get(id)), + ); + + return c.json({ ids: storedContent }); + } + + const vec = await c.env.VECTORIZE_INDEX.getByIds( + sortedHighScoreData.map(({ id }) => id), + ); + + const vecWithScores = vec.map((v, i) => ({ + ...v, + score: sortedHighScoreData[i].score, + })); + + const preparedContext = vecWithScores.map(({ metadata, score }) => ({ + context: `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, + score, + })); + + const initialMessages: CoreMessage[] = [ + { role: "user", content: systemPrompt }, + { role: "assistant", content: "Hello, how can I help?" }, + ]; + + const prompt = template({ + contexts: preparedContext, + question: query.query, + }); + + const userMessage: CoreMessage = { role: "user", content: prompt }; + + const response = await streamText({ + model, + messages: [ + ...initialMessages, + ...((body.chatHistory || []) as CoreMessage[]), + userMessage, + ], + temperature: 0.4, + }); + + return response.toTextStreamResponse(); + }, +); + +app.delete( + "/api/delete", + zValidator( + "query", + z.object({ + websiteUrl: z.string(), + user: z.string(), + }), + ), + async (c) => { + const { websiteUrl, user } = c.req.valid("query"); + + const { store } = await initQuery(c); + + await deleteDocument({ url: websiteUrl, user, c, store }); + + return c.json({ message: "Document deleted" }); + }, +); + +export default app; diff --git a/apps/cf-ai-backend/src/prompts/prompt1.ts b/apps/cf-ai-backend/src/prompts/prompt1.ts new file mode 100644 index 00000000..6e1bdf7b --- /dev/null +++ b/apps/cf-ai-backend/src/prompts/prompt1.ts @@ -0,0 +1,47 @@ +export const systemPrompt = `You are an AI assistant called Supermemory that acts as a "Second Brain" by answering questions based on provided context. Your goal is to directly address the question concisely and to the point, without excessive elaboration. + +Multiple pieces of context, each with an associated relevance score, will be provided. Each context piece and its score will be enclosed within the following tags: and . The question you need to answer will be enclosed within the tags. + +To generate your answer: +- Carefully analyze the question and identify the key information needed to address it +- Locate the specific parts of each context that contain this key information +- Compare the relevance scores of the provided contexts +- In the tags, provide a brief justification for which context(s) are more relevant to answering the question based on the scores +- Concisely summarize the relevant information from the higher-scoring context(s) in your own words +- Provide a direct answer to the question +- Use markdown formatting in your answer, including bold, italics, and bullet points as appropriate to improve readability and highlight key points +- Give detailed and accurate responses for things like 'write a blog' or long-form questions. + +Provide your justification between tags and your final answer between tags, formatting both in markdown. + +If no context is provided, introduce yourself and explain that the user can save content which will allow you to answer questions about that content in the future. Do not provide an answer if no context is provided.`; + +export const template = ({ contexts, question }) => { + // Map over contexts to generate the context and score parts + const contextParts = contexts + .map( + ({ context, score }) => ` + + ${context} + + + + ${score} + `, + ) + .join("\n"); + + // Construct the final prompt using a template literal + const finalPrompt = ` + Here's the given context and question for the task: + ${contextParts} + + The question is provided in the prompt below: + + + ${question} + + `; + + return finalPrompt.trim(); +}; diff --git a/apps/cf-ai-backend/src/routes.ts b/apps/cf-ai-backend/src/routes.ts deleted file mode 100644 index b9e1cdb4..00000000 --- a/apps/cf-ai-backend/src/routes.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import * as apiAdd from './routes/add'; -import * as apiQuery from './routes/query'; -import * as apiAsk from './routes/ask'; -import * as apiChat from './routes/chat'; -import * as apiBatchUploadTweets from './routes/batchUploadTweets'; -import * as apiGetPageContent from './routes/getPageContent'; -import * as apiDelete from './routes/delete'; -import * as apiEdit from './routes/edit'; -import * as apiWipeData from './routes/wipedata'; -import { OpenAIEmbeddings } from './OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; -import { Request } from '@cloudflare/workers-types'; - -type RouteHandler = ( - request: Request, - store: CloudflareVectorizeStore, - embeddings: OpenAIEmbeddings, - model: GenerativeModel, - env: Env, - ctx?: ExecutionContext, -) => Promise; - -const routeMap = new Map>(); - -routeMap.set('/add', apiAdd); - -routeMap.set('/query', apiQuery); - -routeMap.set('/ask', apiAsk); - -routeMap.set('/chat', apiChat); - -routeMap.set('/batchUploadTweets', apiBatchUploadTweets); -routeMap.set('/getPageContent', apiGetPageContent); -routeMap.set('/delete', apiDelete); -routeMap.set('/edit', apiEdit); - -routeMap.set('/wipedata', apiWipeData); -// Add more route mappings as needed -// routeMap.set('/api/otherRoute', { ... }); - -export default routeMap; diff --git a/apps/cf-ai-backend/src/routes/add.ts b/apps/cf-ai-backend/src/routes/add.ts deleted file mode 100644 index 173c12b9..00000000 --- a/apps/cf-ai-backend/src/routes/add.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Request } from '@cloudflare/workers-types'; -import { type CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; -import { seededRandom } from '../util'; - -export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) { - const body = (await request.json()) as { - pageContent: string; - title?: string; - description?: string; - space?: string; - url: string; - user: string; - }; - - if (!body.pageContent || !body.url) { - return new Response(JSON.stringify({ message: 'Invalid Page Content' }), { status: 400 }); - } - - // TODO: FIX THIS,BUT TEMPERORILY TRIM page content to 1000 words - body.pageContent = body.pageContent.split(' ').slice(0, 1000).join(' '); - - const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${body.pageContent}`; - - const ourID = `${body.url}-${body.user}`; - - const random = seededRandom(ourID); - const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - - await env.KV.put(uuid, ourID); - - await store.addDocuments( - [ - { - pageContent: newPageContent, - metadata: { - title: body.title?.slice(0, 50) ?? '', - description: body.description ?? '', - space: body.space ?? '', - url: body.url, - user: body.user, - }, - }, - ], - { - ids: [uuid], - }, - ); - - return new Response(JSON.stringify({ message: 'Document Added' }), { status: 200 }); -} diff --git a/apps/cf-ai-backend/src/routes/ask.ts b/apps/cf-ai-backend/src/routes/ask.ts deleted file mode 100644 index 267c1513..00000000 --- a/apps/cf-ai-backend/src/routes/ask.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { GenerativeModel } from '@google/generative-ai'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { Request } from '@cloudflare/workers-types'; - -export async function POST(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) { - const body = (await request.json()) as { - query: string; - }; - - if (!body.query) { - return new Response(JSON.stringify({ message: 'Invalid Page Content' }), { status: 400 }); - } - - const prompt = `You are an agent that answers a question based on the query. don't say 'based on the context'.\n\n Context:\n${body.query} \nAnswer this question based on the context. Question: ${body.query}\nAnswer:`; - const output = await model.generateContentStream(prompt); - - const response = new Response( - new ReadableStream({ - async start(controller) { - const converter = new TextEncoder(); - for await (const chunk of output.stream) { - const chunkText = await chunk.text(); - console.log(chunkText); - const encodedChunk = converter.encode('data: ' + JSON.stringify({ response: chunkText }) + '\n\n'); - controller.enqueue(encodedChunk); - } - const doneChunk = converter.encode('data: [DONE]'); - controller.enqueue(doneChunk); - controller.close(); - }, - }), - ); - return response; -} diff --git a/apps/cf-ai-backend/src/routes/batchUploadTweets.ts b/apps/cf-ai-backend/src/routes/batchUploadTweets.ts deleted file mode 100644 index 0370436b..00000000 --- a/apps/cf-ai-backend/src/routes/batchUploadTweets.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Request } from '@cloudflare/workers-types'; -import { type CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; - -export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) { - const body = (await request.json()) as TweetData[] | undefined; - - if (!body) { - return new Response(JSON.stringify({ message: 'Body is missing' }), { status: 400 }); - } - - const bytes = new TextEncoder().encode(JSON.stringify(body)).length; - - if (bytes < 128000) { - await env.MY_QUEUE.send(body); - } else { - let bytesTillNow = 0; - let batches: TweetData[] = []; - - const getByteLength = (data: string) => new TextEncoder().encode(data).length; - - for (let i = 0; i < body.length; i++) { - const byteLength = getByteLength(JSON.stringify(body[i])); - - if (bytesTillNow + byteLength < 100000) { - bytesTillNow += byteLength; - batches.push(body[i]); - } else { - await env.MY_QUEUE.send(batches); - batches = [body[i]]; - bytesTillNow = byteLength; - } - } - } - - return new Response(JSON.stringify({ message: 'Document Added' }), { status: 200 }); -} diff --git a/apps/cf-ai-backend/src/routes/chat.ts b/apps/cf-ai-backend/src/routes/chat.ts deleted file mode 100644 index 0a832933..00000000 --- a/apps/cf-ai-backend/src/routes/chat.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { Content, GenerativeModel } from '@google/generative-ai'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { Request } from '@cloudflare/workers-types'; - -export async function POST(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) { - const queryparams = new URL(request.url).searchParams; - const query = queryparams.get('q'); - const topK = parseInt(queryparams.get('topK') ?? '5'); - const user = queryparams.get('user'); - const spaces = queryparams.get('spaces') ?? undefined; - const sp = spaces === 'null' ? undefined : spaces; - const spacesArray = sp ? sp.split(',') : undefined; - - const sourcesOnly = queryparams.get('sourcesOnly') ?? 'false'; - - if (!user) { - return new Response(JSON.stringify({ message: 'Invalid User' }), { status: 400 }); - } - - if (!query) { - return new Response(JSON.stringify({ message: 'Invalid Query' }), { status: 400 }); - } - const filter: VectorizeVectorMetadataFilter = { - user, - }; - - const responses: VectorizeMatches = { matches: [], count: 0 }; - - if (spacesArray) { - for (const space of spacesArray) { - filter.space = space; - - const queryAsVector = await embeddings.embedQuery(query); - - const resp = await env!.VECTORIZE_INDEX.query(queryAsVector, { - topK, - filter, - }); - - if (resp.count > 0) { - responses.matches.push(...resp.matches); - responses.count += resp.count; - } - } - } else { - const queryAsVector = await embeddings.embedQuery(query); - const resp = await env!.VECTORIZE_INDEX.query(queryAsVector, { - topK, - filter: { - user, - }, - }); - - if (resp.count > 0) { - responses.matches.push(...resp.matches); - responses.count += resp.count; - } - } - - // if (responses.count === 0) { - // return new Response(JSON.stringify({ message: "No Results Found" }), { status: 404 }); - // } - - const highScoreIds = responses.matches.filter(({ score }) => score > 0.3).map(({ id }) => id); - - console.log('highscoreIds', highScoreIds); - - if (sourcesOnly === 'true') { - // Try await env.KV.get(id) for each id in a Promise.all - const idsAsStrings = highScoreIds.map(String); - - const storedContent = await Promise.all( - idsAsStrings.map(async (id) => { - const stored = await env!.KV.get(id); - if (stored) { - return stored; - } - return id; - }), - ); - - console.log(storedContent); - return new Response(JSON.stringify({ ids: storedContent }), { status: 200 }); - } - - const vec = await env!.VECTORIZE_INDEX.getByIds(highScoreIds); - - const preparedContext = vec - .map( - ({ metadata }) => - `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - ) - .join('\n\n'); - - const body = (await request.json()) as { - chatHistory?: Content[]; - }; - - const defaultHistory = [ - { - role: 'user', - parts: [ - { - text: `You are an agent that summarizes a page based on the query. don't say 'based on the context'. I expect you to be like a 'Second Brain'. you will be provided with the context (old saved posts) and questions. Answer accordingly. Answer in markdown format`, - }, - ], - }, - { - role: 'model', - parts: [{ text: "Ok, I am a personal assistant, and will act as a second brain to help with user's queries." }], - }, - ] as Content[]; - - const chat = model.startChat({ - history: [...defaultHistory, ...(body.chatHistory ?? [])], - }); - - const prompt = - `You are supermemory - an agent that answers a question based on the context provided. don't say 'based on the context'. Be concise and to the point, make sure that you are addressing the question properly but don't yap too much. I expect you to be like a 'Second Brain'. you will be provided with the context (old saved posts) and questions. Answer accordingly. Answer in markdown format. Use bold, italics, bullet points` + - `Context:\n${preparedContext == '' ? "No context, just introduce yourself and say something like 'I don't know, but you can save things from the sidebar on the right and then query me'" : preparedContext + `Question: ${query}\nAnswer:`}\n\n`; - - const output = await chat.sendMessageStream(prompt); - - const response = new Response( - new ReadableStream({ - async start(controller) { - const converter = new TextEncoder(); - for await (const chunk of output.stream) { - const chunkText = await chunk.text(); - const encodedChunk = converter.encode('data: ' + JSON.stringify({ response: chunkText }) + '\n\n'); - controller.enqueue(encodedChunk); - } - const doneChunk = converter.encode('data: [DONE]'); - controller.enqueue(doneChunk); - controller.close(); - }, - }), - ); - return response; -} diff --git a/apps/cf-ai-backend/src/routes/delete.ts b/apps/cf-ai-backend/src/routes/delete.ts deleted file mode 100644 index bb28ce9d..00000000 --- a/apps/cf-ai-backend/src/routes/delete.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Request } from '@cloudflare/workers-types'; -import { type CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; -import { seededRandom } from '../util'; - -export async function DELETE(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) { - const { searchParams } = new URL(request.url); - const websiteUrl = searchParams.get('websiteUrl'); - const user = searchParams.get('user'); - - if (!websiteUrl || !user) { - return new Response(JSON.stringify({ message: 'Invalid Request, need websiteUrl and user' }), { status: 400 }); - } - - const ourID = `${websiteUrl}-${user}`; - - const uuid = await env.KV.get(ourID); - - if (!uuid) { - return new Response(JSON.stringify({ message: 'Document not found' }), { status: 404 }); - } - - await store.delete({ ids: [uuid] }); - - return new Response(JSON.stringify({ message: 'Document deleted' }), { status: 200 }); -} diff --git a/apps/cf-ai-backend/src/routes/edit.ts b/apps/cf-ai-backend/src/routes/edit.ts deleted file mode 100644 index e32c93ca..00000000 --- a/apps/cf-ai-backend/src/routes/edit.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { Request } from '@cloudflare/workers-types'; -import { type CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; -import { seededRandom } from '../util'; - -export async function POST(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) { - const body = (await request.json()) as { - pageContent: string; - title?: string; - description?: string; - space?: string; - url: string; - user: string; - }; - - if (!body.pageContent || !body.url) { - return new Response(JSON.stringify({ message: 'Invalid Page Content' }), { status: 400 }); - } - - const { searchParams } = new URL(request.url); - const uniqueUrl = searchParams.get('uniqueUrl'); - - const toBeDeleted = `${uniqueUrl}-${body.user}`; - const tbduuid = await env.KV.get(toBeDeleted); - if (tbduuid) { - await store.delete({ ids: [tbduuid] }); - } - - // TODO: FIX THIS,BUT TEMPERORILY TRIM page content to 1000 words - body.pageContent = body.pageContent.split(' ').slice(0, 1000).join(' '); - - const newPageContent = `Title: ${body.title}\nDescription: ${body.description}\nURL: ${body.url}\nContent: ${body.pageContent}`; - - const ourID = `${body.url}-${body.user}`; - - const random = seededRandom(ourID); - const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - - await env.KV.put(uuid, ourID); - - await store.addDocuments( - [ - { - pageContent: newPageContent, - metadata: { - title: body.title?.slice(0, 50) ?? '', - description: body.description ?? '', - space: body.space ?? '', - url: body.url, - user: body.user, - }, - }, - ], - { - ids: [uuid], - }, - ); - - return new Response(JSON.stringify({ message: 'Document Added' }), { status: 200 }); -} diff --git a/apps/cf-ai-backend/src/routes/getPageContent.ts b/apps/cf-ai-backend/src/routes/getPageContent.ts deleted file mode 100644 index 4c465514..00000000 --- a/apps/cf-ai-backend/src/routes/getPageContent.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { GenerativeModel } from '@google/generative-ai'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { Request } from '@cloudflare/workers-types'; -import puppeteer from '@cloudflare/puppeteer'; - -// TODO: THIS DOESN'T WORK PROPERLY. FOR EG, FOR THIS URL https://dev.to/challenges/cloudflare, IT DOESN'T RETURN FULL CONTENT -export async function GET(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) { - const { searchParams } = new URL(request.url); - let url = searchParams.get('url'); - let img: Buffer; - if (url) { - url = new URL(url).toString(); // normalize - const browser = await puppeteer.launch(env?.MYBROWSER); - const page = await browser.newPage(); - await page.goto(url); - - await page.waitForSelector('body'); - - const contentElement = await page.$('body'); - const content = await page.evaluate((element) => element.innerText, contentElement); - - await browser.close(); - - return new Response(content, { - headers: { - 'content-type': 'text/html', - }, - }); - } else { - return new Response('Please add an ?url=https://example.com/ parameter'); - } -} diff --git a/apps/cf-ai-backend/src/routes/query.ts b/apps/cf-ai-backend/src/routes/query.ts deleted file mode 100644 index 090dfbb0..00000000 --- a/apps/cf-ai-backend/src/routes/query.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { GenerativeModel } from '@google/generative-ai'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { Request } from '@cloudflare/workers-types'; - -export async function GET(request: Request, _: CloudflareVectorizeStore, embeddings: OpenAIEmbeddings, model: GenerativeModel, env?: Env) { - const queryparams = new URL(request.url).searchParams; - const query = queryparams.get('q'); - const topK = parseInt(queryparams.get('topK') ?? '5'); - const user = queryparams.get('user'); - const space = queryparams.get('space'); - - const sourcesOnly = queryparams.get('sourcesOnly') ?? 'false'; - - if (!user) { - return new Response(JSON.stringify({ message: 'Invalid User' }), { status: 400 }); - } - - if (!query) { - return new Response(JSON.stringify({ message: 'Invalid Query' }), { status: 400 }); - } - - const filter: VectorizeVectorMetadataFilter = { - user, - }; - - if (space) { - filter.space; - } - - const queryAsVector = await embeddings.embedQuery(query); - - const resp = await env!.VECTORIZE_INDEX.query(queryAsVector, { - topK, - filter, - }); - - if (resp.count === 0) { - return new Response(JSON.stringify({ message: 'No Results Found' }), { status: 404 }); - } - - const highScoreIds = resp.matches.filter(({ score }) => score > 0.3).map(({ id }) => id); - - if (sourcesOnly === 'true') { - const idsAsStrings = highScoreIds.map(String); - - const storedContent = await Promise.all( - idsAsStrings.map(async (id) => { - const stored = await env!.KV.get(id); - if (stored) { - return stored; - } - return id; - }), - ); - - console.log(storedContent); - return new Response(JSON.stringify({ ids: storedContent }), { status: 200 }); - } - - const vec = await env!.VECTORIZE_INDEX.getByIds(highScoreIds); - - if (vec.length === 0 || !vec[0].metadata) { - return new Response(JSON.stringify({ message: 'No Results Found' }), { status: 400 }); - } - - const preparedContext = vec - .slice(0, 3) - .map( - ({ metadata }) => - `Website title: ${metadata!.title}\nDescription: ${metadata!.description}\nURL: ${metadata!.url}\nContent: ${metadata!.text}`, - ) - .join('\n\n'); - - const prompt = `You are an agent that summarizes a page based on the query. Be direct and concise, don't say 'based on the context'.\n\n Context:\n${preparedContext} \nAnswer this question based on the context. Question: ${query}\nAnswer:`; - const output = await model.generateContentStream(prompt); - - const response = new Response( - new ReadableStream({ - async start(controller) { - const converter = new TextEncoder(); - for await (const chunk of output.stream) { - const chunkText = await chunk.text(); - const encodedChunk = converter.encode('data: ' + JSON.stringify({ response: chunkText }) + '\n\n'); - controller.enqueue(encodedChunk); - } - const doneChunk = converter.encode('data: [DONE]'); - controller.enqueue(doneChunk); - controller.close(); - }, - }), - ); - return response; -} diff --git a/apps/cf-ai-backend/src/routes/queue.ts b/apps/cf-ai-backend/src/routes/queue.ts deleted file mode 100644 index 89a00c4b..00000000 --- a/apps/cf-ai-backend/src/routes/queue.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { seededRandom } from '../util'; - -export const queue = async (batch: MessageBatch, env: Env): Promise => { - const messages = batch.messages[0].body as TweetData[]; - - const token = messages[0].saveToUser; - - if (!token) { - return; - } - - const limits = (await fetch('https://supermemory.dhr.wtf/api/getCount', { - headers: { - Authorization: `Bearer ${token}`, - }, - }).then((res) => res.json())) as { tweetsLimit: number; tweetsCount: number; user: string }; - - if (messages.length > limits.tweetsLimit - limits.tweetsCount) { - messages.splice(limits.tweetsLimit - limits.tweetsCount); - } - - if (messages.length === 0) { - return; - } - - const embeddings = new OpenAIEmbeddings({ - apiKey: env.OPENAI_API_KEY, - modelName: 'text-embedding-3-small', - }); - - const store = new CloudflareVectorizeStore(embeddings, { - index: env.VECTORIZE_INDEX, - }); - - const collectedDocsUUIDs: { - document: { - pageContent: string; - metadata: { title: string; description: string; space?: string; url: string; user: string }; - id: string; - }; - }[] = []; - - messages.forEach(async (message) => { - const ourID = `${message.postUrl}-${limits.user}`; - - const random = seededRandom(ourID); - const uuid = random().toString(36).substring(2, 15) + random().toString(36).substring(2, 15); - - await env.KV.put(uuid, ourID); - const pageContent = `This is a tweet from ${message.authorName}, it was posted on ${message.time}. The tweet reads: ${message.tweetText}`; - - collectedDocsUUIDs.push({ - document: { - pageContent, - metadata: { - title: 'Twitter Bookmark', - description: '', - url: message.postUrl, - user: limits.user, - }, - id: uuid, - }, - }); - }); - - console.log(collectedDocsUUIDs); - - await store.addDocuments( - collectedDocsUUIDs.map(({ document }) => document), - { - ids: collectedDocsUUIDs.map(({ document }) => document.id), - }, - ); - - console.log(token); - - const res = await fetch('https://supermemory.dhr.wtf/api/addTweetsToDb', { - method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify(messages), - }); - - console.log(res.status, res.statusText); - - if (res.status !== 200) { - console.log(await res.json()); - console.error('Error adding tweets to db'); - } - - console.log(`consumed from our queue: ${messages}`); -}; diff --git a/apps/cf-ai-backend/src/routes/wipedata.ts b/apps/cf-ai-backend/src/routes/wipedata.ts deleted file mode 100644 index 841330f8..00000000 --- a/apps/cf-ai-backend/src/routes/wipedata.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Request } from '@cloudflare/workers-types'; -import { type CloudflareVectorizeStore } from '@langchain/cloudflare'; -import { OpenAIEmbeddings } from '../OpenAIEmbedder'; -import { GenerativeModel } from '@google/generative-ai'; -import { seededRandom } from '../util'; - -// TODO: Waiting for cloudflare to implement tojson so i can get all IDS for that user and delete them -export async function DELETE(request: Request, store: CloudflareVectorizeStore, _: OpenAIEmbeddings, m: GenerativeModel, env: Env) { - const { searchParams } = new URL(request.url); - const user = searchParams.get('user'); - - console.log(store.toJSONNotImplemented()); - - // for (const match of matches.matches) { - // await store.delete({ ids: [match.id] }); - // } - - return new Response(JSON.stringify({ message: 'Document deleted' }), { status: 200 }); -} diff --git a/apps/cf-ai-backend/src/types.ts b/apps/cf-ai-backend/src/types.ts new file mode 100644 index 00000000..3b6db589 --- /dev/null +++ b/apps/cf-ai-backend/src/types.ts @@ -0,0 +1,48 @@ +import { z } from "zod"; + +export type Env = { + VECTORIZE_INDEX: VectorizeIndex; + AI: Fetcher; + SECURITY_KEY: string; + OPENAI_API_KEY: string; + GOOGLE_AI_API_KEY: string; + MY_QUEUE: Queue; + KV: KVNamespace; + MYBROWSER: unknown; + ANTHROPIC_API_KEY: string; +}; + +export interface TweetData { + tweetText: string; + postUrl: string; + authorName: string; + handle: string; + time: string; + saveToUser: string; +} + +export const contentObj = z.object({ + role: z.string(), + parts: z + .array( + z.object({ + text: z.string(), + }), + ) + .transform((val) => val.map((v) => v.text)) + .optional(), + content: z.string().optional(), +}); + +export const chatObj = z.object({ + chatHistory: z.array(contentObj).optional(), +}); + +export const vectorObj = z.object({ + pageContent: z.string(), + title: z.string().optional(), + description: z.string().optional(), + space: z.string().optional(), + url: z.string(), + user: z.string(), +}); diff --git a/apps/cf-ai-backend/src/util.ts b/apps/cf-ai-backend/src/util.ts deleted file mode 100644 index 81f7628c..00000000 --- a/apps/cf-ai-backend/src/util.ts +++ /dev/null @@ -1,7 +0,0 @@ -export function seededRandom(seed: string) { - let x = [...seed].reduce((acc, cur) => acc + cur.charCodeAt(0), 0); - return () => { - x = (x * 9301 + 49297) % 233280; - return x / 233280; - }; -} diff --git a/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts new file mode 100644 index 00000000..3514f579 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/OpenAIEmbedder.ts @@ -0,0 +1,43 @@ +interface OpenAIEmbeddingsParams { + apiKey: string; + modelName: string; +} + +export class OpenAIEmbeddings { + private apiKey: string; + private modelName: string; + + constructor({ apiKey, modelName }: OpenAIEmbeddingsParams) { + this.apiKey = apiKey; + this.modelName = modelName; + } + + async embedDocuments(texts: string[]): Promise { + const responses = await Promise.all( + texts.map((text) => this.embedQuery(text)), + ); + return responses; + } + + async embedQuery(text: string): Promise { + const response = await fetch("https://api.openai.com/v1/embeddings", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify({ + input: text, + model: this.modelName, + }), + }); + + const data = (await response.json()) as { + data: { + embedding: number[]; + }[]; + }; + + return data.data[0].embedding; + } +} diff --git a/apps/cf-ai-backend/src/utils/chonker.ts b/apps/cf-ai-backend/src/utils/chonker.ts new file mode 100644 index 00000000..39d4b458 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/chonker.ts @@ -0,0 +1,44 @@ +import nlp from "compromise"; + +export default function chunkText( + text: string, + maxChunkSize: number, + overlap: number = 0.2, +): string[] { + const sentences = nlp(text).sentences().out("array"); + const chunks = []; + let currentChunk: string[] = []; + let currentSize = 0; + + for (let i = 0; i < sentences.length; i++) { + const sentence = sentences[i]; + currentChunk.push(sentence); + currentSize += sentence.length; + + if (currentSize >= maxChunkSize) { + // Calculate overlap + const overlapSize = Math.floor(currentChunk.length * overlap); + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: i - currentChunk.length + 1, + end: i, + }); + + // Prepare the next chunk with overlap + currentChunk = currentChunk.slice(-overlapSize); + currentSize = currentChunk.reduce((sum, s) => sum + s.length, 0); + } + } + + if (currentChunk.length > 0) { + const chunkText = currentChunk.join(" "); + chunks.push({ + text: chunkText, + start: sentences.length - currentChunk.length, + end: sentences.length, + }); + } + + return chunks.map((chunk) => chunk.text); +} diff --git a/apps/cf-ai-backend/src/utils/seededRandom.ts b/apps/cf-ai-backend/src/utils/seededRandom.ts new file mode 100644 index 00000000..36a1e4f9 --- /dev/null +++ b/apps/cf-ai-backend/src/utils/seededRandom.ts @@ -0,0 +1,18 @@ +import { MersenneTwister19937, integer } from "random-js"; + +function hashString(seed: string) { + let hash = 0; + for (let i = 0; i < seed.length; i++) { + const char = seed.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash |= 0; // Convert to 32bit integer + } + return hash; +} + +export function seededRandom(seed: string) { + const seedHash = hashString(seed); + const engine = MersenneTwister19937.seed(seedHash); + return () => + integer(0, Number.MAX_SAFE_INTEGER)(engine) / Number.MAX_SAFE_INTEGER; +} diff --git a/apps/cf-ai-backend/tsconfig.json b/apps/cf-ai-backend/tsconfig.json index b08cfd92..9f6f8a73 100644 --- a/apps/cf-ai-backend/tsconfig.json +++ b/apps/cf-ai-backend/tsconfig.json @@ -1,103 +1,6 @@ { - "compilerOptions": { - /* Visit https://aka.ms/tsconfig.json to read more about this file */ - - /* Projects */ - // "incremental": true, /* Enable incremental compilation */ - // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ - // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ - // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ - // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ - // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ - - /* Language and Environment */ - "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, - "lib": ["es2021"] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, - "jsx": "react" /* Specify what JSX code is generated. */, - // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ - // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ - // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ - // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ - // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ - // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ - // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ - // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ - - /* Modules */ - "module": "es2022" /* Specify what module code is generated. */, - // "rootDir": "./", /* Specify the root folder within your source files. */ - "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ - // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ - // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ - // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ - "types": [ - "@cloudflare/workers-types/2023-07-01" - ] /* Specify type package names to be included without being referenced in a source file. */, - // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ - "resolveJsonModule": true /* Enable importing .json files */, - // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ - - /* JavaScript Support */ - "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, - "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, - // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ - - /* Emit */ - // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ - // "declarationMap": true, /* Create sourcemaps for d.ts files. */ - // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ - // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ - // "outDir": "./", /* Specify an output folder for all emitted files. */ - // "removeComments": true, /* Disable emitting comments. */ - "noEmit": true /* Disable emitting files from a compilation. */, - // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ - // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ - // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ - // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ - // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ - // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ - // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ - // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ - // "newLine": "crlf", /* Set the newline character for emitting files. */ - // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ - // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ - // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ - // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ - // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ - // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ - - /* Interop Constraints */ - "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, - "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, - // "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, - // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ - "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, - - /* Type Checking */ - "strict": true /* Enable all strict type-checking options. */, - // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ - // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ - // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ - // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ - // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ - // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ - // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ - // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ - // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ - // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ - // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ - // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ - // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ - // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ - // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ - // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ - // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ - // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ - - /* Completeness */ - // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ - "skipLibCheck": true /* Skip type checking all .d.ts files. */ - } + "compilerOptions": { + "lib": ["ES2020"], + "types": ["jest", "@cloudflare/workers-types"] + } } diff --git a/apps/cf-ai-backend/vite.config.ts b/apps/cf-ai-backend/vite.config.ts new file mode 100644 index 00000000..b7647b0d --- /dev/null +++ b/apps/cf-ai-backend/vite.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vite"; +import honox from "honox/vite"; + +export default defineConfig({ + plugins: [honox()], +}); diff --git a/apps/cf-ai-backend/wrangler.toml b/apps/cf-ai-backend/wrangler.toml index 2d43bd93..83e2d41a 100644 --- a/apps/cf-ai-backend/wrangler.toml +++ b/apps/cf-ai-backend/wrangler.toml @@ -1,29 +1,26 @@ -name = "cf-ai-backend" -main = "src/index.ts" +name = "new-cf-ai-backend" +main = "src/server.ts" compatibility_date = "2024-02-23" -compatibility_flags = ['nodejs_compat'] +node_compat = true [[vectorize]] binding = "VECTORIZE_INDEX" -index_name = "supermem-vector-1" +index_name = "supermem-vector" [ai] binding = "AI" -[[queues.producers]] - queue = "batch-vector-queue" - binding = "MY_QUEUE" +# [[queues.producers]] +# queue = "batch-vector-queue" +# binding = "MY_QUEUE" - [[queues.consumers]] - queue = "batch-vector-queue" +# [[queues.consumers]] +# queue = "batch-vector-queue" [[kv_namespaces]] binding = "KV" id = "37a90353da63401e84e20e71165531d0" preview_id = "c58b6202814f4224acea97627d0c18aa" -[browser] -binding = "MYBROWSER" - [placement] mode = "smart" diff --git a/apps/web/src/app/content.tsx b/apps/web/src/app/content.tsx index effd06e0..5a68d902 100644 --- a/apps/web/src/app/content.tsx +++ b/apps/web/src/app/content.tsx @@ -1,20 +1,18 @@ "use client"; +import SessionProviderWrapper from "@/components/dev/SessionProviderWrapper"; import Main from "@/components/Main"; import Sidebar from "@/components/Sidebar/index"; -import { SessionProvider } from "next-auth/react"; import { useState } from "react"; export default function Content({ jwt }: { jwt: string }) { const [selectedItem, setSelectedItem] = useState(null); return ( -
- -
- -
-
- -
+ +
+ +
+
+ ); } diff --git a/apps/web/src/app/layout.tsx b/apps/web/src/app/layout.tsx index 0c79588e..e96df271 100644 --- a/apps/web/src/app/layout.tsx +++ b/apps/web/src/app/layout.tsx @@ -53,7 +53,10 @@ export default function RootLayout({ > -
+
{children}
diff --git a/apps/web/src/app/page.tsx b/apps/web/src/app/page.tsx index 5523fba6..05cd1ab8 100644 --- a/apps/web/src/app/page.tsx +++ b/apps/web/src/app/page.tsx @@ -12,6 +12,8 @@ import { redirect } from "next/navigation"; import { fetchContentForSpace, fetchFreeMemories } from "@/actions/db"; import { MemoryProvider } from "@/contexts/MemoryContext"; import Content from "./content"; +import Main from "@/components/Main"; +import { TailwindIndicator } from "@/components/dev/tailwindindicator"; export const runtime = "edge"; @@ -88,6 +90,7 @@ export default async function Home() { cachedMemories={contents} > + {/* */} ); diff --git a/apps/web/src/assets/Memories.tsx b/apps/web/src/assets/Memories.tsx index 3b1c177f..cafcd54f 100644 --- a/apps/web/src/assets/Memories.tsx +++ b/apps/web/src/assets/Memories.tsx @@ -14,8 +14,8 @@ export const MemoryIcon: React.FC> = ( height="43.0286" rx="5.5" transform="rotate(-12 0.40697 8.52821)" - fill="var(--gray-5)" - stroke="var(--gray-10)" + fill="white" + stroke="black" /> > = ( width="43" height="43" rx="5.5" - fill="var(--gray-5)" - stroke="var(--gray-10)" + fill="white" + stroke="black" /> > = ( height="43.0286" rx="5.5" transform="rotate(15 47.6965 -0.612372)" - fill="var(--gray-5)" - stroke="var(--gray-10)" + fill="white" + stroke="black" /> ); diff --git a/apps/web/src/assets/MemoryWithImages.tsx b/apps/web/src/assets/MemoryWithImages.tsx index 5cd6f8ca..6f7ba90a 100644 --- a/apps/web/src/assets/MemoryWithImages.tsx +++ b/apps/web/src/assets/MemoryWithImages.tsx @@ -20,7 +20,7 @@ export const MemoryWithImage: React.FC< width="72.207" height="72.207" rx="10" - fill="var(--gray-4)" + fill="white" /> { @@ -10,27 +14,26 @@ export interface Props extends React.ButtonHTMLAttributes { } export function ProfileDrawer({ className, hide = false, ...props }: Props) { + const { data: session } = useSession(); - const { data: session } = useSession(); - return ( - - - - + + + + -
- -
+
+ +
); diff --git a/apps/web/src/components/Sidebar/AddMemoryDialog.tsx b/apps/web/src/components/Sidebar/AddMemoryDialog.tsx index 63f0d122..64147b1e 100644 --- a/apps/web/src/components/Sidebar/AddMemoryDialog.tsx +++ b/apps/web/src/components/Sidebar/AddMemoryDialog.tsx @@ -18,19 +18,29 @@ import { cleanUrl } from "@/lib/utils"; import { motion } from "framer-motion"; import { getMetaData } from "@/server/helpers"; -export function AddMemoryPage({ closeDialog, defaultSpaces, onAdd }: { closeDialog: () => void, defaultSpaces?: number[], onAdd?: (addedData: StoredContent) => void }) { +export function AddMemoryPage({ + closeDialog, + defaultSpaces, + onAdd, +}: { + closeDialog: () => void; + defaultSpaces?: number[]; + onAdd?: (addedData: StoredContent) => void; +}) { const { addMemory } = useMemory(); const [loading, setLoading] = useState(false); const [url, setUrl] = useState(""); - const [selectedSpacesId, setSelectedSpacesId] = useState(defaultSpaces ?? []); + const [selectedSpacesId, setSelectedSpacesId] = useState( + defaultSpaces ?? [], + ); return ( -
+
Add a web page to memory - This will fetch the content of the web page and add it to the memory + This will fetch the content of the web page and add it to the memory @@ -38,7 +48,7 @@ export function AddMemoryPage({ closeDialog, defaultSpaces, onAdd }: { closeDial placeholder="Enter the URL of the page" type="url" data-modal-autofocus - className="bg-rgray-4 mt-2 w-full disabled:cursor-not-allowed disabled:opacity-70" + className="mt-2 w-full disabled:cursor-not-allowed disabled:opacity-70" value={url} onChange={(e) => setUrl(e.target.value)} disabled={loading} @@ -47,7 +57,7 @@ export function AddMemoryPage({ closeDialog, defaultSpaces, onAdd }: { closeDial @@ -69,10 +79,10 @@ export function AddMemoryPage({ closeDialog, defaultSpaces, onAdd }: { closeDial }, selectedSpacesId, ); - if (data) onAdd?.(data.memory) + if (data) onAdd?.(data.memory); closeDialog(); }} - className="bg-rgray-4 hover:bg-rgray-5 focus-visible:bg-rgray-5 focus-visible:ring-rgray-7 relative rounded-md px-4 py-2 ring-transparent transition focus-visible:outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-70" + className="bg-rgray-4 focus-visible:ring-rgray-7 relative rounded-md px-4 py-2 ring-transparent transition hover:bg-slate-100 focus-visible:bg-slate-100 focus-visible:outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-70" > Cancel @@ -99,10 +109,20 @@ export function AddMemoryPage({ closeDialog, defaultSpaces, onAdd }: { closeDial ); } -export function NoteAddPage({ closeDialog, defaultSpaces, onAdd }: { closeDialog: () => void, defaultSpaces?: number[], onAdd?: (addedData: StoredContent) => void }) { +export function NoteAddPage({ + closeDialog, + defaultSpaces, + onAdd, +}: { + closeDialog: () => void; + defaultSpaces?: number[]; + onAdd?: (addedData: StoredContent) => void; +}) { const { addMemory } = useMemory(); - const [selectedSpacesId, setSelectedSpacesId] = useState(defaultSpaces ?? []); + const [selectedSpacesId, setSelectedSpacesId] = useState( + defaultSpaces ?? [], + ); const inputRef = useRef(null); const [name, setName] = useState(""); @@ -137,7 +157,7 @@ export function NoteAddPage({ closeDialog, defaultSpaces, onAdd }: { closeDialog { - if (data?.memory) onAdd?.(data.memory) - closeDialog() - }); + if (data?.memory) onAdd?.(data.memory); + closeDialog(); + }); } }} disabled={loading} - className="bg-rgray-4 hover:bg-rgray-5 focus-visible:bg-rgray-5 focus-visible:ring-rgray-7 relative rounded-md px-4 py-2 ring-transparent transition focus-visible:outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-70" + className="hover:bg-rgray-5 focus-visible:bg-rgray-5 focus-visible:ring-rgray-7 relative rounded-md bg-[#F4F3F2] px-4 py-2 ring-transparent transition focus-visible:outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-70" > void, onAdd?: (addedData: StoredSpace) => void }) { +export function SpaceAddPage({ + closeDialog, + onAdd, +}: { + closeDialog: () => void; + onAdd?: (addedData: StoredSpace) => void; +}) { const { addSpace } = useMemory(); const inputRef = useRef(null); @@ -256,7 +282,7 @@ export function SpaceAddPage({ closeDialog, onAdd }: { closeDialog: () => void, value={name} disabled={loading} onChange={(e) => setName(e.target.value)} - className="bg-rgray-4 mt-2 w-full placeholder:transition placeholder:duration-500 data-[error=true]:placeholder:text-red-400 focus-visible:data-[error=true]:ring-red-500/10" + className="mt-2 w-full placeholder:transition placeholder:duration-500 data-[error=true]:placeholder:text-red-400 focus-visible:data-[error=true]:ring-red-500/10" /> {selected.length > 0 && ( <> @@ -279,7 +305,7 @@ export function SpaceAddPage({ closeDialog, onAdd }: { closeDialog: () => void, selected={selected} setSelected={setSelected} disabled={loading} - className="hover:bg-rgray-4 focus-visible:bg-rgray-4 mr-auto bg-white/5 disabled:cursor-not-allowed disabled:opacity-70" + className="mr-auto bg-white/5 hover:hover:bg-slate-100 focus-visible:hover:bg-slate-100 disabled:cursor-not-allowed disabled:opacity-70" > Memory @@ -293,13 +319,13 @@ export function SpaceAddPage({ closeDialog, onAdd }: { closeDialog: () => void, name, selected.map((s) => s.id), ).then((data) => { - if (data) onAdd?.(data.space) - closeDialog() - }); + if (data) onAdd?.(data.space); + closeDialog(); + }); } }} disabled={loading} - className="bg-rgray-4 hover:bg-rgray-5 focus-visible:bg-rgray-5 focus-visible:ring-rgray-7 relative rounded-md px-4 py-2 ring-transparent transition focus-visible:outline-none focus-visible:ring-2 disabled:cursor-not-allowed disabled:opacity-70" + className="bg-rgray-4 focus-visible:ring-rgray-7 relative rounded-md px-4 py-2 ring-transparent transition hover:hover:bg-slate-100 focus-visible:outline-none focus-visible:ring-2 focus-visible:hover:bg-slate-100 disabled:cursor-not-allowed disabled:opacity-70" > void, Cancel @@ -338,15 +364,17 @@ export function MemorySelectedItem({
{title} @@ -357,17 +385,17 @@ export function MemorySelectedItem({ } export function AddExistingMemoryToSpace({ - space, - closeDialog, - fromSpaces, - notInSpaces, - onAdd -}: { - space: { title: string, id: number }, - closeDialog: () => void, - fromSpaces?: number[], - notInSpaces?: number[], - onAdd?: () => void; + space, + closeDialog, + fromSpaces, + notInSpaces, + onAdd, +}: { + space: { title: string; id: number }; + closeDialog: () => void; + fromSpaces?: number[]; + notInSpaces?: number[]; + onAdd?: () => void; }) { const { addMemoriesToSpace } = useMemory(); @@ -379,9 +407,9 @@ export function AddExistingMemoryToSpace({
Add an existing memory to {space.title} - - Pick the memories you want to add to this space - + + Pick the memories you want to add to this space + {selected.length > 0 && ( <> @@ -404,8 +432,8 @@ export function AddExistingMemoryToSpace({ selected={selected} setSelected={setSelected} disabled={loading} - fromSpaces={fromSpaces} - notInSpaces={notInSpaces} + fromSpaces={fromSpaces} + notInSpaces={notInSpaces} className="hover:bg-rgray-4 focus-visible:bg-rgray-4 mr-auto bg-white/5 disabled:cursor-not-allowed disabled:opacity-70" > @@ -414,11 +442,14 @@ export function AddExistingMemoryToSpace({ Cancel diff --git a/apps/web/src/components/Sidebar/EditNoteDialog.tsx b/apps/web/src/components/Sidebar/EditNoteDialog.tsx index d3278df6..c0ad716d 100644 --- a/apps/web/src/components/Sidebar/EditNoteDialog.tsx +++ b/apps/web/src/components/Sidebar/EditNoteDialog.tsx @@ -1,9 +1,5 @@ - import { Editor } from "novel"; -import { - DialogClose, - DialogFooter, -} from "../ui/dialog"; +import { DialogClose, DialogFooter } from "../ui/dialog"; import { Input } from "../ui/input"; import { Markdown } from "tiptap-markdown"; import { useEffect, useRef, useState } from "react"; @@ -16,11 +12,18 @@ import { fetchContent } from "@/actions/db"; import { isArraysEqual } from "@/lib/utils"; import DeleteConfirmation from "./DeleteConfirmation"; - -export function NoteEdit({ memory, closeDialog, onDelete }: { memory: StoredContent, closeDialog: () => any, onDelete?: () => void }) { +export function NoteEdit({ + memory, + closeDialog, + onDelete, +}: { + memory: StoredContent; + closeDialog: () => any; + onDelete?: () => void; +}) { const { updateMemory, deleteMemory } = useMemory(); - const [initialSpaces, setInitialSpaces] = useState([]) + const [initialSpaces, setInitialSpaces] = useState([]); const [selectedSpacesId, setSelectedSpacesId] = useState([]); const inputRef = useRef(null); @@ -51,21 +54,21 @@ export function NoteEdit({ memory, closeDialog, onDelete }: { memory: StoredCont return true; } - useEffect(() => { - fetchContent(memory.id).then((data) => { - if (data?.spaces) { - setInitialSpaces(data.spaces) - setSelectedSpacesId(data.spaces) - } - }) - }, []) + useEffect(() => { + fetchContent(memory.id).then((data) => { + if (data?.spaces) { + setInitialSpaces(data.spaces); + setSelectedSpacesId(data.spaces); + } + }); + }, []); return (
- { - deleteMemory(memory.id) - onDelete?.() - }}> - - + { + deleteMemory(memory.id); + onDelete?.(); + }} + > + + Cancel @@ -149,4 +153,3 @@ export function NoteEdit({ memory, closeDialog, onDelete }: { memory: StoredCont
); } - diff --git a/apps/web/src/components/Sidebar/FilterCombobox.tsx b/apps/web/src/components/Sidebar/FilterCombobox.tsx index 576dd96b..634a09e3 100644 --- a/apps/web/src/components/Sidebar/FilterCombobox.tsx +++ b/apps/web/src/components/Sidebar/FilterCombobox.tsx @@ -64,88 +64,90 @@ export function FilterSpaces({ }, [open]); return ( - - - - - e.preventDefault()} - > - - spaces - .find((s) => s.id.toString() === val) - ?.name.toLowerCase() - .includes(search.toLowerCase().trim()) - ? 1 - : 0 - } - > - - - - Nothing found - - {sortedSpaces.map((space) => ( - { - setSelectedSpaces((prev: number[]) => - prev.includes(parseInt(val)) - ? prev.filter((v) => v !== parseInt(val)) - : [...prev, parseInt(val)], - ); - }} - asChild - > - - - {space.name.length > 10 ? space.name.slice(0, 10) + "..." : space.name} - {selectedSpaces.includes(space.id)} - - - - ))} - - - - - - + + + + + e.preventDefault()} + > + + spaces + .find((s) => s.id.toString() === val) + ?.name.toLowerCase() + .includes(search.toLowerCase().trim()) + ? 1 + : 0 + } + > + + + + Nothing found + + {sortedSpaces.map((space) => ( + { + setSelectedSpaces((prev: number[]) => + prev.includes(parseInt(val)) + ? prev.filter((v) => v !== parseInt(val)) + : [...prev, parseInt(val)], + ); + }} + asChild + > + + + {space.name.length > 10 + ? space.name.slice(0, 10) + "..." + : space.name} + {selectedSpaces.includes(space.id)} + + + + ))} + + + + + + ); } @@ -155,8 +157,8 @@ export type FilterMemoriesProps = { onClose?: () => void; selected: StoredContent[]; setSelected: React.Dispatch>; - fromSpaces?: number[]; - notInSpaces?: number[]; + fromSpaces?: number[]; + notInSpaces?: number[]; } & React.ButtonHTMLAttributes; export function FilterMemories({ @@ -166,8 +168,8 @@ export function FilterMemories({ onClose, selected, setSelected, - fromSpaces, - notInSpaces, + fromSpaces, + notInSpaces, ...props }: FilterMemoriesProps) { const { search } = useMemory(); @@ -195,9 +197,10 @@ export function FilterMemories({ memories: true, spaces: false, }, - memoriesRelativeToSpace: { - fromSpaces, notInSpaces - } + memoriesRelativeToSpace: { + fromSpaces, + notInSpaces, + }, }); setSearchResults(results); setIsSearching(false); @@ -275,7 +278,9 @@ export function FilterMemories({ } className="mr-2 h-4 w-4" /> - {(m.title && m.title?.length > 14) ? m.title?.slice(0, 14) + "..." : m.title} + {m.title && m.title?.length > 14 + ? m.title?.slice(0, 14) + "..." + : m.title} i.id === m.id) !== undefined diff --git a/apps/web/src/components/Sidebar/MemoriesBar.tsx b/apps/web/src/components/Sidebar/MemoriesBar.tsx index 5d9cf01e..a81d00c0 100644 --- a/apps/web/src/components/Sidebar/MemoriesBar.tsx +++ b/apps/web/src/components/Sidebar/MemoriesBar.tsx @@ -28,19 +28,16 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { Variant, useAnimate, motion } from "framer-motion"; import { SearchResult, useMemory } from "@/contexts/MemoryContext"; import { SpaceIcon } from "@/assets/Memories"; -import { - Dialog, - DialogContent, - DialogTitle, - DialogDescription, - DialogHeader, - DialogFooter, - DialogClose, -} from "../ui/dialog"; +import { Dialog, DialogContent } from "../ui/dialog"; import useViewport from "@/hooks/useViewport"; import useTouchHold from "@/hooks/useTouchHold"; import { DialogTrigger } from "@radix-ui/react-dialog"; -import { AddExistingMemoryToSpace, AddMemoryPage, MemorySelectedItem, NoteAddPage, SpaceAddPage } from "./AddMemoryDialog"; +import { + AddExistingMemoryToSpace, + AddMemoryPage, + NoteAddPage, + SpaceAddPage, +} from "./AddMemoryDialog"; import { ExpandedSpace } from "./ExpandedSpace"; import { StoredContent, StoredSpace } from "@/server/db/schema"; import { useDebounce } from "@/hooks/useDebounce"; @@ -48,7 +45,6 @@ import { NoteEdit } from "./EditNoteDialog"; import DeleteConfirmation from "./DeleteConfirmation"; export function MemoriesBar({ isOpen }: { isOpen: boolean }) { - const [parent, enableAnimations] = useAutoAnimate(); const { spaces, deleteSpace, freeMemories, search } = useMemory(); @@ -80,17 +76,17 @@ export function MemoriesBar({ isOpen }: { isOpen: boolean }) { })(); }, [query]); - useEffect(() => { - if (!isOpen) { - setExpandedSpace(null) - } - }, [isOpen]) + useEffect(() => { + if (!isOpen) { + setExpandedSpace(null); + } + }, [isOpen]); if (expandedSpace) { return ( setExpandedSpace(null)} + back={() => setExpandedSpace(null)} // close={() => setExpandedSpace(null)} /> ); @@ -160,30 +156,34 @@ export function MemoriesBar({ isOpen }: { isOpen: boolean }) {
{query.trim().length > 0 ? ( <> {searchResults.map(({ type, space, memory }, i) => ( <> {type === "memory" && ( - { - setSearchResults(prev => prev.filter(i => i.memory?.id !== memory.id)) - }} - /> - )} + { + setSearchResults((prev) => + prev.filter((i) => i.memory?.id !== memory.id), + ); + }} + /> + )} {type === "space" && ( - { - setSearchResults(prev => prev.filter(i => i.space?.id !== space.id)) - deleteSpace(space.id) - }} - /> + { + setSearchResults((prev) => + prev.filter((i) => i.space?.id !== space.id), + ); + deleteSpace(space.id); + }} + /> )} ))} @@ -218,11 +218,15 @@ const SpaceExitVariant: Variant = { }, }; -export function MemoryItem(props: StoredContent & { onDelete?: () => void; removeFromSpace?: () => Promise; }) { - - const { id, title, image, type, url, onDelete, removeFromSpace } = props +export function MemoryItem( + props: StoredContent & { + onDelete?: () => void; + removeFromSpace?: () => Promise; + }, +) { + const { id, title, image, type, url, onDelete, removeFromSpace } = props; - const { deleteMemory } = useMemory() + const { deleteMemory } = useMemory(); const name = title ? title.length > 10 @@ -230,9 +234,9 @@ export function MemoryItem(props: StoredContent & { onDelete?: () => void; remov : title : ""; - const [isDialogOpen, setIsDialogOpen] = useState(false); + const [isDialogOpen, setIsDialogOpen] = useState(false); - const [moreDropdownOpen, setMoreDropdownOpen] = useState(false) + const [moreDropdownOpen, setMoreDropdownOpen] = useState(false); const touchEventProps = useTouchHold({ onHold() { @@ -240,52 +244,81 @@ export function MemoryItem(props: StoredContent & { onDelete?: () => void; remov }, }); return ( - -
- { - type === "note" ? - ( - - - - ) : ( - - ) - } + +
+ {type === "note" ? ( + + + + ) : ( + + )} - {type === "page" ? - { deleteMemory(id); onDelete?.() }} url={url} /> : - type === "note" ? - setIsDialogOpen(true)} onDelete={() => { deleteMemory(id); onDelete?.() }} /> - : null} + {type === "page" ? ( + { + deleteMemory(id); + onDelete?.(); + }} + url={url} + /> + ) : type === "note" ? ( + setIsDialogOpen(true)} + onDelete={() => { + deleteMemory(id); + onDelete?.(); + }} + /> + ) : null} -
- {type === "page" ? ( - window.open(url)} - className="h-16 w-16" - id={id.toString()} - src={image!} - onError={(e) => { - (e.target as HTMLImageElement).src = - "/icons/white_without_bg.png"; - }} - /> - ) : type === "note" ? ( - setIsDialogOpen(true)} className="h-16 w-16" /> - ) : ( - <> - )} -
-
- - setIsDialogOpen(false)} memory={props} /> - - +
+ {type === "page" ? ( + window.open(url)} + className="h-16 w-16" + id={id.toString()} + src={image!} + onError={(e) => { + (e.target as HTMLImageElement).src = + "/icons/white_without_bg.png"; + }} + /> + ) : type === "note" ? ( + setIsDialogOpen(true)} className="h-16 w-16" /> + ) : ( + <> + )} +
+
+ + setIsDialogOpen(false)} + memory={props} + /> + + ); } @@ -314,20 +347,23 @@ export function SpaceItem({ const _name = name.length > 10 ? name.slice(0, 10) + "..." : name; - return ( - { onDelete(); return; @@ -412,7 +448,7 @@ export function SpaceItem({ /> {spaceMemories.length > 2 ? ( ) : spaceMemories.length > 1 ? ( ) : spaceMemories.length === 1 ? ( ) : ( -
+
)} ); @@ -454,15 +493,15 @@ export function SpaceMoreButton({ onDelete, isOpen, setIsOpen, - onEdit, + onEdit, }: { onDelete?: () => void; isOpen?: boolean; - onEdit?: () => void; + onEdit?: () => void; setIsOpen?: (open: boolean) => void; }) { return ( - + -); +}) => { + const handleClick = () => + setSelectedItem((prev) => (prev === label ? null : label)); + + return ( + + ); +}; export function SubSidebar({ children }: { children?: React.ReactNode }) { return ( @@ -176,7 +154,7 @@ export function SubSidebar({ children }: { children?: React.ReactNode }) { transition={{ duration: 0.2, }} - className="bg-rgray-3 border-r-rgray-6 absolute left-[100%] top-0 z-[10] hidden h-screen w-[30vw] items-start justify-center overflow-x-hidden border-r font-light md:flex" + className="absolute left-[100%] top-0 z-[10] hidden h-screen w-[30vw] items-start justify-center overflow-x-hidden border-r bg-stone-100 font-light md:flex" > ); } - -export function ProfileTab({ open }: { open: boolean }) { - const { data: session } = useSession(); - - const [tweetStat, setTweetStat] = useState<[number, number] | null>(); - const [memoryStat, setMemoryStat] = useState<[number, number] | null>(); - - const [loading, setLoading] = useState(true); - - useEffect(() => { - fetch("/api/getCount").then(async (resp) => { - const data = (await resp.json()) as any; - setTweetStat([data.tweetsCount, data.tweetsLimit]); - setMemoryStat([data.pageCount, data.pageLimit]); - setLoading(false); - }); - }, [open]); - - return ( -
-
-

Profile

-
- { - (e.target as HTMLImageElement).src = - "/icons/white_without_bg.png"; - }} - /> -
-

{session?.user?.name}

- {session?.user?.email} - -
-
-
-
-

- - Storage -

- {loading ? ( -
-
-
-
- ) : ( - <> -
-

- Memories -
- {memoryStat?.join("/")} -
-

-
-
0 ? "5%" : "0%", - }} - className="bg-rgray-5 h-full rounded-full" - /> -
-
-
-

- Tweets -
- {tweetStat?.join("/")} -
-

-
-
0 ? "5%" : "0%", - }} - className="bg-rgray-5 h-full rounded-full" - /> -
-
- - )} -
-
- ); -} diff --git a/apps/web/src/components/ui/avatar.tsx b/apps/web/src/components/ui/avatar.tsx index 47795451..b36abf28 100644 --- a/apps/web/src/components/ui/avatar.tsx +++ b/apps/web/src/components/ui/avatar.tsx @@ -39,7 +39,7 @@ const AvatarFallback = React.forwardRef< {children} - + Close @@ -58,10 +58,7 @@ const DialogHeader = ({ ...props }: React.HTMLAttributes) => (
); @@ -102,7 +99,7 @@ const DialogDescription = React.forwardRef< >(({ className, ...props }, ref) => ( )); diff --git a/apps/web/src/components/ui/dropdown-menu.tsx b/apps/web/src/components/ui/dropdown-menu.tsx index cbc5cb1e..fbe2d99c 100644 --- a/apps/web/src/components/ui/dropdown-menu.tsx +++ b/apps/web/src/components/ui/dropdown-menu.tsx @@ -47,7 +47,7 @@ const DropdownMenuSubContent = React.forwardRef< (({ className, ...props }, ref) => ( )); diff --git a/apps/web/src/components/ui/input.tsx b/apps/web/src/components/ui/input.tsx index f697d540..9d925512 100644 --- a/apps/web/src/components/ui/input.tsx +++ b/apps/web/src/components/ui/input.tsx @@ -11,7 +11,7 @@ const Input = React.forwardRef( ( return (
@@ -39,7 +39,7 @@ const InputWithIcon = React.forwardRef(