This commit is contained in:
Yash 2024-04-11 11:02:41 +00:00
commit fc3130d562
8 changed files with 444 additions and 11 deletions

View file

@ -4,4 +4,13 @@ interface Env {
SECURITY_KEY: string;
OPENAI_API_KEY: string;
GOOGLE_AI_API_KEY: string;
MY_QUEUE: Queue<TweetData>;
}
interface TweetData {
tweetText: string;
postUrl: string;
authorName: string;
handle: string;
time: string;
}

View file

@ -60,8 +60,9 @@ export async function POST(request: Request, _: CloudflareVectorizeStore, embedd
// if (responses.count === 0) {
// return new Response(JSON.stringify({ message: "No Results Found" }), { status: 404 });
// }
console.log(responses.matches);
const highScoreIds = responses.matches.filter(({ score }) => score > 0.35).map(({ id }) => id);
const highScoreIds = responses.matches.filter(({ score }) => score > 0.3).map(({ id }) => id);
if (sourcesOnly === 'true') {
return new Response(JSON.stringify({ ids: highScoreIds }), { status: 200 });

View file

@ -9,6 +9,10 @@ index_name = "any-vector"
[ai]
binding = "AI"
[[queues.producers]]
queue = "batch-vector-queue"
binding = "MY_QUEUE"
# Variable bindings. These are arbitrary, plaintext strings (similar to environment variables)
# Note: Use secrets to store sensitive data.
# Docs: https://developers.cloudflare.com/workers/platform/environment-variables

View file

@ -13,7 +13,7 @@ function App() {
null,
);
const doStuff = () => {
const getUserData = () => {
chrome.runtime.sendMessage({ type: "getJwt" }, (response) => {
const jwt = response.jwt;
const loginButton = document.getElementById("login");
@ -41,9 +41,69 @@ function App() {
};
useEffect(() => {
doStuff();
getUserData();
}, []);
// TODO: Implement getting bookmarks from API directly
// const [status, setStatus] = useState('');
// const [bookmarks, setBookmarks] = useState<TweetData[]>([]);
// const fetchBookmarks = (e: React.MouseEvent<HTMLButtonElement>) => {
// e.preventDefault();
// chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
// chrome.tabs.sendMessage(tabs[0].id!, { action: 'showProgressIndicator' });
// });
// chrome.tabs.create(
// { url: 'https://twitter.com/i/bookmarks/all' },
// function (tab) {
// chrome.tabs.onUpdated.addListener(function listener(tabId, info) {
// if (tabId === tab.id && info.status === 'complete') {
// chrome.tabs.onUpdated.removeListener(listener);
// chrome.runtime.sendMessage(
// { action: 'getAuthData' },
// function (response) {
// const authorizationHeader = response.authorizationHeader;
// const csrfToken = response.csrfToken;
// const cookies = response.cookies;
// if (authorizationHeader && csrfToken && cookies) {
// fetchAllBookmarks(authorizationHeader, csrfToken, cookies)
// .then((bookmarks) => {
// console.log('Bookmarks data:', bookmarks);
// setBookmarks(bookmarks);
// chrome.tabs.sendMessage(tabId, {
// action: 'hideProgressIndicator',
// });
// setStatus(
// `Fetched ${bookmarks.length} bookmarked tweets.`,
// );
// })
// .catch((error) => {
// console.error('Error:', error);
// chrome.tabs.sendMessage(tabId, {
// action: 'hideProgressIndicator',
// });
// setStatus(
// 'Error fetching bookmarks. Please check the console for details.',
// );
// });
// } else {
// chrome.tabs.sendMessage(tabId, {
// action: 'hideProgressIndicator',
// });
// setStatus('Missing authentication data');
// }
// },
// );
// }
// });
// },
// );
// };
return (
<div className="p-8">
<button
@ -69,6 +129,19 @@ function App() {
<h3>{userData.data.user.name}</h3>
<p>{userData.data.user.email}</p>
</div>
{/* TODO: Implement getting bookmarks from API directly */}
{/* <button onClick={(e) => fetchBookmarks(e)}>Fetch Bookmarks</button>
<div>{status}</div>
<div>
{bookmarks.map((bookmark) => (
<div key={bookmark.tweet_id}>
<p>{bookmark.author}</p>
<p>{bookmark.date}</p>
<p>{bookmark.full_text}</p>
</div>
))}
</div> */}
</div>
)}
</div>
@ -76,4 +149,160 @@ function App() {
);
}
// TODO: Implement getting bookmarks from API directly
// async function fetchAllBookmarks(
// authorizationHeader: string,
// csrfToken: string,
// cookies: string,
// ): Promise<TweetData[]> {
// const baseUrl =
// 'https://twitter.com/i/api/graphql/uJEL6XARgGmo2EAsO2Pfkg/Bookmarks';
// const params = new URLSearchParams({
// variables: JSON.stringify({
// count: 100,
// includePromotedContent: true,
// }),
// features: JSON.stringify({
// graphql_timeline_v2_bookmark_timeline: true,
// rweb_tipjar_consumption_enabled: false,
// responsive_web_graphql_exclude_directive_enabled: true,
// verified_phone_label_enabled: true,
// creator_subscriptions_tweet_preview_api_enabled: true,
// responsive_web_graphql_timeline_navigation_enabled: true,
// responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
// communities_web_enable_tweet_community_results_fetch: true,
// c9s_tweet_anatomy_moderator_badge_enabled: true,
// tweetypie_unmention_optimization_enabled: true,
// responsive_web_edit_tweet_api_enabled: true,
// graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
// view_counts_everywhere_api_enabled: true,
// longform_notetweets_consumption_enabled: true,
// responsive_web_twitter_article_tweet_consumption_enabled: true,
// tweet_awards_web_tipping_enabled: false,
// creator_subscriptions_quote_tweet_preview_enabled: false,
// freedom_of_speech_not_reach_fetch_enabled: true,
// standardized_nudges_misinfo: true,
// tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled:
// true,
// tweet_with_visibility_results_prefer_gql_media_interstitial_enabled:
// false,
// rweb_video_timestamps_enabled: true,
// longform_notetweets_rich_text_read_enabled: true,
// longform_notetweets_inline_media_enabled: true,
// responsive_web_enhance_cards_enabled: false,
// }),
// });
// const requestUrl = `${baseUrl}?${params}`;
// const headers = {
// Authorization: authorizationHeader,
// 'X-Csrf-Token': csrfToken,
// Cookie: cookies,
// };
// const bookmarks: TweetData[] = [];
// let nextCursor = null;
// let requestCount = 0;
// const maxRequestsPerWindow = 450;
// const windowDuration = 15 * 60 * 1000; // 15 minutes in milliseconds
// let windowStartTime = Date.now();
// do {
// if (nextCursor) {
// params.set(
// 'variables',
// JSON.stringify({
// count: 100,
// cursor: nextCursor,
// includePromotedContent: true,
// }),
// );
// }
// // Check if the rate limit is exceeded
// if (requestCount >= maxRequestsPerWindow) {
// const elapsedTime = Date.now() - windowStartTime;
// if (elapsedTime < windowDuration) {
// const waitTime = windowDuration - elapsedTime;
// await new Promise((resolve) => setTimeout(resolve, waitTime));
// }
// requestCount = 0;
// windowStartTime = Date.now();
// }
// try {
// const response = await fetch(requestUrl, {
// method: 'GET',
// headers: headers,
// });
// requestCount++;
// if (!response.ok) {
// throw new Error(`HTTP error! status: ${response.status}`);
// }
// const data = await response.json();
// const timeline = data.data.bookmark_timeline_v2.timeline;
// timeline.instructions.forEach(
// (instruction: {
// type: string;
// entries: {
// content: {
// entryType: string;
// itemContent: {
// tweet_results: {
// result: {
// legacy: {
// full_text: string;
// created_at: string;
// };
// core: {
// user_results: {
// result: {
// legacy: {
// screen_name: string;
// };
// };
// };
// };
// rest_id: string;
// };
// };
// };
// };
// }[];
// }) => {
// if (instruction.type === 'TimelineAddEntries') {
// instruction.entries.forEach((entry) => {
// if (entry.content.entryType === 'TimelineTimelineItem') {
// const tweet = entry.content.itemContent.tweet_results.result;
// const tweetData = {
// full_text: tweet.legacy.full_text,
// url: `https://twitter.com/${tweet.core.user_results.result.legacy.screen_name}/status/${tweet.rest_id}`,
// author: tweet.core.user_results.result.legacy.screen_name,
// date: tweet.legacy.created_at,
// tweet_id: tweet.rest_id,
// };
// bookmarks.push(tweetData);
// }
// });
// }
// },
// );
// nextCursor = timeline.instructions.find(
// (instruction: { type: string }) =>
// instruction.type === 'TimelineTerminateTimeline',
// )?.direction?.cursor;
// } catch (error) {
// console.error('Error fetching bookmarks:', error);
// throw error;
// }
// } while (nextCursor);
// return bookmarks;
// }
export default App;

View file

@ -25,20 +25,176 @@ function sendUrlToAPI() {
}
function SideBar() {
// TODO: Implement getting bookmarks from API directly
// chrome.runtime.onMessage.addListener(function (request) {
// if (request.action === 'showProgressIndicator') {
// // TODO: SHOW PROGRESS INDICATOR
// // showProgressIndicator();
// } else if (request.action === 'hideProgressIndicator') {
// // hideProgressIndicator();
// }
// });
const [savedWebsites, setSavedWebsites] = useState<string[]>([]);
const [isSendingData, setIsSendingData] = useState(false);
interface TweetData {
tweetText: string;
postUrl: string;
authorName: string;
handle: string;
time: string;
}
const fetchBookmarks = () => {
const tweets: TweetData[] = []; // Initialize an empty array to hold all tweet elements
const scrollInterval = 1000;
const scrollStep = 5000; // Pixels to scroll on each step
let previousTweetCount = 0;
let unchangedCount = 0;
const scrollToEndIntervalID = setInterval(() => {
window.scrollBy(0, scrollStep);
const currentTweetCount = tweets.length;
if (currentTweetCount === previousTweetCount) {
unchangedCount++;
if (unchangedCount >= 2) {
// Stop if the count has not changed 5 times
console.log("Scraping complete");
console.log("Total tweets scraped: ", tweets.length);
console.log("Downloading tweets as JSON...");
clearInterval(scrollToEndIntervalID); // Stop scrolling
observer.disconnect(); // Stop observing DOM changes
downloadTweetsAsJson(tweets); // Download the tweets list as a JSON file
}
} else {
unchangedCount = 0; // Reset counter if new tweets were added
}
previousTweetCount = currentTweetCount; // Update previous count for the next check
}, scrollInterval);
function updateTweets() {
document
.querySelectorAll('article[data-testid="tweet"]')
.forEach((tweetElement) => {
const authorName = (
tweetElement.querySelector(
'[data-testid="User-Name"]',
) as HTMLElement
)?.innerText;
const handle = (
tweetElement.querySelector('[role="link"]') as HTMLLinkElement
).href
.split("/")
.pop();
const tweetText = (
tweetElement.querySelector(
'[data-testid="tweetText"]',
) as HTMLElement
)?.innerText;
const time = (
tweetElement.querySelector("time") as HTMLTimeElement
).getAttribute("datetime");
const postUrl = (
tweetElement.querySelector(
".css-175oi2r.r-18u37iz.r-1q142lx a",
) as HTMLLinkElement
)?.href;
const isTweetNew = !tweets.some((tweet) => tweet.postUrl === postUrl);
if (isTweetNew) {
tweets.push({
authorName,
handle: handle ?? "",
tweetText,
time: time ?? "",
postUrl,
});
console.log("Tweets capturados: ", tweets.length);
}
});
}
// Initially populate the tweets array
updateTweets();
// Create a MutationObserver to observe changes in the DOM
const observer = new MutationObserver((mutations) => {
mutations.forEach((mutation) => {
if (mutation.addedNodes.length) {
updateTweets(); // Call updateTweets whenever new nodes are added to the DOM
}
});
});
// Start observing the document body for child list changes
observer.observe(document.body, { childList: true, subtree: true });
function downloadTweetsAsJson(tweetsArray: TweetData[]) {
const jsonData = JSON.stringify(tweetsArray); // Convert the array to JSON
// TODO: SEND jsonData to server
console.log(jsonData);
}
};
return (
<>
<TooltipProvider>
<div className="anycontext-flex anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
{/* <Tooltip delayDuration={300}>
<TooltipContent side="left">
<p>Open Sidebar</p>
</TooltipContent>
</Tooltip> */}
<div className="anycontext-flex anycontext-group anycontext-flex-col anycontext-gap-2 anycontext-fixed anycontext-bottom-12 anycontext-right-0 anycontext-z-[99999] anycontext-font-sans">
{window.location.href.includes("twitter.com") ||
window.location.href.includes("x.com") ? (
<Tooltip delayDuration={300}>
<TooltipTrigger
className="anycontext-bg-transparent
anycontext-border-none anycontext-m-0 anycontext-p-0"
>
<button
onClick={() => {
if (window.location.href.endsWith("/i/bookmarks/all")) {
fetchBookmarks();
} else {
window.location.href =
"https://twitter.com/i/bookmarks/all";
setTimeout(() => {
fetchBookmarks();
}, 2500);
}
}}
className="anycontext-open-button disabled:anycontext-opacity-30 anycontext-bg-transparent
anycontext-border-none anycontext-m-0 anycontext-p-0"
>
<svg
xmlns="http://www.w3.org/2000/svg"
fill="none"
viewBox="0 0 24 24"
strokeWidth={1.5}
stroke="currentColor"
className="anycontext-w-6 anycontext-h-6"
>
<path
strokeLinecap="round"
strokeLinejoin="round"
d="M17.593 3.322c1.1.128 1.907 1.077 1.907 2.185V21L12 17.25 4.5 21V5.507c0-1.108.806-2.057 1.907-2.185a48.507 48.507 0 0 1 11.186 0Z"
/>
</svg>
</button>
</TooltipTrigger>
<TooltipContent className="anycontext-p-0" side="left">
<p className="anycontext-p-0 anycontext-m-0">
Import twitter bookmarks
</p>
</TooltipContent>
</Tooltip>
) : (
<></>
)}
<Tooltip delayDuration={300}>
<TooltipTrigger
className="anycontext-bg-transparent

View file

@ -5,6 +5,30 @@ const backendUrl =
? "http://localhost:3000"
: "https://supermemory.dhr.wtf";
// TODO: Implement getting bookmarks from API directly
// let authorizationHeader: string | null = null;
// let csrfToken: string | null = null;
// let cookies: string | null = null;
// chrome.webRequest.onBeforeSendHeaders.addListener(
// (details) => {
// for (let i = 0; i < details.requestHeaders!.length; ++i) {
// const header = details.requestHeaders![i];
// if (header.name.toLowerCase() === 'authorization') {
// authorizationHeader = header.value || null;
// } else if (header.name.toLowerCase() === 'x-csrf-token') {
// csrfToken = header.value || null;
// } else if (header.name.toLowerCase() === 'cookie') {
// cookies = header.value || null;
// }
// console.log(header, authorizationHeader, csrfToken, cookies)
// }
// },
// { urls: ['https://twitter.com/*', 'https://x.com/*'] },
// ['requestHeaders']
// );
chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
if (request.type === "getJwt") {
chrome.storage.local.get(["jwt"], ({ jwt }) => {
@ -73,4 +97,12 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
return true;
})();
}
// TODO: Implement getting bookmarks from API directly
// else if (request.action === 'getAuthData') {
// sendResponse({
// authorizationHeader: authorizationHeader,
// csrfToken: csrfToken,
// cookies: cookies
// });
// }
});

View file

@ -34,7 +34,7 @@ CREATE TABLE `session` (
--> statement-breakpoint
CREATE TABLE `space` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`name` text DEFAULT 'all' NOT NULL,
`name` text DEFAULT 'none' NOT NULL,
`user` text(255),
FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action
);
@ -47,6 +47,7 @@ CREATE TABLE `storedContent` (
`url` text NOT NULL,
`savedAt` integer NOT NULL,
`baseUrl` text(255),
`type` text DEFAULT 'page',
`image` text(255),
`user` text(255),
FOREIGN KEY (`user`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE no action
@ -69,6 +70,7 @@ CREATE TABLE `verificationToken` (
--> statement-breakpoint
CREATE INDEX `account_userId_idx` ON `account` (`userId`);--> statement-breakpoint
CREATE INDEX `session_userId_idx` ON `session` (`userId`);--> statement-breakpoint
CREATE UNIQUE INDEX `space_name_unique` ON `space` (`name`);--> statement-breakpoint
CREATE INDEX `spaces_name_idx` ON `space` (`name`);--> statement-breakpoint
CREATE INDEX `spaces_user_idx` ON `space` (`user`);--> statement-breakpoint
CREATE INDEX `storedContent_url_idx` ON `storedContent` (`url`);--> statement-breakpoint

View file

@ -67,7 +67,7 @@ export async function POST(req: NextRequest) {
let storeToSpace = data.space;
if (!storeToSpace) {
storeToSpace = "all";
storeToSpace = "none";
}
const storedContentId = await db.insert(storedContent).values({